diff --git a/.gitignore b/.gitignore index 5abfb5e0c..a84039004 100755 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ app/Lib/EventWarning/Custom/* !/app/files/misp-objects/* !/app/files/misp-decaying-models !/app/files/misp-decaying-models/* +!/app/files/misp-workflow-blueprints +!/app/files/misp-workflow-blueprints/* /app/files/scripts/*.pyc /app/files/scripts/*.py~ /app/files/scripts/__pycache__ diff --git a/.gitmodules b/.gitmodules index c7a9da43c..f40a1d272 100644 --- a/.gitmodules +++ b/.gitmodules @@ -48,3 +48,6 @@ [submodule "app/files/scripts/python-maec"] path = app/files/scripts/python-maec url = https://github.com/MAECProject/python-maec +[submodule "app/files/misp-workflow-blueprints"] + path = app/files/misp-workflow-blueprints + url = https://github.com/MISP/misp-workflow-blueprints diff --git a/PyMISP b/PyMISP index cd4b5d533..962b296f0 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit cd4b5d533b68de19b714be7f83f231640404f0bf +Subproject commit 962b296f0c2de3366deaf69ce3484187f255f5e0 diff --git a/VERSION.json b/VERSION.json index 4c4d719a4..31cc5a873 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":159} +{"major":2, "minor":4, "hotfix":160} diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 3d8d11e8e..ff04aa230 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -16,8 +16,9 @@ App::uses('JsonTool', 'Tools'); */ class AdminShell extends AppShell { - public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint'); - + public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation'); + public $tasks = array('ConfigLoad'); + public function getOptionParser() { $parser = parent::getOptionParser(); @@ -99,6 +100,7 @@ class AdminShell extends AppShell public function jobGenerateCorrelation() { + $this->ConfigLoad->execute(); if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL); } @@ -1186,4 +1188,38 @@ class AdminShell extends AppShell ); $this->out($message); } + + public function truncateTable() + { + $this->ConfigLoad->execute(); + if (!isset($this->args[0])) { + die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL); + } + $userId = $this->args[0]; + if ($userId) { + $user = $this->User->getAuthUser($userId); + } else { + $user = [ + 'id' => 0, + 'email' => 'SYSTEM', + 'Organisation' => [ + 'name' => 'SYSTEM' + ] + ]; + } + if (empty($this->args[1])) { + die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL); + } + if (!empty($this->args[2])) { + $jobId = $this->args[2]; + } + $table = trim($this->args[1]); + $this->Correlation->truncate($user, $table); + if ($jobId) { + $this->Job->id = $jobId; + $this->Job->saveField('progress', 100); + $this->Job->saveField('date_modified', date("Y-m-d H:i:s")); + $this->Job->saveField('message', __('Database truncated: ' . $table)); + } + } } diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index 86c20828f..30aa4056b 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -40,6 +40,9 @@ class EventShell extends AppShell 'event_id' => ['help' => __('Event ID'), 'required' => true], 'user_id' => ['help' => __('User ID'), 'required' => true], ], + 'options' => [ + 'send' => ['help' => __('Send email to given user'), 'boolean' => true], + ], ], ]); $parser->addSubcommand('duplicateTags', [ @@ -607,6 +610,7 @@ class EventShell extends AppShell public function testEventNotificationEmail() { list($eventId, $userId) = $this->args; + $send = $this->param('send'); $user = $this->getUser($userId); $eventForUser = $this->Event->fetchEvent($user, [ @@ -626,10 +630,16 @@ class EventShell extends AppShell App::uses('SendEmail', 'Tools'); App::uses('GpgTool', 'Tools'); $sendEmail = new SendEmail(GpgTool::initializeGpg()); - $sendEmail->setTransport('Debug'); + if (!$send) { + $sendEmail->setTransport('Debug'); + } $result = $sendEmail->sendToUser(['User' => $user], null, $emailTemplate); - echo $result['contents']['headers'] . "\n\n" . $result['contents']['message'] . "\n"; + if ($send) { + var_dump($result); + } else { + echo $result['contents']['headers'] . "\n\n" . $result['contents']['message'] . "\n"; + } } /** diff --git a/app/Console/Command/Ls22Shell.php b/app/Console/Command/Ls22Shell.php index 63a945bbe..e5791abcb 100644 --- a/app/Console/Command/Ls22Shell.php +++ b/app/Console/Command/Ls22Shell.php @@ -52,6 +52,55 @@ class Ls22Shell extends AppShell ), ), ]); + $parser->addSubcommand('checkSyncConnections', [ + 'help' => __('Check the given sync connection(s) for the given server(s).'), + 'parser' => array( + 'options' => array( + 'instances' => [ + 'help' => 'Path to the instance file, by default "instances.csv" from the local directory', + 'short' => 'i', + 'required' => true + ], + 'misp_url_filter' => [ + 'help' => 'The url of the instance to execute changes on. If not set, all are updated.', + 'short' => 'm', + 'required' => false + ], + 'synced_misp_url_filter' => [ + 'help' => 'The sync connection to modify on each valid instance (as selected by the misp_url_filter). If not set, all sync connections on the selected instances will be updated.', + 'short' => 's', + 'required' => false + ] + ), + ), + ]); + $parser->addSubcommand('modifySyncConnection', [ + 'help' => __('Modify sync connection(s).'), + 'parser' => array( + 'options' => array( + 'instances' => [ + 'help' => 'Path to the instance file, by default "instances.csv" from the local directory', + 'short' => 'i', + 'required' => true + ], + 'misp_url_filter' => [ + 'help' => 'The url of the instance to execute changes on. If not set, all are updated.', + 'short' => 'm', + 'required' => false + ], + 'synced_misp_url_filter' => [ + 'help' => 'The sync connection to modify on each valid instance (as selected by the misp_url_filter). If not set, all sync connections on the selected instances will be updated.', + 'short' => 's', + 'required' => false + ], + 'json' => [ + 'help' => 'JSON delta to push (such as \'{"push": 1}\').', + 'short' => 'j', + 'required' => true + ] + ), + ), + ]); $parser->addSubcommand('addWarninglist', [ 'help' => __('Inject warninglist'), 'parser' => array( @@ -104,6 +153,11 @@ class Ls22Shell extends AppShell 'help' => 'Upper bound of the date. Accepts timestamp or date distance (such as 1d or 5h). Defaults to unbounded.', 'short' => 't', 'required' => false + ], + 'org' => [ + 'help' => 'Name the org that should be evaluated. If not set, all will be included.', + 'short' => 'o', + 'required' => false ] ), ), @@ -111,6 +165,103 @@ class Ls22Shell extends AppShell return $parser; } + public function checkSyncConnections() + { + $this->__getInstances($this->param('instances')); + $results = []; + $instanceFilter = $this->param('misp_url_filter'); + $syncedInstanceFilter = $this->param('synced_misp_url_filter'); + foreach ($this->__servers as $server) { + if (!empty($instanceFilter) && strtolower(trim($server['Server']['url'])) !== strtolower(trim($instanceFilter))) { + continue; + } + $HttpSocket = $this->Server->setupHttpSocket($server, null); + $request = $this->Server->setupSyncRequest($server, 'Server'); + $start_time = microtime(true); + $response = $HttpSocket->get($server['Server']['url'] . '/servers/index', false, $request); + $baseline = round((microtime(true) - $start_time) * 1000); + if (!$response->isOk()) { + $this->out($server['Server']['url'] . ': ' . 'Connection or auth failed', 1, Shell::NORMAL); + continue; + } + $synced_servers = json_decode($response->body, true); + foreach ($synced_servers as $synced_server) { + $success = false; + if (empty($syncedInstanceFilter) || strtolower($synced_server['Server']['url']) === strtolower($syncedInstanceFilter)) { + $start_time = microtime(true); + $response = $HttpSocket->get($server['Server']['url'] . '/servers/testConnection/' . $synced_server['Server']['id'], '{}', $request); + $execution_time = round((microtime(true) - $start_time) * 1000) - $baseline; + if ($response->isOk()) { + $success = true; + } + $this->out( + sprintf( + '%s connection to %s: %s (%sms)', + $server['Server']['url'], + $synced_server['Server']['url'], + sprintf( + '<%s>%s', + $success ? 'info' : 'error', + $success ? 'Success' : 'Failed', + $success ? 'info' : 'error' + ), + $execution_time + ), + 1, + Shell::NORMAL + ); + } + } + } + } + + public function modifySyncConnection() + { + $this->__getInstances($this->param('instances')); + $results = []; + $instanceFilter = $this->param('misp_url_filter'); + $syncedInstanceFilter = $this->param('synced_misp_url_filter'); + $json = $this->param('json'); + foreach ($this->__servers as $server) { + if (!empty($instanceFilter) && strtolower(trim($server['Server']['url'])) !== strtolower(trim($instanceFilter))) { + continue; + } + $HttpSocket = $this->Server->setupHttpSocket($server, null); + $request = $this->Server->setupSyncRequest($server, 'Server'); + $response = $HttpSocket->get($server['Server']['url'] . '/servers/index', false, $request); + if (!$response->isOk()) { + $this->out($server['Server']['url'] . ': ' . 'Connection or auth failed', 1, Shell::NORMAL); + } + $synced_servers = json_decode($response->body, true); + $success = false; + foreach ($synced_servers as $synced_server) { + if (empty($syncedInstanceFilter) || strtolower($synced_server['Server']['url']) === strtolower($syncedInstanceFilter)) { + debug($json); + $response = $HttpSocket->post($server['Server']['url'] . '/servers/edit/' . $synced_server['Server']['id'], $json, $request); + debug($response->body); + if ($response->isOk()) { + $success = true; + } + $this->out( + sprintf( + '%s connection to %s: %s', + $server['Server']['url'], + $synced_server['Server']['url'], + sprintf( + '<%s>%s', + $success ? 'info' : 'error', + $success ? 'Success' : 'Failed', + $success ? 'info' : 'error' + ) + ), + 1, + Shell::NORMAL + ); + } + } + } + } + public function enableTaxonomy() { $taxonomyToEnable = $this->param('taxonomy'); @@ -162,16 +313,29 @@ class Ls22Shell extends AppShell $HttpSocket = $this->Server->setupHttpSocket($server, null); $request = $this->Server->setupSyncRequest($server, 'Server'); $start_time = microtime(true); - $response = $HttpSocket->get($server['Server']['url'] . '/users/view/me', false, $request); - $execution_time = round((microtime(true) - $start_time) * 1000); - $statusWrapped = sprintf( - '<%s>%s', - $response->isOk() ? 'info' : 'error', - $response->isOk() ? 'OK (' . $execution_time . 'ms)' : 'Failed. (' . $response->code . ')', - $response->isOk() ? 'info' : 'error' - ); + $fatal_error = false; + try { + $response = $HttpSocket->get($server['Server']['url'] . '/users/view/me', false, $request); + } catch (Exception $e) { + $fatal_error = true; + echo "\x07"; + $statusWrapped = sprintf( + '%s %s: %s', + 'Something went wrong while trying to reach', + $server['Server']['url'], + $e->getMessage() + ); + } + if (!$fatal_error) { + $execution_time = round((microtime(true) - $start_time) * 1000); + $statusWrapped = sprintf( + '<%s>%s', + $response->isOk() ? 'info' : 'error', + $response->isOk() ? 'OK (' . $execution_time . 'ms)' : 'Failed. (' . $response->code . ')', + $response->isOk() ? 'info' : 'error' + ); + } $this->out($server['Server']['url'] . ': ' . $statusWrapped, 1, Shell::NORMAL); - $results[$server['Server']['url']] = $response->isOk() ? $execution_time : false; } } @@ -215,28 +379,30 @@ class Ls22Shell extends AppShell } $HttpSocket = $this->Server->setupHttpSocket($server, null); $request = $this->Server->setupSyncRequest($server); - $response = $HttpSocket->get($server['Server']['url'] . '/organisations/index', false, $request); + $response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:all', false, $request); $orgs = json_decode($response->body(), true); $this->out(__('Organisations fetched. %d found.', count($orgs)), 1, Shell::VERBOSE); $org_mapping = []; foreach ($orgs as $org) { - $name = explode(' ', $org['Organisation']['name']); - if ($name[0] !== 'BT') { + if (!empty($this->param('org')) && $org['Organisation']['name'] !== $this->param('org')) { + continue; + } + if ($org['Organisation']['name'] === 'YT') { continue; } $org_mapping[$org['Organisation']['name']] = $org['Organisation']['id']; } + if (!empty($this->param['from'])) { + $time_range[] = $this->param['from']; + } + if (!empty($this->param['to'])) { + if (empty($time_range)) { + $time_range[] = '365d'; + } + $time_range[] = $this->param['to']; + } foreach ($org_mapping as $org_name => $org_id) { $time_range = []; - if (!empty($this->param['from'])) { - $time_range[] = $this->param['from']; - } - if (!empty($this->param['to'])) { - if (empty($time_range)) { - $time_range[] = '365d'; - } - $time_range[] = $this->param['to']; - } $params = [ 'org' => $org_id ]; @@ -256,7 +422,8 @@ class Ls22Shell extends AppShell 'other' => 0, 'attribute_attack' => 0, 'attribute_other' => 0, - 'score' => 0 + 'score' => 0, + 'warnings' => 0 ]; foreach ($events['response'] as $event) { if (!empty($event['Event']['Tag'])) { @@ -290,6 +457,9 @@ class Ls22Shell extends AppShell } } } + if (!empty($attribute['warnings'])) { + $result[$org_name]['warnings'] += 1; + } } $results[$org_name]['attribute_count'] += count($event['Event']['Attribute']); if (!empty($event['Event']['Object'])) { @@ -319,11 +489,18 @@ class Ls22Shell extends AppShell foreach ($results as $k => $result) { $totalCount = $result['attribute_count'] + $result['object_count']; if ($totalCount) { + if (empty($result['warnings'])) { + $results[$k]['metrics']['warnings'] = 100; + } else if (100 * $result['warnings'] < $result['attribute_count']) { + $results[$k]['metrics']['warnings'] = 50; + } else { + $results[$k]['metrics']['warnings'] = 0; + } $results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count'])); $results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']); $results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']); } - foreach (['connectedness', 'attack_weight', 'other_weight'] as $metric) { + foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings'] as $metric) { if (empty($results[$k]['metrics'][$metric])) { $results[$k]['metrics'][$metric] = 0; } @@ -331,8 +508,17 @@ class Ls22Shell extends AppShell $results[$k]['metrics'][$metric] = 100; } } - $results[$k]['score'] = round(40 * $results[$k]['metrics']['connectedness'] + 40 * $results[$k]['metrics']['attack_weight'] + 20 * $results[$k]['metrics']['other_weight']) / 100; - $scores[$k] = $results[$k]['score']; + $results[$k]['score'] = round( + 20 * $results[$k]['metrics']['warnings'] + + 20 * $results[$k]['metrics']['connectedness'] + + 40 * $results[$k]['metrics']['attack_weight'] + + 20 * $results[$k]['metrics']['other_weight'] + ) / 100; + $scores[$k]['total'] = $results[$k]['score']; + $scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']); + $scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']); + $scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']); + $scores[$k]['other_weight'] = round(20 * $results[$k]['metrics']['other_weight']); } arsort($scores, SORT_DESC); $this->out(str_repeat('=', 128), 1, Shell::NORMAL); @@ -344,18 +530,34 @@ class Ls22Shell extends AppShell ), 1, Shell::NORMAL); $this->out(str_repeat('=', 128), 1, Shell::NORMAL); foreach ($scores as $org => $score) { - $score_string = str_repeat('█', round($score)); + $score_string[0] = str_repeat('█', round($score['warnings']/100)); + $score_string[1] = str_repeat('█', round($score['connectedness']/100)); + $score_string[2] = str_repeat('█', round($score['attack_weight']/100)); + $score_string[3] = str_repeat('█', round($score['other_weight']/100)); $this->out(sprintf( '| %s | %s | %s |', str_pad($org, 10, ' ', STR_PAD_RIGHT), sprintf( - '%s%s', - $score_string, - str_repeat(' ', 100 - mb_strlen($score_string)) + '%s%s%s%s%s', + $score_string[0], + $score_string[1], + $score_string[2], + $score_string[3], + str_repeat(' ', 100 - mb_strlen(implode('', $score_string))) ), - str_pad($score . '%', 8, ' ', STR_PAD_RIGHT) + str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT) ), 1, Shell::NORMAL); } $this->out(str_repeat('=', 128), 1, Shell::NORMAL); + $this->out(sprintf( + '| Legend: %s %s %s %s %s |', + '█: Warnings', + '█: Connectedness', + '█: ATT&CK context', + '█: Other Context', + str_repeat(' ', 52) + ), 1, Shell::NORMAL); + $this->out(str_repeat('=', 128), 1, Shell::NORMAL); + file_put_contents(APP . 'tmp/report.json', json_encode($results, JSON_PRETTY_PRINT)); } } diff --git a/app/Console/Command/WorkflowShell.php b/app/Console/Command/WorkflowShell.php new file mode 100644 index 000000000..24a283f02 --- /dev/null +++ b/app/Console/Command/WorkflowShell.php @@ -0,0 +1,77 @@ +ConfigLoad->execute(); + if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) { + die(__('Invalid number of arguments.')); + } + + $trigger_id = $this->args[0]; + $data = JsonTool::decode($this->args[1]); + $logging = JsonTool::decode($this->args[2]); + $jobId = $this->args[3]; + + $blockingErrors = []; + $executionSuccess = $this->Workflow->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors); + + $job = $this->Job->read(null, $jobId); + $job['Job']['progress'] = 100; + $job['Job']['status'] = Job::STATUS_COMPLETED; + $job['Job']['date_modified'] = date("Y-m-d H:i:s"); + if ($executionSuccess) { + $job['Job']['message'] = __('Workflow for trigger `%s` completed execution', $trigger_id); + } else { + $errorMessage = implode(', ', $blockingErrors); + $message = __('Error while executing workflow for trigger `%s`: %s. %s%s', $trigger_id, $logging['message'], PHP_EOL . __('Returned message: %s', $errorMessage)); + $job['Job']['message'] = $message; + } + $this->Job->save($job); + } + + public function walkGraph() + { + $this->ConfigLoad->execute(); + if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) { + die(__('Invalid number of arguments.')); + } + + $workflow_id = (int)$this->args[0]; + $workflow = $this->Workflow->fetchWorkflow($workflow_id); + $node_id_to_exec = (int)$this->args[1]; + $roamingData = JsonTool::decode($this->args[2]); + $for_path = $this->args[3]; + $jobId = $this->args[4]; + + $concurrentErrors = []; + $walkResult = []; + $executionSuccess = $this->Workflow->walkGraph( + $workflow, + $node_id_to_exec, + $for_path, + $roamingData, + $concurrentErrors, + $walkResult + ); + $job = $this->Job->read(null, $jobId); + $job['Job']['progress'] = 100; + $job['Job']['status'] = Job::STATUS_COMPLETED; + $job['Job']['date_modified'] = date("Y-m-d H:i:s"); + if ($executionSuccess) { + $job['Job']['message'] = __('Workflow concurrent task executed %s nodes starting from node %s.', count($walkResult['executed_nodes']), $node_id_to_exec); + } else { + $message = __('Error while executing workflow concurrent task. %s', PHP_EOL . implode(', ', $concurrentErrors)); + $this->Workflow->logExecutionError($workflow, $message); + $job['Job']['message'] = $message; + } + $this->Job->save($job); + } +} diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 5308c9cda..e9a9486ce 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -4,6 +4,7 @@ App::uses('Controller', 'Controller'); App::uses('File', 'Utility'); App::uses('RequestRearrangeTool', 'Tools'); App::uses('BlowfishConstantPasswordHasher', 'Controller/Component/Auth'); +App::uses('BetterCakeEventManager', 'Tools'); /** * Application Controller @@ -34,8 +35,8 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '141'; - public $pyMispVersion = '2.4.159'; + private $__queryVersion = '143'; + public $pyMispVersion = '2.4.160'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; @@ -1259,17 +1260,17 @@ class AppController extends Controller ]); } } + /** @var TmpFileTool $final */ $final = $model->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView); - if (!empty($renderView) && !empty($final)) { + if ($renderView) { $this->layout = false; $final = json_decode($final->intoString(), true); - foreach ($final as $key => $data) { - $this->set($key, $data); - } + $this->set($final); $this->render('/Events/module_views/' . $renderView); } else { $filename = $this->RestSearch->getFilename($filters, $scope, $responseType); - return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType)); + $headers = ['X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType]; + return $this->RestResponse->viewData($final, $responseType, false, true, $filename, $headers); } } @@ -1443,6 +1444,16 @@ class AppController extends Controller return parent::_getViewObject(); } + public function getEventManager() + { + if (empty($this->_eventManager)) { + $this->_eventManager = new BetterCakeEventManager(); + $this->_eventManager->attach($this->Components); + $this->_eventManager->attach($this); + } + return $this->_eventManager; + } + /** * Close session without writing changes to them and return current user. * @return array @@ -1462,18 +1473,34 @@ class AppController extends Controller protected function _jsonDecode($dataToDecode) { try { - if (defined('JSON_THROW_ON_ERROR')) { - // JSON_THROW_ON_ERROR is supported since PHP 7.3 - return json_decode($dataToDecode, true, 512, JSON_THROW_ON_ERROR); - } else { - $decoded = json_decode($dataToDecode, true); - if ($decoded === null) { - throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error()); - } - return $decoded; - } + return JsonTool::decode($dataToDecode); } catch (Exception $e) { throw new HttpException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.', 405, $e); } } + + /** + * Mimics what PaginateComponent::paginate() would do, when Model::paginate() is not called + * + * @param integer $page + * @param integer $limit + * @param integer $current + * @param string $type + * @return void + */ + protected function __setPagingParams(int $page, int $limit, int $current, string $type = 'named') + { + $this->request->params['paging'] = [ + 'Correlation' => [ + 'page' => $page, + 'limit' => $limit, + 'current' => $current, + 'pageCount' => 0, + 'prevPage' => $page > 1, + 'nextPage' => $current >= $limit, + 'options' => [], + 'paramType' => $type + ] + ]; + } } diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 309586fbe..8abc3e7cb 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -84,10 +84,24 @@ class AttributesController extends AppController 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], ]); $orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc'); + $sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user()); foreach ($attributes as &$attribute) { if (isset($orgTable[$attribute['Event']['orgc_id']])) { $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']]; } + $temp = $this->Attribute->Correlation->getRelatedAttributes( + $this->Auth->user(), + $sgids, + $attribute['Attribute'], + [], + true + ); + foreach ($temp as &$t) { + $t['info'] = $t['Event']['info']; + $t['org_id'] = $t['Event']['org_id']; + $t['date'] = $t['Event']['date']; + } + $attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp; } list($attributes, $sightingsData) = $this->__searchUI($attributes); @@ -732,7 +746,13 @@ class AttributesController extends AppController } } $dateObj = new DateTime(); - $existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']); + $existingAttribute = $this->Attribute->find('first', [ + 'conditions' => [ + 'Attribute.uuid' => $this->Attribute->data['Attribute']['uuid'] + ], + 'recursive' => -1 + ] + ); // check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation) // check which attribute is newer if (count($existingAttribute) && !$existingAttribute['Attribute']['deleted']) { @@ -828,7 +848,10 @@ class AttributesController extends AppController } } } else { - $this->request->data = $this->Attribute->read(null, $id); + $this->request->data = $this->Attribute->find('first', [ + 'recursive' => -1, + 'conditions' => ['Attribute.id' => $id] + ]); } $this->set('attribute', $this->request->data); if (!empty($this->request->data['Attribute']['object_id'])) { @@ -877,7 +900,9 @@ class AttributesController extends AppController // ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur. public function editField($id) { - $attribute = $this->__fetchAttribute($id); + $attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [ + 'conditions' => ['Attribute.id' => $id], + ]); if (empty($attribute)) { return new CakeResponse(array('body'=> json_encode(array('fail' => false, 'errors' => 'Invalid attribute')), 'status' => 200, 'type' => 'json')); } @@ -1459,57 +1484,63 @@ class AttributesController extends AppController return new CakeResponse(array('body'=> json_encode(array('saved' => true)), 'status' => 200, 'type' => 'json')); } + private function __getSearchFilters(&$exception) + { + if (isset($this->request->data['Attribute'])) { + $this->request->data = $this->request->data['Attribute']; + } + $checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen'); + foreach ($checkForEmpty as $field) { + if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') { + unset($this->request->data[$field]); + } + } + if (empty($this->request->data['to_ids'])) { + unset($this->request->data['to_ids']); + $this->request->data['ignore'] = 1; + } + $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen'); + $filterData = array( + 'request' => $this->request, + 'named_params' => $this->request->params['named'], + 'paramArray' => $paramArray, + 'additional_delimiters' => PHP_EOL + ); + $exception = false; + $filters = $this->_harvestParameters($filterData, $exception); + if (!empty($filters['uuid'])) { + if (!is_array($filters['uuid'])) { + $filters['uuid'] = array($filters['uuid']); + } + $uuid = array(); + $ids = array(); + foreach ($filters['uuid'] as $k => $filter) { + if ($filter[0] === '!') { + $filter = substr($filter, 1); + } + if (Validation::uuid($filter)) { + $uuid[] = $filters['uuid'][$k]; + } else { + $ids[] = $filters['uuid'][$k]; + } + } + if (empty($uuid)) { + unset($filters['uuid']); + } else { + $filters['uuid'] = $uuid; + } + if (!empty($ids)) { + $filters['eventid'] = $ids; + } + } + return $filters; + } + public function search($continue = false) { + $exception = null; + $filters = $this->__getSearchFilters($exception); if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) { - if (isset($this->request->data['Attribute'])) { - $this->request->data = $this->request->data['Attribute']; - } - $checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen'); - foreach ($checkForEmpty as $field) { - if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') { - unset($this->request->data[$field]); - } - } - if (empty($this->request->data['to_ids'])) { - unset($this->request->data['to_ids']); - $this->request->data['ignore'] = 1; - } - $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen'); - $filterData = array( - 'request' => $this->request, - 'named_params' => $this->request->params['named'], - 'paramArray' => $paramArray, - 'additional_delimiters' => PHP_EOL - ); - $exception = false; - $filters = $this->_harvestParameters($filterData, $exception); - if (!empty($filters['uuid'])) { - if (!is_array($filters['uuid'])) { - $filters['uuid'] = array($filters['uuid']); - } - $uuid = array(); - $ids = array(); - foreach ($filters['uuid'] as $k => $filter) { - if ($filter[0] === '!') { - $filter = substr($filter, 1); - } - if (Validation::uuid($filter)) { - $uuid[] = $filters['uuid'][$k]; - } else { - $ids[] = $filters['uuid'][$k]; - } - } - if (empty($uuid)) { - unset($filters['uuid']); - } else { - $filters['uuid'] = $uuid; - } - if (!empty($ids)) { - $filters['eventid'] = $ids; - } - } - unset($filterData); if ($filters === false) { return $exception; } @@ -1536,7 +1567,8 @@ class AttributesController extends AppController $this->Session->write('search_attributes_filters', null); } - if (isset($filters)) { + if (!empty($filters)) { + $filters['includeCorrelations'] = 1; $params = $this->Attribute->restSearch($this->Auth->user(), 'json', $filters, true); if (!isset($params['conditions']['Attribute.deleted'])) { $params['conditions']['Attribute.deleted'] = 0; @@ -1555,6 +1587,7 @@ class AttributesController extends AppController 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], ]); $orgTable = array_column(array_column($orgTable, 'Orgc'), null, 'id'); + $sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user()); foreach ($attributes as &$attribute) { if (isset($orgTable[$attribute['Event']['orgc_id']])) { $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']]; @@ -1562,6 +1595,21 @@ class AttributesController extends AppController if (isset($orgTable[$attribute['Event']['org_id']])) { $attribute['Event']['Org'] = $orgTable[$attribute['Event']['org_id']]; } + if (isset($filters['includeCorrelations'])) { + $temp = $this->Attribute->Correlation->getRelatedAttributes( + $this->Auth->user(), + $sgids, + $attribute['Attribute'], + [], + true + ); + foreach ($temp as &$t) { + $t['info'] = $t['Event']['info']; + $t['org_id'] = $t['Event']['org_id']; + $t['date'] = $t['Event']['date']; + } + $attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp; + } } if ($this->_isRest()) { return $this->RestResponse->viewData($attributes, $this->response->type()); @@ -1877,36 +1925,38 @@ class AttributesController extends AppController public function generateCorrelation() { - $this->request->allowMethod(['post']); + if ($this->request->is('post')) { + if (!Configure::read('MISP.background_jobs')) { + $k = $this->Attribute->generateCorrelation(); + $this->Flash->success(__('All done. %s attributes processed.', $k)); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } else { + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate correlation', + 'All attributes', + 'Job created.' + ); - if (!Configure::read('MISP.background_jobs')) { - $k = $this->Attribute->generateCorrelation(); - $this->Flash->success(__('All done. %s attributes processed.', $k)); - $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); - } else { - /** @var Job $job */ - $job = ClassRegistry::init('Job'); - $jobId = $job->createJob( - 'SYSTEM', - Job::WORKER_DEFAULT, - 'generate correlation', - 'All attributes', - 'Job created.' - ); - - $this->Attribute->getBackgroundJobsTool()->enqueue( - BackgroundJobsTool::DEFAULT_QUEUE, - BackgroundJobsTool::CMD_ADMIN, - [ - 'jobGenerateCorrelation', + $this->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateCorrelation', + $jobId + ], + true, $jobId - ], - true, - $jobId - ); + ); - $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).')); - $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).')); + $this->redirect(Router::url($this->referer(), true)); + } + } else { + $this->render('ajax/recorrelationConfirmation'); } } @@ -2417,7 +2467,7 @@ class AttributesController extends AppController } else { $data[$attribute[0]['Attribute']['type']] = $attribute[0]['Attribute']['value']; } - $result = $this->Module->queryModuleServer($data, true); + $result = $this->Module->queryModuleServer($data, true, 'Enrichment', false, $attribute[0]); if ($result) { if (!is_array($result)) { $resultArray[$type] = ['error' => $result]; @@ -2770,14 +2820,18 @@ class AttributesController extends AppController $tag_id = $this->request->data['tag']; } $this->Attribute->id = $id; - if (!$this->Attribute->exists()) { + $attribute = $this->__fetchAttribute($id); + $attribute = $this->Attribute->find('first', [ + 'recursive' => -1, + 'conditions' => ['Attribute.id' => $id], + 'fields' => ['Attribute.deleted', 'Attribute.event_id', 'Attribute.id', 'Attribute.object_id'] + ]); + if (empty($attribute)) { throw new NotFoundException(__('Invalid attribute')); } - $this->Attribute->read(); - if ($this->Attribute->data['Attribute']['deleted']) { + if ($attribute['Attribute']['deleted']) { throw new NotFoundException(__('Invalid attribute')); } - $eventId = $this->Attribute->data['Attribute']['event_id']; if (empty($tag_id)) { return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status' => 200, 'type' => 'json')); } @@ -2792,12 +2846,13 @@ class AttributesController extends AppController $id = $this->request->data['Attribute']['id']; } - $this->Attribute->Event->recursive = -1; - $event = $this->Attribute->Event->read(array(), $eventId); + $event = $this->Attribute->Event->find('first', [ + 'recursive' => -1, + 'conditons' => ['Event.id' => $attribute['Attribute']['event_id']] + ]); if (!$this->_isRest()) { - $this->Attribute->Event->insertLock($this->Auth->user(), $eventId); + $this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']); } - $this->Attribute->recursive = -1; $attributeTag = $this->Attribute->AttributeTag->find('first', array( 'conditions' => array( 'attribute_id' => $id, @@ -2825,11 +2880,11 @@ class AttributesController extends AppController $date = new DateTime(); $event['Event']['timestamp'] = $date->getTimestamp(); $this->Attribute->Event->save($event); - if ($this->Attribute->data['Attribute']['object_id'] != 0) { - $this->Attribute->Object->updateTimestamp($this->Attribute->data['Attribute']['object_id'], $date->getTimestamp()); + if ($attribute['Attribute']['object_id'] != 0) { + $this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp()); } - $this->Attribute->data['Attribute']['timestamp'] = $date->getTimestamp(); - $this->Attribute->save($this->Attribute->data); + $attribute['Attribute']['timestamp'] = $date->getTimestamp(); + $this->Attribute->save($attribute); } $log = ClassRegistry::init('Log'); $log->createLogEntry($this->Auth->user(), 'tag', 'Attribute', $id, 'Removed tag (' . $tag_id . ') "' . $tag['Tag']['name'] . '" from attribute (' . $id . ')', 'Attribute (' . $id . ') untagged of Tag (' . $tag_id . ')'); @@ -2945,6 +3000,7 @@ class AttributesController extends AppController /** * @param int|string $id Attribute ID or UUID * @return array + * @throws Exception */ private function __fetchAttribute($id) { diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php index edbe07bf2..c61ee0f0d 100644 --- a/app/Controller/AuditLogsController.php +++ b/app/Controller/AuditLogsController.php @@ -52,6 +52,7 @@ class AuditLogsController extends AppController 'GalaxyClusterRelation', 'News', 'Warninglist', + 'Workflow', ]; public $paginate = [ @@ -67,9 +68,9 @@ class AuditLogsController extends AppController ], ]; - public function __construct($id = false, $table = null, $ds = null) + public function __construct($request = null, $response = null) { - parent::__construct($id, $table, $ds); + parent::__construct($request, $response); $this->actions = [ AuditLog::ACTION_ADD => __('Add'), AuditLog::ACTION_EDIT => __('Edit'), @@ -344,7 +345,7 @@ class AuditLogsController extends AppController return ['event_id' => $event['Event']['id']]; } - $event = $this->Event->fetchEvent($this->Auth->user(), [ + $event = $this->AuditLog->Event->fetchEvent($this->Auth->user(), [ 'eventid' => $event['Event']['id'], 'sgReferenceOnly' => 1, 'deleted' => [0, 1], diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 98ae91873..4be0938d1 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -98,7 +98,10 @@ class ACLComponent extends Component ], 'correlations' => [ 'generateTopCorrelations' => [], - 'top' => [] + 'overCorrelations' => [], + 'switchEngine' => [], + 'top' => [], + 'truncate' => [] ], 'cryptographicKeys' => [ 'add' => ['perm_add'], @@ -499,6 +502,7 @@ class ACLComponent extends Component 'servers' => array( 'add' => array(), 'dbSchemaDiagnostic' => array(), + 'dbConfiguration' => array(), 'cache' => array(), 'changePriority' => array(), 'checkout' => array(), @@ -669,6 +673,7 @@ class ACLComponent extends Component 'view' => array('*'), 'unhideTag' => array('perm_tagger'), 'hideTag' => array('perm_tagger'), + 'normalizeCustomTagsToTaxonomyFormat' => [], ), 'templateElements' => array( 'add' => array('perm_template'), @@ -738,6 +743,7 @@ class ACLComponent extends Component 'verifyGPG' => array(), 'view' => array('*'), 'getGpgPublicKey' => array('*'), + 'unsubscribe' => ['*'], ), 'userSettings' => array( 'index' => array('*'), @@ -762,6 +768,32 @@ class ACLComponent extends Component 'export' => ['*'], 'import' => ['perm_warninglist'], ), + 'workflows' => [ + 'index'=> [], + 'rebuildRedis'=> [], + 'edit'=> [], + 'delete'=> [], + 'view'=> [], + 'editor'=> [], + 'triggers'=> [], + 'moduleIndex'=> [], + 'moduleView'=> [], + 'toggleModule'=> [], + 'checkGraph'=> [], + 'executeWorkflow'=> [], + 'debugToggleField'=> [], + 'massToggleField'=> [], + ], + 'workflowBlueprints' => [ + 'add' => [], + 'delete' => [], + 'edit' => [], + 'export' => [], + 'import' => [], + 'index' => [], + 'update' => [], + 'view' => [], + ], 'allowedlists' => array( 'admin_add' => array('perm_regexp_access'), 'admin_delete' => array('perm_regexp_access'), @@ -979,6 +1011,8 @@ class ACLComponent extends Component private function __findAllFunctions() { + $functionsToIgnore = ['beforeFilter', 'afterFilter', 'beforeRender', 'getEventManager']; + $functionFinder = '/function[\s\n]+(\S+)[\s\n]*\(/'; $dir = new Folder(APP . 'Controller'); $files = $dir->find('.*\.php'); @@ -989,11 +1023,11 @@ class ACLComponent extends Component $controllerName = '*'; } $functionArray = array(); - $fileContents = file_get_contents(APP . 'Controller' . DS . $file); + $fileContents = FileAccessTool::readFromFile(APP . 'Controller' . DS . $file); $fileContents = preg_replace('/\/\*[^\*]+?\*\//', '', $fileContents); preg_match_all($functionFinder, $fileContents, $functionArray); foreach ($functionArray[1] as $function) { - if ($function[0] !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter' && $function !== 'beforeRender') { + if ($function[0] !== '_' && !in_array($function, $functionsToIgnore, true)) { $results[$controllerName][] = $function; } } @@ -1014,8 +1048,7 @@ class ACLComponent extends Component $missing = array(); foreach ($results as $controller => $functions) { foreach ($functions as $function) { - if (!isset(self::ACL_LIST[$controller]) - || !in_array($function, array_keys(self::ACL_LIST[$controller]))) { + if (!isset(self::ACL_LIST[$controller]) || !in_array($function, array_keys(self::ACL_LIST[$controller]))) { $missing[$controller][] = $function; } } diff --git a/app/Controller/Component/Auth/BlowfishConstantPasswordHasher.php b/app/Controller/Component/Auth/BlowfishConstantPasswordHasher.php index 52c639f85..19edf2963 100644 --- a/app/Controller/Component/Auth/BlowfishConstantPasswordHasher.php +++ b/app/Controller/Component/Auth/BlowfishConstantPasswordHasher.php @@ -1,8 +1,21 @@ signContents) { - $this->CryptographicKey = ClassRegistry::init('CryptographicKey'); $data = $response->intoString(); - $headers['x-pgp-signature'] = base64_encode($this->CryptographicKey->signWithInstanceKey($data)); + $headers['x-pgp-signature'] = $this->sign($data); $cakeResponse = new CakeResponse(['body' => $data, 'status' => $code, 'type' => $type]); } else { App::uses('CakeResponseFile', 'Tools'); @@ -657,7 +656,7 @@ class RestResponseComponent extends Component } $cakeResponse = new CakeResponse(['body' => $response, 'status' => $code, 'type' => $type]); if ($this->signContents) { - $headers['x-pgp-signature'] = base64_encode($this->CryptographicKey->signWithInstanceKey($response)); + $headers['x-pgp-signature'] = $this->sign($response); } } @@ -682,6 +681,25 @@ class RestResponseComponent extends Component return $cakeResponse; } + /** + * @param string $response + * @return string Signature as base64 encoded string + * @throws Crypt_GPG_BadPassphraseException + * @throws Crypt_GPG_Exception + * @throws Crypt_GPG_KeyNotFoundException + * @throws Exception + */ + private function sign($response) + { + /** @var CryptographicKey $cryptographicKey */ + $cryptographicKey = ClassRegistry::init('CryptographicKey'); + $signature = $cryptographicKey->signWithInstanceKey($response); + if (!$signature) { + throw new Exception('Could not sign data.'); + } + return base64_encode($signature); + } + /** * Detect if request comes from automatic tool (like other MISP instance or PyMISP) or AJAX * @return bool diff --git a/app/Controller/CorrelationsController.php b/app/Controller/CorrelationsController.php index 1819dd2d3..61719b76f 100644 --- a/app/Controller/CorrelationsController.php +++ b/app/Controller/CorrelationsController.php @@ -44,6 +44,9 @@ class CorrelationsController extends AppController } } } + + $this->__setPagingParams($query['page'], $query['limit'], count($data), 'named'); + $this->set('age', $age); $this->set('age_unit', $unit); $this->set('data', $data); @@ -72,4 +75,129 @@ class CorrelationsController extends AppController $this->redirect(['controller' => 'correlations', 'action' => 'top']); } } + + public function overCorrelations() + { + $query = [ + 'limit' => 50, + 'page' => 1, + 'order' => 'occurrence desc' + ]; + foreach (array_keys($query) as $custom_param) { + if (isset($this->params['named'][$custom_param])) { + $query[$custom_param] = $this->params['named'][$custom_param]; + } + } + if (isset($this->params['named']['scope'])) { + $limit = $this->Correlation->OverCorrelatingValue->getLimit(); + if ($this->params['named']['scope'] === 'over_correlating') { + $query['conditions'][] = ['occurrence >=' => $limit]; + } else if ($this->params['named']['scope'] === 'not_over_correlating') { + $query['conditions'][] = ['occurrence <' => $limit]; + } + } + $data = $this->Correlation->OverCorrelatingValue->getOverCorrelations($query); + $data = $this->Correlation->attachExclusionsToOverCorrelations($data); + + if ($this->_isRest()) { + return $this->RestResponse->viewData($data, 'json'); + } else { + $this->__setPagingParams($query['page'], $query['limit'], count($data), 'named'); + $this->set('data', $data); + $this->set('title_for_layout', __('Index of over correlating values')); + $this->set('menuData', [ + 'menuList' => 'correlationExclusions', + 'menuItem' => 'over' + ]); + } + } + + public function switchEngine(string $engine) + { + $this->loadModel('Server'); + if (!isset($this->Correlation->validEngines[$engine])) { + throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines)))); + } + if ($this->request->is('post')) { + $setting = $this->Server->getSettingData('MISP.correlation_engine'); + $result = $this->Server->serverSettingsEditValue($this->Auth->user(), $setting, $engine); + if ($result === true) { + $message = __('Engine switched.'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Correlations', 'switchEngine', false, $this->response->type(), $message); + } else { + $this->Flash->success($message); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']); + } + } else { + $message = __('Couldn\'t switch to the requested engine.'); + if ($this->_isRest()) { + return $this->RestResponse->saveFailResponse('Correlations', 'switchEngine', false, $message, $this->response->type()); + } else { + $this->Flash->error($message); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']); + } + } + } else { + $this->set('engine', $engine); + $this->render('ajax/switch_engine_confirmation'); + } + } + + public function truncate(string $engine) + { + if (!isset($this->Correlation->validEngines[$engine])) { + throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines)))); + } + if ($this->request->is('post')) { + if (!Configure::read('MISP.background_jobs')) { + $result = $this->Correlation->truncate($this->Auth->user(), $engine); + $message = $result ? __('Table truncated.') : __('Could not truncate table'); + if ($this->_isRest()) { + if ($result) { + $this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message); + } else { + $this->RestResponse->saveFailResponse('Correlations', 'truncate', false, $message, $this->response->type()); + } + } else { + $this->Flash->{$result ? 'success' : 'error'}($message); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']); + } + } else { + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'truncate table', + $this->Correlation->validEngines[$engine], + 'Job created.' + ); + + $this->Correlation->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'truncateTable', + $this->Auth->user('id'), + $engine, + $jobId + ], + true, + $jobId + ); + + $message = __('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message); + } else { + $this->Flash->success($message); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']); + } + } + } else { + $this->set('engine', $engine); + $this->set('table_name', $this->Correlation->validEngines[$engine]); + $this->render('ajax/truncate_confirmation'); + } + } } diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index ab9debc5e..6eaa3b52a 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -619,7 +619,7 @@ class EventsController extends AppController if (empty($usersToMatch)) { $nothing = true; } else { - $this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch)]; + $this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch, SORT_REGULAR)]; } } break; @@ -2960,8 +2960,12 @@ class EventsController extends AppController $result = $this->Event->publishRouter($event['Event']['id'], null, $this->Auth->user()); if (!Configure::read('MISP.background_jobs')) { if (!is_array($result)) { - // redirect to the view event page - $message = __('Event published without alerts'); + if ($result === true) { + $message = __('Event published without alerts'); + } else { + $message = __('Event publishing failed due to a blocking module failing. The reason for the failure: %s', $result); + $errors['Module'] = 'Module failure.'; + } } else { $lastResult = array_pop($result); $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult; @@ -2969,11 +2973,6 @@ class EventsController extends AppController $message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString); } } else { - // update the DB to set the published flag - // for background jobs, this should be done already - $event['Event']['published'] = 1; - $event['Event']['publish_timestamp'] = time(); - $this->Event->save($event, true, ['id', 'published', 'publish_timestamp', 'info']); // info field is required because of SysLogLogableBehavior $message = 'Job queued'; } if ($this->_isRest()) { @@ -2983,7 +2982,11 @@ class EventsController extends AppController return $this->RestResponse->saveSuccessResponse('Events', 'publish', $event['Event']['id'], false, $message); } } else { - $this->Flash->success($message); + if (!empty($errors)) { + $this->Flash->error($message); + } else { + $this->Flash->success($message); + } $this->redirect(array('action' => 'view', $event['Event']['id'])); } } else { @@ -3020,13 +3023,19 @@ class EventsController extends AppController $errors['failed_servers'] = $result; $message = __('Not published given no connection to %s but email sent to all participants.', $resultString); } + } elseif (!is_bool($emailResult)) { // Performs all the actions required to publish an event $result = $this->Event->publishRouter($event['Event']['id'], null, $this->Auth->user()); if (!is_array($result)) { + if ($result === true) { + $message = __('Published but no email sent given GnuPG is not configured.'); + $errors['GnuPG'] = 'GnuPG not set up.'; + } else { + $message = $result; + $errors['Module'] = 'Module failure.'; + } // redirect to the view event page - $message = __('Published but no email sent given GnuPG is not configured.'); - $errors['GnuPG'] = 'GnuPG not set up.'; } else { $lastResult = array_pop($result); $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult; @@ -3315,20 +3324,40 @@ class EventsController extends AppController public function restSearchExport($id = null, $returnFormat = null) { - if (is_null($returnFormat)) { - if (is_numeric($id)) { - $idList = [$id]; - } else { - $idList = $this->_jsonDecode($id); - } + if ($returnFormat === null) { + $exportFormats = [ + 'attack' => __('Attack matrix'), + 'attack-sightings' => __('Attack matrix by sightings'), + 'context' => __('Aggregated context data'), + 'context-markdown' => __('Aggregated context data as Markdown'), + 'csv' => __('CSV'), + 'hashes' => __('Hashes'), + 'hosts' => __('Hosts file'), + 'json' => __('MISP JSON'), + 'netfilter' => __('Netfilter'), + 'opendata' => __('Open data'), + 'openioc' => __('OpenIOC'), + 'rpz' => __('RPZ'), + 'snort' => __('Snort rules'), + 'stix' => __('STIX 1 XML'), + 'stix-json' => __('STIX 1 JSON'), + 'stix2' => __('STIX 2'), + 'suricata' => __('Suricata rules'), + 'text' => __('Text file'), + 'xml' => __('MISP XML'), + 'yara' => __('YARA rules'), + 'yara-json' => __('YARA rules (JSON)'), + ]; + + $idList = is_numeric($id) ? [$id] : $this->_jsonDecode($id); if (empty($idList)) { throw new NotFoundException(__('Invalid input.')); } $this->set('idList', $idList); - $this->set('exportFormats', array_keys($this->Event->validFormats)); + $this->set('exportFormats', $exportFormats); $this->render('ajax/eventRestSearchExportConfirmationForm'); } else { - $returnFormat = empty($this->Event->validFormats[$returnFormat]) ? 'json' : $returnFormat; + $returnFormat = !isset($this->Event->validFormats[$returnFormat]) ? 'json' : $returnFormat; $idList = $id; if (!is_array($idList)) { if (is_numeric($idList) || Validation::uuid($idList)) { @@ -3341,19 +3370,17 @@ class EventsController extends AppController throw new NotFoundException(__('Invalid input.')); } $filters = [ - 'eventid' => $idList + 'eventid' => $idList, + 'published' => [true, false], // fetch published and unpublished events ]; $elementCounter = 0; $renderView = false; - $validFormat = $this->Event->validFormats[$returnFormat]; - $responseType = $validFormat[0]; + $responseType = $this->Event->validFormats[$returnFormat][0]; $final = $this->Event->restSearch($this->Auth->user(), $returnFormat, $filters, false, false, $elementCounter, $renderView); - if (!empty($renderView) && !empty($final)) { + if ($renderView) { $final = json_decode($final->intoString(), true); - foreach ($final as $key => $data) { - $this->set($key, $data); - } + $this->set($final); $this->set('responseType', $responseType); $this->set('returnFormat', $returnFormat); $this->set('renderView', $renderView); @@ -4097,29 +4124,31 @@ class EventsController extends AppController public function filterEventIdsForPush() { - if ($this->request->is('post')) { - $incomingIDs = array(); - $incomingEvents = array(); - foreach ($this->request->data as $event) { - $incomingIDs[] = $event['Event']['uuid']; - $incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp']; - } - $events = $this->Event->find('all', array( - 'conditions' => array('Event.uuid' => $incomingIDs), - 'recursive' => -1, - 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'), - )); - foreach ($events as $event) { - if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) { - unset($incomingEvents[$event['Event']['uuid']]); - continue; - } - if ($event['Event']['locked'] == 0) { - unset($incomingEvents[$event['Event']['uuid']]); - } - } - return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type()); + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This endpoint requires a POST request.')); } + + $incomingUuids = []; + $incomingEvents = []; + foreach ($this->request->data as $event) { + $incomingUuids[] = $event['Event']['uuid']; + $incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp']; + } + $events = $this->Event->find('all', [ + 'conditions' => ['Event.uuid' => $incomingUuids], + 'recursive' => -1, + 'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'], + ]); + foreach ($events as $event) { + if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) { + unset($incomingEvents[$event['Event']['uuid']]); + continue; + } + if ($event['Event']['locked'] == 0) { + unset($incomingEvents[$event['Event']['uuid']]); + } + } + return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type()); } public function checkuuid($uuid) @@ -5105,7 +5134,14 @@ class EventsController extends AppController if (!Configure::read('Plugin.' . $type . '_services_enable')) { throw new MethodNotAllowedException(__('%s services are not enabled.', $type)); } - $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $attribute_id), 'flatten' => 1)); + $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [ + 'conditions' => [ + 'Attribute.id' => $attribute_id + ], + 'flatten' => 1, + 'includeEventTags' => 1, + 'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]], + ]); if (empty($attribute)) { throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.')); } @@ -5166,7 +5202,7 @@ class EventsController extends AppController if (!empty($options)) { $data['config'] = $options; } - $result = $this->Module->queryModuleServer($data, false, $type); + $result = $this->Module->queryModuleServer($data, false, $type, false, $attribute[0]); if (!$result) { throw new InternalErrorException(__('%s service not reachable.', $type)); } @@ -5212,7 +5248,7 @@ class EventsController extends AppController if (!empty($options)) { $data['config'] = $options; } - $result = $this->Module->queryModuleServer($data, false, $type); + $result = $this->Module->queryModuleServer($data, false, $type, false, $attribute[0]); if (!$result) { throw new InternalErrorException(__('%s service not reachable.', $type)); } diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index fa22f4b0f..271846b3e 100644 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -30,17 +30,17 @@ class LogsController extends AppController public function admin_index() { + $paramArray = array('id', 'title', 'created', 'model', 'model_id', 'action', 'user_id', 'change', 'email', 'org', 'description', 'ip'); + $filterData = array( + 'request' => $this->request, + 'named_params' => $this->params['named'], + 'paramArray' => $paramArray, + 'ordered_url_params' => func_get_args() + ); + $exception = false; + $filters = $this->_harvestParameters($filterData, $exception); + unset($filterData); if ($this->_isRest()) { - $paramArray = array('id', 'title', 'created', 'model', 'model_id', 'action', 'user_id', 'change', 'email', 'org', 'description', 'ip'); - $filterData = array( - 'request' => $this->request, - 'named_params' => $this->params['named'], - 'paramArray' => $paramArray, - 'ordered_url_params' => func_get_args() - ); - $exception = false; - $filters = $this->_harvestParameters($filterData, $exception); - unset($filterData); if ($filters === false) { return $exception; } @@ -100,6 +100,9 @@ class LogsController extends AppController if (isset($this->params['named']['filter']) && in_array($this->params['named']['filter'], array_keys($validFilters))) { $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values']; } + foreach ($filters as $key => $value) { + $this->paginate['conditions']["Log.$key"] = $value; + } $this->set('validFilters', $validFilters); $this->set('filter', isset($this->params['named']['filter']) ? $this->params['named']['filter'] : false); $this->set('list', $this->paginate()); @@ -389,6 +392,7 @@ class LogsController extends AppController 'Galaxy', 'GalaxyCluster', 'GalaxyClusterRelation', + 'Workflow', ]; sort($models); $models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models); diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index e9b9f20e1..d4042f1d9 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -948,7 +948,14 @@ class ServersController extends AppController public function serverSettingsReloadSetting($setting, $id) { $pathToSetting = explode('.', $setting); - if (strpos($setting, 'Plugin.Enrichment') !== false || strpos($setting, 'Plugin.Import') !== false || strpos($setting, 'Plugin.Export') !== false || strpos($setting, 'Plugin.Cortex') !== false) { + if ( + strpos($setting, 'Plugin.Enrichment') !== false || + strpos($setting, 'Plugin.Import') !== false || + strpos($setting, 'Plugin.Export') !== false || + strpos($setting, 'Plugin.Cortex') !== false || + strpos($setting, 'Plugin.Action') !== false || + strpos($setting, 'Plugin.Workflow') !== false + ) { $settingObject = $this->Server->getCurrentServerSettings(); } else { $settingObject = $this->Server->serverSettings; @@ -1065,6 +1072,11 @@ class ServersController extends AppController $diagnostic_errors = 0; App::uses('File', 'Utility'); App::uses('Folder', 'Utility'); + if ($tab === 'correlations') { + $this->loadModel('Correlation'); + $correlation_metrics = $this->Correlation->collectMetrics(); + $this->set('correlation_metrics', $correlation_metrics); + } if ($tab === 'files') { $files = $this->Server->grabFiles(); $this->set('files', $files); @@ -1146,6 +1158,7 @@ class ServersController extends AppController // get the DB diagnostics $dbDiagnostics = $this->Server->dbSpaceUsage(); $dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic(); + $dbConfiguration = $this->Server->dbConfiguration(); $redisInfo = $this->Server->redisInfo(); @@ -1166,7 +1179,7 @@ class ServersController extends AppController $securityAudit = (new SecurityAudit())->run($this->Server); - $view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo', 'attachmentScan', 'securityAudit'); + $view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'dbConfiguration', 'redisInfo', 'attachmentScan', 'securityAudit'); } else { $view = []; } @@ -1207,6 +1220,7 @@ class ServersController extends AppController 'readableFiles' => $readableFiles, 'dbDiagnostics' => $dbDiagnostics, 'dbSchemaDiagnostics' => $dbSchemaDiagnostics, + 'dbConfiguration' => $dbConfiguration, 'redisInfo' => $redisInfo, 'finalSettings' => $dumpResults, 'extensions' => $extensions, @@ -1444,7 +1458,6 @@ class ServersController extends AppController } $this->set('id', $id); } - $setting = $this->Server->getSettingData($settingName); if ($setting === false) { throw new NotFoundException(__('Setting %s is invalid.', $settingName)); @@ -1916,7 +1929,7 @@ class ServersController extends AppController $dbVersion = $this->AdminSetting->getSetting('db_version'); $updateProgress = $this->Server->getUpdateProgress(); $updateProgress['db_version'] = $dbVersion; - $maxUpdateNumber = max(array_keys($this->Server->db_changes)); + $maxUpdateNumber = max(array_keys(Server::DB_CHANGES)); $updateProgress['complete_update_remaining'] = max($maxUpdateNumber - $dbVersion, 0); $updateProgress['update_locked'] = $this->Server->isUpdateLocked(); $updateProgress['lock_remaining_time'] = $this->Server->getLockRemainingTime(); @@ -2205,6 +2218,17 @@ class ServersController extends AppController } } + public function dbConfiguration() + { + $dbConfiguration = $this->Server->dbConfiguration(); + if ($this->_isRest()) { + return $this->RestResponse->viewData($dbConfiguration, $this->response->type()); + } else { + $this->set('dbConfiguration', $dbConfiguration); + $this->render('/Elements/healthElements/db_config_diagnostic'); + } + } + public function cspReport() { if (!$this->request->is('post')) { diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 1b75d1e24..8dd4d08e6 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -518,19 +518,19 @@ class ShadowAttributesController extends AppController } } else { $shadowAttribute = array( - 'ShadowAttribute' => array( - 'value' => $filename, - 'category' => $this->request->data['ShadowAttribute']['category'], - 'type' => 'attachment', - 'event_id' => $this->request->data['ShadowAttribute']['event_id'], - 'comment' => $this->request->data['ShadowAttribute']['comment'], - 'data' => base64_encode($tmpfile->read()), - 'to_ids' => 0, - 'email' => $this->Auth->user('email'), - 'org_id' => $this->Auth->user('org_id'), - 'event_uuid' => $event['Event']['uuid'], - 'event_org_id' => $event['Event']['orgc_id'], - ) + 'ShadowAttribute' => array( + 'value' => $filename, + 'category' => $this->request->data['ShadowAttribute']['category'], + 'type' => 'attachment', + 'event_id' => $this->request->data['ShadowAttribute']['event_id'], + 'comment' => $this->request->data['ShadowAttribute']['comment'], + 'data' => base64_encode($tmpfile->read()), + 'to_ids' => 0, + 'email' => $this->Auth->user('email'), + 'org_id' => $this->Auth->user('org_id'), + 'event_uuid' => $event['Event']['uuid'], + 'event_org_id' => $event['Event']['orgc_id'], + ) ); $this->ShadowAttribute->create(); $r = $this->ShadowAttribute->save($shadowAttribute); @@ -585,6 +585,7 @@ class ShadowAttributesController extends AppController $this->set('typeDefinitions', $this->ShadowAttribute->typeDefinitions); $this->set('categoryDefinitions', $this->ShadowAttribute->categoryDefinitions); $this->set('isMalwareSampleCategory', $isMalwareSampleCategory); + $this->set('mayModify', $this->__canModifyEvent($event)); $this->set('event', $event); $this->set('title_for_layout', __('Propose attachment')); } diff --git a/app/Controller/TaxonomiesController.php b/app/Controller/TaxonomiesController.php index d4dec12c2..afbb6c22c 100644 --- a/app/Controller/TaxonomiesController.php +++ b/app/Controller/TaxonomiesController.php @@ -594,4 +594,14 @@ class TaxonomiesController extends AppController return $taxonomyIds; } + + + public function normalizeCustomTagsToTaxonomyFormat() + { + $this->request->allowMethod(['post', 'put']); + $conversionResult = $this->Taxonomy->normalizeCustomTagsToTaxonomyFormat(); + $this->Flash->success(__('%s tags successfully converted. %s row updated.', $conversionResult['tag_converted'], $conversionResult['row_updated'])); + $this->redirect(array('controller' => 'taxonomies', 'action' => 'index')); + } + } diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 30058897a..899769dc4 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -118,6 +118,24 @@ class UsersController extends AppController return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Something went wrong, please try again later.')), 'status'=>200, 'type' => 'json')); } + public function unsubscribe($code) + { + $user = $this->Auth->user(); + + if (!hash_equals($this->User->unsubscribeCode($user), rtrim($code, '.'))) { + $this->Flash->error(__('Invalid unsubscribe code.')); + $this->redirect(['action' => 'view', 'me']); + } + + if ($user['autoalert']) { + $this->User->updateField($this->Auth->user(), 'autoalert', false); + $this->Flash->success(__('Successfully unsubscribed from event alert.')); + } else { + $this->Flash->info(__('Already unsubscribed from event alert.')); + } + $this->redirect(['action' => 'view', 'me']); + } + public function edit() { $currentUser = $this->User->find('first', array( diff --git a/app/Controller/WorkflowBlueprintsController.php b/app/Controller/WorkflowBlueprintsController.php new file mode 100644 index 000000000..711e7cd5c --- /dev/null +++ b/app/Controller/WorkflowBlueprintsController.php @@ -0,0 +1,127 @@ +request->allowMethod(['post', 'put']); + $this->WorkflowBlueprint->update($force); + $message = __('Default workflow blueprints updated'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('WorkflowBlueprint', 'update', false, $this->response->type(), $message); + } else { + $this->Flash->success($message); + $this->redirect(array('controller' => 'workflowBlueprints', 'action' => 'index')); + } + } + + public function index() + { + $params = [ + 'filters' => ['name', 'uuid', 'timestamp'], + 'quickFilters' => ['name', 'uuid'], + ]; + $this->CRUD->index($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'index']); + } + + public function add($fromEditor = false) + { + $params = [ + 'beforeSave' => function(array $blueprint) { + $blueprint['WorkflowBlueprint']['default'] = false; + return $blueprint; + }, + ]; + $this->CRUD->add($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('fromEditor', !empty($fromEditor)); + $this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'add']); + } + + public function edit($id) + { + $params = [ + 'beforeSave' => function (array $blueprint) { + $blueprint['WorkflowBlueprint']['default'] = false; + return $blueprint; + }, + ]; + $this->CRUD->edit($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->request->data['WorkflowBlueprint']['data'] = JsonTool::encode($this->data['WorkflowBlueprint']['data']); + $this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'edit']); + $this->set('id', $id); + $this->render('add'); + } + + public function delete($id) + { + $params = [ + ]; + $this->CRUD->delete($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'delete']); + } + + public function view($id) + { + $filters = $this->IndexFilter->harvestParameters(['format']); + if (!empty($filters['format'])) { + if ($filters['format'] == 'dot') { + $dot = $this->WorkflowBlueprint->getDotNotation($id); + return $this->RestResponse->viewData($dot, $this->response->type()); + } else if ($filters['format'] == 'mermaid') { + $mermaid = $this->WorkflowBlueprint->getMermaid($id); + return $this->RestResponse->viewData($mermaid, $this->response->type()); + } + } + $this->CRUD->view($id, [ + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('id', $id); + $this->set('menuData', ['menuList' => 'workflowBlueprints', 'menuItem' => 'view']); + } + + public function import() + { + if ($this->request->is('post') || $this->request->is('put')) { + $workflowBlueprintData = JsonTool::decode($this->request->data['WorkflowBlueprint']['data']); + if ($workflowBlueprintData === null) { + throw new MethodNotAllowedException(__('Error while decoding JSON')); + } + $this->request->data['WorkflowBlueprint']['data'] = JsonTool::encode($workflowBlueprintData); + $this->add(); + } + } + + public function export($id) + { + $workflowBlueprint = $this->WorkflowBlueprint->find('first', [ + 'conditions' => [ + 'id' => $id, + ] + ]); + $content = JsonTool::encode($workflowBlueprint, JSON_PRETTY_PRINT); + $this->response->body($content); + $this->response->type('json'); + $this->response->download(sprintf('workflowblueprint_%s_%s.json', $workflowBlueprint['WorkflowBlueprint']['name'], time())); + return $this->response; + } +} diff --git a/app/Controller/WorkflowsController.php b/app/Controller/WorkflowsController.php new file mode 100644 index 000000000..fc85a780b --- /dev/null +++ b/app/Controller/WorkflowsController.php @@ -0,0 +1,425 @@ +Security->unlockedActions[] = 'checkGraph'; + $requirementErrors = []; + if (empty(Configure::read('MISP.background_jobs'))) { + $requirementErrors[] = __('Background workers must be enabled to use workflows'); + $this->render('error'); + } + if (empty(Configure::read('Plugin.Workflow_enable'))) { + $requirementErrors[] = __('The workflow plugin must be enabled to use workflows. Go to `/servers/serverSettings/Plugin` the enable the `Plugin.Workflow` setting'); + $this->render('error'); + } + try { + $this->Workflow->setupRedisWithException(); + } catch (Exception $e) { + $requirementErrors[] = $e->getMessage(); + } + if (!empty($requirementErrors)) { + $this->set('requirementErrors', $requirementErrors); + $this->render('error'); + } + } + + public function index() + { + $params = [ + 'filters' => ['name', 'uuid'], + 'quickFilters' => ['name', 'uuid'], + ]; + $this->CRUD->index($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'index')); + } + + public function rebuildRedis() + { + $this->Workflow->rebuildRedis(); + } + + public function edit($id) + { + $this->set('id', $id); + $savedWorkflow = $this->Workflow->fetchWorkflow($id); + if ($this->request->is('post') || $this->request->is('put')) { + $newWorkflow = $this->request->data; + $newWorkflow['Workflow']['data'] = JsonTool::decode($newWorkflow['Workflow']['data']); + $newWorkflow = $this->__applyDataFromSavedWorkflow($newWorkflow, $savedWorkflow); + $result = $this->Workflow->editWorkflow($newWorkflow); + $redirectTarget = ['action' => 'view', $id]; + if (!empty($result['errors'])) { + return $this->__getFailResponseBasedOnContext($result['errors'], null, 'edit', $this->Workflow->id, $redirectTarget); + } else { + $successMessage = __('Workflow saved.'); + $savedWorkflow = $result['saved']; + return $this->__getSuccessResponseBasedOnContext($successMessage, $savedWorkflow, 'edit', false, $redirectTarget); + } + } else { + $savedWorkflow['Workflow']['data'] = JsonTool::encode($savedWorkflow['Workflow']['data']); + $this->request->data = $savedWorkflow; + } + + $this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'edit')); + $this->render('add'); + } + + public function delete($id) + { + $params = [ + ]; + $this->CRUD->delete($id, $params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } + + public function view($id) + { + $filters = $this->IndexFilter->harvestParameters(['format']); + if (!empty($filters['format'])) { + if ($filters['format'] == 'dot') { + $dot = $this->Workflow->getDotNotation($id); + return $this->RestResponse->viewData($dot, $this->response->type()); + } else if ($filters['format'] == 'mermaid') { + $mermaid = $this->Workflow->getMermaid($id); + return $this->RestResponse->viewData($mermaid, $this->response->type()); + } + } + $this->CRUD->view($id, [ + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('id', $id); + $this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'view')); + } + + public function editor($id) + { + $trigger_id = false; + $workflow = false; + if (is_numeric($id)) { + $workflow_id = $id; + } else { + $trigger_id = $id; + } + $modules = $this->Workflow->getModulesByType(); + if (!empty($trigger_id)) { + $trigger_ids = Hash::extract($modules['modules_trigger'], '{n}.id'); + if (!in_array($trigger_id, $trigger_ids)) { + return $this->__getFailResponseBasedOnContext( + [__('Unkown trigger %s', $trigger_id)], + null, + 'add', + $trigger_id, + ['controller' => 'workflows', 'action' => 'triggers'] + ); + } + $workflow = $this->Workflow->fetchWorkflowByTrigger($trigger_id, false); + if (empty($workflow)) { // Workflow do not exists yet. Create it. + $result = $this->Workflow->addWorkflow([ + 'name' => sprintf('Workflow for trigger %s', $trigger_id), + 'data' => $this->Workflow->genGraphDataForTrigger($trigger_id), + 'trigger_id' => $trigger_id, + ]); + if (!empty($result['errors'])) { + return $this->__getFailResponseBasedOnContext( + [__('Could not create workflow for trigger %s', $trigger_id), $result['errors']], + null, + 'add', + $trigger_id, + ['controller' => 'workflows', 'action' => 'editor'] + ); + } + $workflow = $this->Workflow->fetchWorkflowByTrigger($trigger_id, false); + } + } else { + $workflow = $this->Workflow->fetchWorkflow($workflow_id); + } + $modules = $this->Workflow->attachNotificationToModules($modules, $workflow); + $this->loadModel('WorkflowBlueprint'); + $workflowBlueprints = $this->WorkflowBlueprint->find('all'); + $this->set('selectedWorkflow', $workflow); + $this->set('workflowTriggerId', $trigger_id); + $this->set('modules', $modules); + $this->set('workflowBlueprints', $workflowBlueprints); + } + + public function executeWorkflow($workflow_id) + { + if ($this->request->is('post') || $this->request->is('put')) { + $blockingErrors = []; + $data = JsonTool::decode($this->request->data['Workflow']['data']); + $result = $this->Workflow->executeWorkflow($workflow_id, $data, $blockingErrors); + if (!empty($logging) && empty($result['success'])) { + $logging['message'] = !empty($logging['message']) ? $logging['message'] : __('Error while executing workflow.'); + $errorMessage = implode(', ', $blockingErrors); + $this->Workflow->loadLog()->createLogEntry('SYSTEM', $logging['action'], $logging['model'], $logging['id'], $logging['message'], __('Returned message: %s', $errorMessage)); + } + return $this->RestResponse->viewData([ + 'success' => $result['success'], + 'outcome' => $result['outcomeText'], + ], $this->response->type()); + } + $this->render('ajax/executeWorkflow'); + } + + public function triggers() + { + $triggers = $this->Workflow->getModulesByType('trigger'); + $triggers = $this->Workflow->attachWorkflowToTriggers($triggers); + $data = $triggers; + App::uses('CustomPaginationTool', 'Tools'); + $customPagination = new CustomPaginationTool(); + $customPagination->truncateAndPaginate($data, $this->params, 'Workflow', true); + if ($this->_isRest()) { + return $this->RestResponse->viewData($data, $this->response->type()); + } + + $this->set('data', $data); + $this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'index_trigger']); + } + + public function moduleIndex() + { + $modules = $this->Workflow->getModulesByType(); + $errorWhileLoading = $this->Workflow->getModuleLoadingError(); + $this->Module = ClassRegistry::init('Module'); + $mispModules = $this->Module->getModules('Action'); + $this->set('module_service_error', !is_array($mispModules)); + $filters = $this->IndexFilter->harvestParameters(['type', 'actiontype', 'enabled']); + $moduleType = $filters['type'] ?? 'action'; + $actionType = $filters['actiontype'] ?? 'all'; + $enabledState = $filters['enabled'] ?? false; + if ($moduleType == 'all' || $moduleType == 'custom') { + $data = array_merge( + $modules["modules_action"], + $modules["modules_logic"] + ); + } else { + $data = $modules["modules_{$moduleType}"]; + } + if ($actionType == 'mispmodule') { + $data = array_filter($data, function($module) { + return !empty($module['is_misp_module']); + }); + } else if ($actionType == 'blocking') { + $data = array_filter($data, function ($module) { + return !empty($module['blocking']); + }); + } else if ($moduleType == 'custom') { + $data = array_filter($data, function ($module) { + return !empty($module['is_custom']); + }); + } + if ($enabledState !== false) { + $moduleType = !empty($enabledState) ? 'enabled' : 'disabled'; + $data = array_filter($data, function ($module) use ($enabledState) { + return !empty($enabledState) ? empty($module['disabled']) : !empty($module['disabled']); + }); + } + if ($this->_isRest()) { + return $this->RestResponse->viewData($data, $this->response->type()); + } + App::uses('CustomPaginationTool', 'Tools'); + $customPagination = new CustomPaginationTool(); + $params = $customPagination->createPaginationRules($data, $this->passedArgs, 'Workflow'); + $params = $customPagination->applyRulesOnArray($data, $params, 'Workflow'); + $params['options'] = array_merge($params['options'], $filters); + $this->params['paging'] = [$this->modelClass => $params]; + $this->set('data', $data); + $this->set('indexType', $moduleType); + $this->set('actionType', $actionType); + $this->set('errorWhileLoading', $errorWhileLoading); + $this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'index_module']); + } + + public function moduleView($module_id) + { + $module = $this->Workflow->getModuleByID($module_id); + if (empty($module)) { + throw new NotFoundException(__('Invalid trigger ID')); + } + $is_trigger = $module['module_type'] == 'trigger'; + if ($is_trigger) { + $module = $this->Workflow->attachWorkflowToTriggers([$module])[0]; + $module['listening_workflows'] = $this->Workflow->getListeningWorkflowForTrigger($module); + } + if ($this->_isRest()) { + return $this->RestResponse->viewData($module, $this->response->type()); + } + $this->set('data', $module); + $this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'view_module']); + } + + public function toggleModule($module_id, $enabled, $is_trigger=false) + { + $this->request->allowMethod(['post', 'put']); + $saved = $this->Workflow->toggleModule($module_id, $enabled, $is_trigger); + if ($saved) { + return $this->__getSuccessResponseBasedOnContext( + __('%s module %s', ($enabled ? 'Enabled' : 'Disabled'), $module_id), + null, + 'toggle_module', + $module_id, + ['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')] + ); + } else { + return $this->__getFailResponseBasedOnContext( + __('Could not %s module %s', ($enabled ? 'Enabled' : 'Disabled'), $module_id), + null, + 'toggle_module', + $module_id, + ['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')] + ); + } + } + + public function debugToggleField($workflow_id, $enabled) + { + if (!$this->request->is('ajax')) { + throw new MethodNotAllowedException(__('This action is available via AJAX only.')); + } + $this->layout = false; + $this->render('ajax/getDebugToggleField'); + if ($this->request->is('post') || $this->request->is('put')) { + $success = $this->Workflow->toggleDebug($workflow_id, $enabled); + if (!empty($success)) { + return $this->__getSuccessResponseBasedOnContext( + __('%s debug mode', ($enabled ? __('Enabled') : __('Disabled'))), + null, + 'toggle_debug', + $workflow_id, + ['action' => 'triggers'] + ); + } else { + return $this->__getFailResponseBasedOnContext( + __('Could not %s debug mode', ($enabled ? __('enable') : __('disable'))), + null, + 'toggle_debug', + $workflow_id, + ['action' => 'triggers'] + ); + } + } + } + + public function massToggleField($fieldName, $enabled, $is_trigger=false) + { + if (!in_array($fieldName, $this->toggleableFields)) { + throw new MethodNotAllowedException(__('The field `%s` cannot be toggled', $fieldName)); + } + if ($this->request->is('post') || $this->request->is('put')) { + $module_ids = JsonTool::decode($this->request->data['Workflow']['module_ids']); + $enabled_count = $this->Workflow->toggleModules($module_ids, $enabled, $is_trigger); + if (!empty($enabled_count)) { + return $this->__getSuccessResponseBasedOnContext( + __('%s %s modules', ($enabled ? 'Enabled' : 'Disabled'), $enabled_count), + null, + 'toggle_module', + $module_ids, + ['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')] + ); + } else { + return $this->__getFailResponseBasedOnContext( + __('Could not %s modules', ($enabled ? 'enable' : 'disable')), + null, + 'toggle_module', + $module_ids, + ['action' => (!empty($is_trigger) ? 'triggers' : 'moduleIndex')] + ); + } + } + } + + private function __getSuccessResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array()) + { + if ($this->_isRest()) { + if (!is_null($data)) { + return $this->RestResponse->viewData($data, $this->response->type()); + } else { + return $this->RestResponse->saveSuccessResponse('Workflow', $action, $id, false, $message); + } + } elseif ($this->request->is('ajax')) { + return $this->RestResponse->saveSuccessResponse('Workflow', $action, $id, false, $message, $data); + } else { + $this->Flash->success($message); + $this->redirect($redirect); + } + return; + } + + private function __getFailResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array()) + { + if (is_array($message)) { + $message = implode(', ', $message); + } + if ($this->_isRest()) { + if ($data !== null) { + return $this->RestResponse->viewData($data, $this->response->type()); + } else { + return $this->RestResponse->saveFailResponse('Workflow', $action, $id, $message); + } + } elseif ($this->request->is('ajax')) { + return $this->RestResponse->saveFailResponse('Workflow', $action, $id, $message, false, $data); + } else { + $this->Flash->error($message); + $this->redirect($redirect); + } + } + + private function __applyDataFromSavedWorkflow($newWorkflow, $savedWorkflow) + { + if (!isset($newWorkflow['Workflow'])) { + $newWorkflow = ['Workflow' => $newWorkflow]; + } + $ignoreFieldList = ['id', 'uuid']; + foreach (Workflow::CAPTURE_FIELDS_EDIT as $field) { + if (!in_array($field, $ignoreFieldList) && isset($newWorkflow['Workflow'][$field])) { + $savedWorkflow['Workflow'][$field] = $newWorkflow['Workflow'][$field]; + } + } + return $savedWorkflow; + } + + public function checkGraph() + { + $this->request->allowMethod(['post']); + $graphData = JsonTool::decode($this->request->data['graph']); + $cycles = []; + $isAcyclic = $this->Workflow->workflowGraphTool->isAcyclic($graphData, $cycles); + $edgesMultipleOutput = []; + $hasMultipleOutputConnection = $this->Workflow->workflowGraphTool->hasMultipleOutputConnection($graphData, $edgesMultipleOutput); + $edgesWarnings = []; + $hasPathWarnings = $this->Workflow->hasPathWarnings($graphData, $edgesWarnings); + $data = [ + 'is_acyclic' => [ + 'is_acyclic' => $isAcyclic, + 'cycles' => $cycles, + ], + 'multiple_output_connection' => [ + 'has_multiple_output_connection' => $hasMultipleOutputConnection, + 'edges' => $edgesMultipleOutput, + ], + 'path_warnings' => [ + 'has_path_warnings' => $hasPathWarnings, + 'edges' => $edgesWarnings, + ], + ]; + return $this->RestResponse->viewData($data, 'json'); + } +} diff --git a/app/Lib/Export/HashesExport.php b/app/Lib/Export/HashesExport.php index e5c1ff7e7..76fae440f 100644 --- a/app/Lib/Export/HashesExport.php +++ b/app/Lib/Export/HashesExport.php @@ -6,16 +6,16 @@ class HashesExport 'flatten' => 1 ); - public $validTypes = array( + const VALID_TYPES = array( 'simple' => array( - 'md5', 'sha1', 'sha256', 'sha224', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'tlsh', - 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'pehash', 'authentihash', - 'impfuzzy' + 'md5', 'sha1', 'sha256', 'sha224', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', + 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'tlsh', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', + 'x509-fingerprint-sha256', 'pehash', 'authentihash', 'impfuzzy' ), 'composite' => array( 'malware-sample', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha224', 'filename|sha512', - 'filename|sha512/224', 'filename|sha512/256', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh', - 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'filename|pehash', + 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', + 'filename|sha3-512', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh', 'filename|pehash', 'filename|authentihash', 'filename|impfuzzy' ) ); @@ -23,18 +23,17 @@ class HashesExport public function handler($data, $options = array()) { if ($options['scope'] === 'Attribute') { - if (in_array($data['Attribute']['type'], $this->validTypes['composite'])) { + if (in_array($data['Attribute']['type'], self::VALID_TYPES['composite'], true)) { return explode('|', $data['Attribute']['value'])[1]; - } else if (in_array($data['Attribute']['type'], $this->validTypes['simple'])) { + } else if (in_array($data['Attribute']['type'], self::VALID_TYPES['simple'], true)) { return $data['Attribute']['value']; } - } - if ($options['scope'] === 'Event') { + } else if ($options['scope'] === 'Event') { $result = array(); foreach ($data['Attribute'] as $attribute) { - if (in_array($attribute['type'], $this->validTypes['composite'])) { + if (in_array($attribute['type'], self::VALID_TYPES['composite'], true)) { $result[] = explode('|', $attribute['value'])[1]; - } else if (in_array($attribute['type'], $this->validTypes['simple'])) { + } else if (in_array($attribute['type'], self::VALID_TYPES['simple'], true)) { $result[] = $attribute['value']; } } diff --git a/app/Lib/Export/JsonExport.php b/app/Lib/Export/JsonExport.php index 840d0aea1..23e59028a 100644 --- a/app/Lib/Export/JsonExport.php +++ b/app/Lib/Export/JsonExport.php @@ -2,7 +2,7 @@ class JsonExport { - public $non_restrictive_export = true; + public $non_restrictive_export = true; /** * @param $data @@ -11,17 +11,17 @@ class JsonExport */ public function handler($data, $options = array()) { - if ($options['scope'] === 'Attribute') { - return $this->__attributeHandler($data, $options); - } else if($options['scope'] === 'Event') { - return $this->__eventHandler($data, $options); - } else if($options['scope'] === 'Object') { + if ($options['scope'] === 'Attribute') { + return $this->__attributeHandler($data, $options); + } else if ($options['scope'] === 'Event') { + return $this->__eventHandler($data, $options); + } else if ($options['scope'] === 'Object') { return $this->__objectHandler($data, $options); - } else if($options['scope'] === 'Sighting') { - return $this->__sightingsHandler($data, $options); - } else if($options['scope'] === 'GalaxyCluster') { - return $this->__galaxyClusterHandler($data, $options); - } + } else if ($options['scope'] === 'Sighting') { + return $this->__sightingsHandler($data, $options); + } else if ($options['scope'] === 'GalaxyCluster') { + return $this->__galaxyClusterHandler($data, $options); + } } /** @@ -29,66 +29,68 @@ class JsonExport * @param array $options * @return Generator */ - private function __eventHandler($event, $options = array()) + private function __eventHandler($event, $options = array()) { App::uses('JSONConverterTool', 'Tools'); - return JSONConverterTool::streamConvert($event); - } - - private function __objectHandler($object, $options = array()) { - App::uses('JSONConverterTool', 'Tools'); - return json_encode(JSONConverterTool::convertObject($object, false, true)); + return JSONConverterTool::streamConvert($event); } - private function __attributeHandler($attribute, $options = array()) - { - $attribute = array_merge($attribute['Attribute'], $attribute); - unset($attribute['Attribute']); - if (isset($attribute['Object']) && empty($attribute['Object']['id'])) { - unset($attribute['Object']); - } - $tagTypes = array('AttributeTag', 'EventTag'); - foreach($tagTypes as $tagType) { - if (isset($attribute[$tagType])) { - foreach ($attribute[$tagType] as $tk => $tag) { - if ($tagType === 'EventTag') { - $attribute[$tagType][$tk]['Tag']['inherited'] = 1; - } - $attribute['Tag'][] = $attribute[$tagType][$tk]['Tag']; - } - unset($attribute[$tagType]); - } - } - unset($attribute['value1']); - unset($attribute['value2']); - return json_encode($attribute); - } + private function __objectHandler($object, $options = array()) + { + App::uses('JSONConverterTool', 'Tools'); + return JsonTool::encode(JSONConverterTool::convertObject($object, false, true)); + } + + private function __attributeHandler($attribute, $options = array()) + { + $attribute = array_merge($attribute['Attribute'], $attribute); + unset($attribute['Attribute']); + if (isset($attribute['Object']) && empty($attribute['Object']['id'])) { + unset($attribute['Object']); + } + $tagTypes = array('AttributeTag', 'EventTag'); + foreach ($tagTypes as $tagType) { + if (isset($attribute[$tagType])) { + foreach ($attribute[$tagType] as $tag) { + if ($tagType === 'EventTag') { + $tag['Tag']['inherited'] = 1; + } + $attribute['Tag'][] = $tag['Tag']; + } + unset($attribute[$tagType]); + } + } + unset($attribute['value1']); + unset($attribute['value2']); + return JsonTool::encode($attribute); + } private function __sightingsHandler($sighting, $options = array()) { - return json_encode($sighting); + return JsonTool::encode($sighting); } + private function __galaxyClusterHandler($cluster, $options = array()) { - return json_encode($cluster); + return JsonTool::encode($cluster); } public function header($options = array()) { - if ($options['scope'] === 'Attribute') { - return '{"response": {"Attribute": ['; - } else { - return '{"response": ['; - } + if ($options['scope'] === 'Attribute') { + return '{"response": {"Attribute": ['; + } else { + return '{"response": ['; + } } public function footer($options = array()) { - if ($options['scope'] === 'Attribute') { - return ']}}' . PHP_EOL; - } else { - return ']}' . PHP_EOL; - } + if ($options['scope'] === 'Attribute') { + return ']}}' . PHP_EOL; + } else { + return ']}' . PHP_EOL; + } } public function separator() diff --git a/app/Lib/Export/NidsExport.php b/app/Lib/Export/NidsExport.php index 43fc0e5ef..b44c66ec2 100644 --- a/app/Lib/Export/NidsExport.php +++ b/app/Lib/Export/NidsExport.php @@ -7,84 +7,125 @@ class NidsExport public $classtype = 'trojan-activity'; public $format = ""; // suricata (default), snort + + public $supportedObjects = array('network-connection', 'ddos'); - public $checkWhitelist = true; + public $checkWhitelist = true; - public $additional_params = array( - 'contain' => array( - 'Event' => array( - 'fields' => array('threat_level_id') - ) - ), - 'flatten' => 1 - ); + public $additional_params = array( + 'contain' => array( + 'Event' => array( + 'fields' => array('threat_level_id') + ) + ), - public function handler($data, $options = array()) - { - $continue = empty($format); - $this->checkWhitelist = false; - if ($options['scope'] === 'Attribute') { - $this->export( - array($data), - $options['user']['nids_sid'], - $options['returnFormat'], - $continue - ); - } else if ($options['scope'] === 'Event') { - if (!empty($data['EventTag'])) { - $data['Event']['EventTag'] = $data['EventTag']; - } - if (!empty($data['Attribute'])) { - $this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue); - } - if (!empty($data['Object'])) { - foreach ($data['Object'] as $object) { - $this->__convertFromEventFormat($object['Attribute'], $data, $options, $continue); - } - } - } - return ''; - } + ); - private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) { - $rearranged = array(); - foreach ($attributes as $attribute) { - $attributeTag = array(); - if (!empty($attribute['AttributeTag'])) { - $attributeTag = $attribute['AttributeTag']; - unset($attribute['AttributeTag']); - } - $rearranged[] = array( - 'Attribute' => $attribute, - 'AttributeTag' => $attributeTag, - 'Event' => $event['Event'] - ); - } - $this->export( - $rearranged, - $options['user']['nids_sid'], - $options['returnFormat'], - $continue - ); - return true; + public function handler($data, $options = array()) + { + $continue = empty($format); + $this->checkWhitelist = false; + if ($options['scope'] === 'Attribute') { + $this->export( + array($data), + $options['user']['nids_sid'], + $options['returnFormat'], + $continue + ); + } else if ($options['scope'] === 'Event') { + if (!empty($data['EventTag'])) { + $data['Event']['EventTag'] = $data['EventTag']; + } + if (!empty($data['Attribute'])) { + $this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue); + } + if (!empty($data['Object'])) { + $this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue); + } + } + return ''; + } - } + private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) { - public function header($options = array()) - { - $this->explain(); - return ''; - } + $rearranged = array(); + foreach ($attributes as $attribute) { + $attributeTag = array(); + if (!empty($attribute['AttributeTag'])) { + $attributeTag = $attribute['AttributeTag']; + unset($attribute['AttributeTag']); + } + $rearranged[] = array( + 'Attribute' => $attribute, + 'AttributeTag' => $attributeTag, + 'Event' => $event['Event'] + ); + } + $this->export( + $rearranged, + $options['user']['nids_sid'], + $options['returnFormat'], + $continue + ); + return true; - public function footer() - { - return implode ("\n", $this->rules); - } + } - public function separator() - { - return ''; - } + private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false) { + + $rearranged = array(); + foreach ($objects as $object) { + + if(in_array($object['name'], $this->supportedObjects)){ + + $objectTag = array(); + + foreach($object['Attribute'] as $attribute) { + + if (!empty($attribute['AttributeTag'])) { + $objectTag = array_merge($objectTag, $attribute['AttributeTag']); + unset($attribute['AttributeTag']); + } + + } + + $rearranged[] = array( + 'Attribute' => $object, // Using 'Attribute' instead of 'Object' to comply with function export + 'AttributeTag' => $objectTag, // Using 'AttributeTag' instead of 'ObjectTag' to comply with function export + 'Event' => $event['Event'] + ); + + } else { // In case no custom export exists for the object, the approach falls back to the attribute case + $this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue); + } + + } + + $this->export( + $rearranged, + $options['user']['nids_sid'], + $options['returnFormat'], + $continue + ); + return true; + + } + + public function header($options = array()) + { + $this->explain(); + return ''; + } + + public function footer() + { + return implode ("\n", $this->rules); + } + + public function separator() + { + return ''; + } public function explain() { @@ -93,7 +134,7 @@ class NidsExport $this->rules[] = '# These NIDS rules contain some variables that need to exist in your configuration.'; $this->rules[] = '# Make sure you have set:'; $this->rules[] = '#'; - $this->rules[] = '# $HOME_NET - Your internal network range'; + $this->rules[] = '# $HOME_NET - Your internal network range'; $this->rules[] = '# $EXTERNAL_NET - The network considered as outside'; $this->rules[] = '# $SMTP_SERVERS - All your internal SMTP servers'; $this->rules[] = '# $HTTP_PORTS - The ports used to contain HTTP traffic (not required with suricata export)'; @@ -106,10 +147,10 @@ class NidsExport public function export($items, $startSid, $format="suricata", $continue = false) { $this->format = $format; - if ($this->checkWhitelist && !isset($this->Whitelist)) { - $this->Whitelist = ClassRegistry::init('Whitelist'); - $this->whitelist = $this->Whitelist->getBlockedValues(); - } + if ($this->checkWhitelist && !isset($this->Whitelist)) { + $this->Whitelist = ClassRegistry::init('Whitelist'); + $this->whitelist = $this->Whitelist->getBlockedValues(); + } // output a short explanation if (!$continue) { @@ -119,20 +160,20 @@ class NidsExport foreach ($items as $item) { // retrieve all tags for this item to add them to the msg $tagsArray = []; - if (!empty($item['AttributeTag'])) { - foreach ($item['AttributeTag'] as $tag_attr) { - if (array_key_exists('name', $tag_attr['Tag'])) { - array_push($tagsArray, $tag_attr['Tag']['name']); - } - } - } - if (!empty($item['Event']['EventTag'])) { - foreach ($item['Event']['EventTag'] as $tag_event) { - if (array_key_exists('name', $tag_event['Tag'])) { - array_push($tagsArray, $tag_event['Tag']['name']); - } - } - } + if (!empty($item['AttributeTag'])) { + foreach ($item['AttributeTag'] as $tag_attr) { + if (array_key_exists('name', $tag_attr['Tag'])) { + array_push($tagsArray, $tag_attr['Tag']['name']); + } + } + } + if (!empty($item['Event']['EventTag'])) { + foreach ($item['Event']['EventTag'] as $tag_event) { + if (array_key_exists('name', $tag_event['Tag'])) { + array_push($tagsArray, $tag_event['Tag']['name']); + } + } + } $ruleFormatMsgTags = implode(",", $tagsArray); # proto src_ip src_port direction dst_ip dst_port msg rule_content tag sid rev @@ -142,69 +183,179 @@ class NidsExport $sid = $startSid + ($item['Attribute']['id'] * 10); // leave 9 possible rules per attribute type $sid++; - switch ($item['Attribute']['type']) { - // LATER nids - test all the snort attributes - // LATER nids - add the tag keyword in the rules to capture network traffic - // LATER nids - sanitize every $attribute['value'] to not conflict with snort - case 'ip-dst': - $this->ipDstRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'ip-src': - $this->ipSrcRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'ip-dst|port': - $this->ipDstRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'ip-src|port': - $this->ipSrcRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'email': - $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid++); - $this->emailDstRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'email-src': - $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'email-dst': - $this->emailDstRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'email-subject': - $this->emailSubjectRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'email-attachment': - $this->emailAttachmentRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'domain': - $this->domainRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'domain|ip': - $this->domainIpRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'hostname': - $this->hostnameRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'url': - $this->urlRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'user-agent': - $this->userAgentRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'ja3-fingerprint-md5': - $this->ja3Rule($ruleFormat, $item['Attribute'], $sid); - break; - case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created. - $this->ja3sRule($ruleFormat, $item['Attribute'], $sid); - break; - case 'snort': - $this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); - // no break - default: - break; + + if(!empty($item['Attribute']['type'])) { // item is an 'Attribute' + + switch ($item['Attribute']['type']) { + // LATER nids - test all the snort attributes + // LATER nids - add the tag keyword in the rules to capture network traffic + // LATER nids - sanitize every $attribute['value'] to not conflict with snort + case 'ip-dst': + $this->ipDstRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ip-src': + $this->ipSrcRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ip-dst|port': + $this->ipDstRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ip-src|port': + $this->ipSrcRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'email': + $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid); + $this->emailDstRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'email-src': + $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'email-dst': + $this->emailDstRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'email-subject': + $this->emailSubjectRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'email-attachment': + $this->emailAttachmentRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'domain': + $this->domainRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'domain|ip': + $this->domainIpRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'hostname': + $this->hostnameRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'url': + $this->urlRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'user-agent': + $this->userAgentRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ja3-fingerprint-md5': + $this->ja3Rule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created. + $this->ja3sRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'snort': + $this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); + // no break + default: + break; + } + + } else if(!empty($item['Attribute']['name'])) { // Item is an 'Object' + + switch ($item['Attribute']['name']) { + case 'network-connection': + $this->networkConnectionRule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ddos': + $this->ddosRule($ruleFormat, $item['Attribute'], $sid); + break; + default: + break; + } + } + } + return $this->rules; } + public function networkConnectionRule($ruleFormat, $object, &$sid) + { + + $attributes = NidsExport::getObjectAttributes($object); + + if(!array_key_exists('layer4-protocol', $attributes)){ + $attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip') + } + if(!array_key_exists('ip-src', $attributes)){ + $attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET + } + if(!array_key_exists('ip-dst', $attributes)){ + $attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET + } + if(!array_key_exists('src-port', $attributes)){ + $attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any' + } + if(!array_key_exists('dst-port', $attributes)){ + $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' + } + + $this->rules[] = sprintf( + $ruleFormat, + false, + $attributes['layer4-protocol'], // proto + $attributes['ip-src'], // src_ip + $attributes['src-port'], // src_port + '->', // direction + $attributes['ip-dst'], // dst_ip + $attributes['dst-port'], // dst_port + 'Network connection between ' . $attributes['ip-src'] . ' and ' . $attributes['ip-dst'], // msg + '', // rule_content + '', // tag + $sid, // sid + 1 // rev + ); + + } + + public function ddosRule($ruleFormat, $object, &$sid) + { + + $attributes = NidsExport::getObjectAttributes($object); + + if(!array_key_exists('protocol', $attributes)){ + $attributes['protocol'] = 'ip'; // If protocol is unknown, we roll-back to 'ip' + } + if(!array_key_exists('ip-src', $attributes)){ + $attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET + } + if(!array_key_exists('ip-dst', $attributes)){ + $attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET + } + if(!array_key_exists('src-port', $attributes)){ + $attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any' + } + if(!array_key_exists('dst-port', $attributes)){ + $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' + } + + $this->rules[] = sprintf( + $ruleFormat, + false, + $attributes['protocol'], // proto + $attributes['ip-src'], // src_ip + $attributes['src-port'], // src_port + '->', // direction + $attributes['ip-dst'], // dst_ip + $attributes['dst-port'], // dst_port + 'DDOS attack detected between ' . $attributes['ip-src'] . ' and ' . $attributes['ip-dst'], // msg + '', // rule_content + '', // tag + $sid, // sid + 1 // rev + ); + + } + + public static function getObjectAttributes($object) + { + + $attributes = array(); + + foreach ($object['Attribute'] as $attribute) { + $attributes[$attribute['object_relation']] = $attribute['value']; + } + + return $attributes; + } + public function domainIpRule($ruleFormat, $attribute, &$sid) { $values = explode('|', $attribute['value']); @@ -225,17 +376,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'ip', // proto - '$HOME_NET', // src_ip - 'any', // src_port - '->', // direction - $ipport[0], // dst_ip - $ipport[1], // dst_port - 'Outgoing To IP: ' . $attribute['value'], // msg - '', // rule_content - '', // tag - $sid, // sid - 1 // rev + 'ip', // proto + '$HOME_NET', // src_ip + 'any', // src_port + '->', // direction + $ipport[0], // dst_ip + $ipport[1], // dst_port + 'Outgoing To IP: ' . $attribute['value'], // msg + '', // rule_content + '', // tag + $sid, // sid + 1 // rev ); } @@ -246,17 +397,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'ip', // proto - $ipport[0], // src_ip - $ipport[1], // src_port - '->', // direction - '$HOME_NET', // dst_ip - 'any', // dst_port - 'Incoming From IP: ' . $attribute['value'], // msg - '', // rule_content - '', // tag - $sid, // sid - 1 // rev + 'ip', // proto + $ipport[0], // src_ip + $ipport[1], // src_port + '->', // direction + '$HOME_NET', // dst_ip + 'any', // dst_port + 'Incoming From IP: ' . $attribute['value'], // msg + '', // rule_content + '', // tag + $sid, // sid + 1 // rev ); } @@ -268,17 +419,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$EXTERNAL_NET', // src_ip - 'any', // src_port - '->', // direction - '$SMTP_SERVERS', // dst_ip - '25', // dst_port - 'Source Email Address: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$EXTERNAL_NET', // src_ip + 'any', // src_port + '->', // direction + '$SMTP_SERVERS', // dst_ip + '25', // dst_port + 'Source Email Address: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -290,17 +441,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$EXTERNAL_NET', // src_ip - 'any', // src_port - '->', // direction - '$SMTP_SERVERS', // dst_ip - '25', // dst_port - 'Destination Email Address: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$EXTERNAL_NET', // src_ip + 'any', // src_port + '->', // direction + '$SMTP_SERVERS', // dst_ip + '25', // dst_port + 'Destination Email Address: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -313,17 +464,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$EXTERNAL_NET', // src_ip - 'any', // src_port - '->', // direction - '$SMTP_SERVERS', // dst_ip - '25', // dst_port - 'Bad Email Subject', // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$EXTERNAL_NET', // src_ip + 'any', // src_port + '->', // direction + '$SMTP_SERVERS', // dst_ip + '25', // dst_port + 'Bad Email Subject', // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -336,17 +487,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$EXTERNAL_NET', // src_ip - 'any', // src_port - '->', // direction - '$SMTP_SERVERS', // dst_ip - '25', // dst_port - 'Bad Email Attachment', // msg - $content, // rule_content // LATER nids - test and finetune this snort rule https://secure.wikimedia.org/wikipedia/en/wiki/MIME#Content-Disposition - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$EXTERNAL_NET', // src_ip + 'any', // src_port + '->', // direction + '$SMTP_SERVERS', // dst_ip + '25', // dst_port + 'Bad Email Attachment', // msg + $content, // rule_content // LATER nids - test and finetune this snort rule https://secure.wikimedia.org/wikipedia/en/wiki/MIME#Content-Disposition + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -358,33 +509,33 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'udp', // proto - 'any', // src_ip - 'any', // src_port - '->', // direction - 'any', // dst_ip - '53', // dst_port - 'Hostname: ' . $attribute['value'], // msg - $content, // rule_content - '', // tag - $sid, // sid - 1 // rev + 'udp', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + '53', // dst_port + 'Hostname: ' . $attribute['value'], // msg + $content, // rule_content + '', // tag + $sid, // sid + 1 // rev ); $sid++; $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - 'any', // src_ip - 'any', // src_port - '->', // direction - 'any', // dst_ip - '53', // dst_port - 'Hostname: ' . $attribute['value'], // msg - $content. ' flow:established;', // rule_content - '', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + '53', // dst_port + 'Hostname: ' . $attribute['value'], // msg + $content. ' flow:established;', // rule_content + '', // tag + $sid, // sid + 1 // rev ); $sid++; // also do http requests @@ -392,17 +543,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$HOME_NET', // src_ip - 'any', // src_port - '->', // direction - '$EXTERNAL_NET', // dst_ip - '$HTTP_PORTS', // dst_port - 'Outgoing HTTP Hostname: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$HOME_NET', // src_ip + 'any', // src_port + '->', // direction + '$EXTERNAL_NET', // dst_ip + '$HTTP_PORTS', // dst_port + 'Outgoing HTTP Hostname: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -414,33 +565,33 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'udp', // proto - 'any', // src_ip - 'any', // src_port - '->', // direction - 'any', // dst_ip - '53', // dst_port - 'Domain: ' . $attribute['value'], // msg - $content, // rule_content - '', // tag - $sid, // sid - 1 // rev + 'udp', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + '53', // dst_port + 'Domain: ' . $attribute['value'], // msg + $content, // rule_content + '', // tag + $sid, // sid + 1 // rev ); $sid++; $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - 'any', // src_ip - 'any', // src_port - '->', // direction - 'any', // dst_ip - '53', // dst_port - 'Domain: ' . $attribute['value'], // msg - $content. ' flow:established;', // rule_content - '', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + '53', // dst_port + 'Domain: ' . $attribute['value'], // msg + $content. ' flow:established;', // rule_content + '', // tag + $sid, // sid + 1 // rev ); $sid++; // also do http requests, @@ -448,17 +599,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$HOME_NET', // src_ip - 'any', // src_port - '->', // direction - '$EXTERNAL_NET', // dst_ip - '$HTTP_PORTS', // dst_port - 'Outgoing HTTP Domain: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$HOME_NET', // src_ip + 'any', // src_port + '->', // direction + '$EXTERNAL_NET', // dst_ip + '$HTTP_PORTS', // dst_port + 'Outgoing HTTP Domain: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -473,17 +624,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$HOME_NET', // src_ip - 'any', // src_port - '->', // direction - '$EXTERNAL_NET', // dst_ip - '$HTTP_PORTS', // dst_port - 'Outgoing HTTP URL: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$HOME_NET', // src_ip + 'any', // src_port + '->', // direction + '$EXTERNAL_NET', // dst_ip + '$HTTP_PORTS', // dst_port + 'Outgoing HTTP URL: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -495,17 +646,17 @@ class NidsExport $this->rules[] = sprintf( $ruleFormat, ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', - 'tcp', // proto - '$HOME_NET', // src_ip - 'any', // src_port - '->', // direction - '$EXTERNAL_NET', // dst_ip - '$HTTP_PORTS', // dst_port - 'Outgoing User-Agent: ' . $attribute['value'], // msg - $content, // rule_content - 'tag:session,600,seconds;', // tag - $sid, // sid - 1 // rev + 'tcp', // proto + '$HOME_NET', // src_ip + 'any', // src_port + '->', // direction + '$EXTERNAL_NET', // dst_ip + '$HTTP_PORTS', // dst_port + 'Outgoing User-Agent: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev ); } @@ -527,37 +678,37 @@ class NidsExport $tmpRule = str_replace(array("\r","\n"), " ", $attribute['value']); // rebuild the rule by overwriting the different keywords using preg_replace() - // sid - '/sid\s*:\s*[0-9]+\s*;/' - // rev - '/rev\s*:\s*[0-9]+\s*;/' + // sid - '/sid\s*:\s*[0-9]+\s*;/' + // rev - '/rev\s*:\s*[0-9]+\s*;/' // classtype - '/classtype:[a-zA-Z_-]+;/' - // msg - '/msg\s*:\s*".*?"\s*;/' + // msg - '/msg\s*:\s*".*?"\s*;/' // reference - '/reference\s*:\s*.+?;/' - // tag - '/tag\s*:\s*.+?;/' + // tag - '/tag\s*:\s*.+?;/' $replaceCount = array(); $tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex $tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex $tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex $tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); if (null == $tmpRule) { return false; - } // don't output the rule on error with the regex + } // don't output the rule on error with the regex // FIXME nids - implement priority overwriting // some values were not replaced, so we need to add them ourselves, and insert them in the rule @@ -667,13 +818,13 @@ class NidsExport public function checkWhitelist($value) { - if ($this->checkWhitelist && is_array($this->whitelist)) { - foreach ($this->whitelist as $wlitem) { - if (preg_match($wlitem, $value)) { - return true; - } - } - } + if ($this->checkWhitelist && is_array($this->whitelist)) { + foreach ($this->whitelist as $wlitem) { + if (preg_match($wlitem, $value)) { + return true; + } + } + } return false; } @@ -717,4 +868,4 @@ class NidsExport } return $ipport; } -} +} \ No newline at end of file diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 14f764218..110cf7dfe 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -7,17 +7,17 @@ App::uses('BackgroundJob', 'Tools/BackgroundJobs'); /** * BackgroundJobs Tool - * + * * Utility class to queue jobs, run them and monitor workers. - * + * * To run a worker manually (debug only): * $ ./Console/cake start_worker [queue] - * + * * It is recommended to run these commands with [Supervisor](http://supervisord.org). - * `Supervisor` has an extensive feature set to manage scripts as services, - * such as autorestart, parallel execution, logging, monitoring and much more. + * `Supervisor` has an extensive feature set to manage scripts as services, + * such as autorestart, parallel execution, logging, monitoring and much more. * All can be managed via the terminal or a XML-RPC API. - * + * * Use the following configuration as a template for the services: * /etc/supervisor/conf.d/misp-workers.conf: * [group:misp-workers] @@ -27,14 +27,14 @@ App::uses('BackgroundJob', 'Tools/BackgroundJobs'); * [program:default] * command=/var/www/MISP/app/Console/cake start_worker default * process_name=%(program_name)s_%(process_num)02d - * numprocs=5 ; adjust the amount of parallel workers to your MISP usage + * numprocs=5 ; adjust the amount of parallel workers to your MISP usage * autostart=true * autorestart=true * redirect_stderr=false * stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log * stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log * user=www-data - * + * */ class BackgroundJobsTool { @@ -73,18 +73,21 @@ class BackgroundJobsTool const CMD_EVENT = 'event', CMD_SERVER = 'server', - CMD_ADMIN = 'admin'; + CMD_ADMIN = 'admin', + CMD_WORKFLOW = 'workflow'; const ALLOWED_COMMANDS = [ self::CMD_EVENT, self::CMD_SERVER, - self::CMD_ADMIN + self::CMD_ADMIN, + self::CMD_WORKFLOW, ]; const CMD_TO_SHELL_DICT = [ self::CMD_EVENT => 'EventShell', self::CMD_SERVER => 'ServerShell', - self::CMD_ADMIN => 'AdminShell' + self::CMD_ADMIN => 'AdminShell', + self::CMD_WORKFLOW => 'WorkflowShell', ]; const JOB_STATUS_PREFIX = 'job_status'; @@ -176,7 +179,7 @@ class BackgroundJobsTool /** * Enqueue a Job using the CakeResque. * @deprecated - * + * * @param string $queue Name of the queue to enqueue the job to. * @param string $class Class of the job. * @param array $args Arguments passed to the job. @@ -212,9 +215,9 @@ class BackgroundJobsTool * * @param string $queue Queue name, e.g. 'default'. * @param int $timeout Time to block the read if the queue is empty. - * Must be less than your configured `read_write_timeout` + * Must be less than your configured `read_write_timeout` * for the redis connection. - * + * * @throws Exception */ public function dequeue($queue, int $timeout = 30) @@ -262,7 +265,7 @@ class BackgroundJobsTool * Clear all the queue's jobs. * * @param string $queue Queue name, e.g. 'default'. - * + * * @return boolean True on success, false on failure. */ public function clearQueue($queue): bool @@ -309,7 +312,7 @@ class BackgroundJobsTool * Get the number of jobs inside a queue. * * @param string $queue Queue name, e.g. 'default'. - * + * * @return integer Number of jobs. */ public function getQueueSize(string $queue): int @@ -327,7 +330,7 @@ class BackgroundJobsTool * Update job * * @param BackgroundJob $job - * + * * @return void */ public function update(BackgroundJob $job) diff --git a/app/Lib/Tools/BetterCakeEventManager.php b/app/Lib/Tools/BetterCakeEventManager.php new file mode 100644 index 000000000..9c0f41baf --- /dev/null +++ b/app/Lib/Tools/BetterCakeEventManager.php @@ -0,0 +1,63 @@ +listeners($event->name()); + if (empty($listeners)) { + return null; + } + + foreach ($listeners as $listener) { + if ($event->isStopped()) { + break; + } + if ($listener['passParams'] === true) { + $result = call_user_func_array($listener['callable'], $event->data); + } else { + $result = $listener['callable']($event); + } + if ($result === false) { + $event->stopPropagation(); + } + if ($result !== null) { + $event->result = $result; + } + } + } + + /** + * @param $eventKey + * @return array + */ + public function listeners($eventKey) + { + if ($this->_isGlobal) { + $localListeners = []; + } else { + $localListeners = $this->_listeners[$eventKey] ?? []; + } + + $globalListeners = static::instance()->prioritisedListeners($eventKey); + + $priorities = array_merge(array_keys($globalListeners), array_keys($localListeners)); + $priorities = array_unique($priorities, SORT_REGULAR); + asort($priorities); + + $result = []; + foreach ($priorities as $priority) { + if (isset($globalListeners[$priority])) { + $result = array_merge($result, $globalListeners[$priority]); + } + if (isset($localListeners[$priority])) { + $result = array_merge($result, $localListeners[$priority]); + } + } + return $result; + } +} diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index d6857cb94..b124a6c1c 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -96,7 +96,7 @@ class FileAccessTool * @param bool $createFolder * @throws Exception */ - public static function writeToFile($file, $content, $createFolder = false) + public static function writeToFile($file, $content, $createFolder = false, $append = false) { $dir = dirname($file); if ($createFolder && !is_dir($dir)) { @@ -105,7 +105,7 @@ class FileAccessTool } } - if (file_put_contents($file, $content, LOCK_EX) === false) { + if (file_put_contents($file, $content, LOCK_EX | (!empty($append) ? FILE_APPEND : 0)) === false) { $freeSpace = disk_free_space($dir); throw new Exception("An error has occurred while attempt to write to file `$file`. Maybe not enough space? ($freeSpace bytes left)"); } diff --git a/app/Lib/Tools/GraphvizDOTTool.php b/app/Lib/Tools/GraphvizDOTTool.php new file mode 100644 index 000000000..89fa4ac60 --- /dev/null +++ b/app/Lib/Tools/GraphvizDOTTool.php @@ -0,0 +1,97 @@ + [ + 'margin' => 0, + 'shape' => 'diamond', + ], + 'logic' => [ + 'margin' => 0, + 'shape' => 'parallelogram', + ], + 'action' => [ + 'margin' => 0, + 'shape' => 'box', + ], + ]; + const EDGE_STYLE = [ + ]; + + /** + * dot Get DOT language format of the provided graph + * + * @return string + */ + public static function dot(array $graph_data) + { + $parsedGraph = self::__parseGraph($graph_data); + $str = self::__header(); + $str .= self::__nodes($parsedGraph['nodes']); + $str .= self::__edges($parsedGraph['edges']); + $str .= self::__footer(); + return $str; + } + + private static function __parseGraph($graph_data) + { + $graphUtil = new GraphUtil($graph_data); + $nodes = $graphUtil->graph; + $edges = $graphUtil->edgeList; + return [ + 'nodes' => $nodes, + 'edges' => $edges, + ]; + } + + private static function __header() + { + return 'digraph G {' . PHP_EOL; + } + + private static function __footer() + { + return '}'; + } + + private static function __nodes($nodes) + { + $str = ' {' . PHP_EOL; + foreach ($nodes as $node) { + $str .= ' ' . self::__node($node); + } + $str .= ' }' . PHP_EOL; + return $str; + } + + private static function __node(array $node) + { + $node_attributes = self::NODE_STYLE[$node['data']['module_type']]; + $node_attributes['label'] = $node['data']['name']; + $node_attributes_text = self::__arrayToAttributes($node_attributes); + return sprintf('%s [%s]' . PHP_EOL, $node['id'], $node_attributes_text); + } + + private static function __edges($edges) + { + $str = ''; + foreach ($edges as $source_id => $target_ids) { + foreach ($target_ids as $target_id) { + $str .= ' ' . self::__edge($source_id, $target_id); + } + } + return $str; + } + + private static function __edge($source_id, $target_id) + { + return sprintf('%s -> %s [%s]' . PHP_EOL, $source_id, $target_id, self::__arrayToAttributes(self::EDGE_STYLE)); + } + + private static function __arrayToAttributes(array $list) + { + return implode(', ', array_map(function ($key, $value) { + return sprintf('%s="%s"', $key, $value); + }, array_keys($list), $list)); + } +} diff --git a/app/Lib/Tools/MermaidFlowchartTool.php b/app/Lib/Tools/MermaidFlowchartTool.php new file mode 100644 index 000000000..5db77ed46 --- /dev/null +++ b/app/Lib/Tools/MermaidFlowchartTool.php @@ -0,0 +1,83 @@ + '{{%s}}', + 'logic' => '[/%s/]', + 'action' => '[%s]', + ]; + + /** + * dot Get DOT language format of the provided graph + * + * @return string + */ + public static function mermaid(array $graph_data) + { + $parsedGraph = self::__parseGraph($graph_data); + $str = self::__header(); + $str .= self::__nodes($parsedGraph['nodes'], $parsedGraph['edges']); + $str .= self::__footer(); + return $str; + } + + private static function __parseGraph($graph_data) + { + $graphUtil = new GraphUtil($graph_data); + $nodes = Hash::combine($graphUtil->graph, '{n}.id', '{n}'); + $edges = $graphUtil->edgeList; + return [ + 'nodes' => $nodes, + 'edges' => $edges, + ]; + } + + private static function __header() + { + return 'flowchart LR' . PHP_EOL; + } + + private static function __footer() + { + return ''; + } + + private static function __nodes($nodes, $edges) + { + $str = ''; + foreach ($nodes as $node) { + $str .= self::__node($nodes, $node, $edges[$node['id']]); + } + return $str; + } + + private static function __node(array $all_nodes, array $node, array $edges) + { + $str = ''; + foreach ($edges as $target_id) { + if (empty($all_nodes[$target_id])) { + continue; + } + $target_node = $all_nodes[$target_id]; + $sourceNode = self::__singleNode($node); + $targetNode = self::__singleNode($target_node); + $str .= ' ' . sprintf('%s --> %s', $sourceNode, $targetNode) . PHP_EOL; + } + return $str; + } + + private static function __singleNode(array $node) + { + $str = $node['id']; + $icon = sprintf("%s:fa-%s ", FontAwesomeHelper::findNamespace($node['data']['module_data']['icon']), $node['data']['module_data']['icon']); + $node_content = sprintf('"%s%s"',(!empty($node['data']['module_data']['icon']) ? "$icon " : ''), $node['name']); + $str .= sprintf( + self::NODE_STYLE[$node['data']['module_type']], + $node_content + ); + return $str; + } +} diff --git a/app/Lib/Tools/ProcessTool.php b/app/Lib/Tools/ProcessTool.php index 08b021e29..14eb8ce4c 100644 --- a/app/Lib/Tools/ProcessTool.php +++ b/app/Lib/Tools/ProcessTool.php @@ -1,7 +1,7 @@ stderr = $stderr; $this->stdout = $stdout; parent::__construct($message, $returnCode); @@ -48,13 +49,13 @@ class ProcessTool public static function execute(array $command, $cwd = null, $stderrToFile = false) { $descriptorSpec = [ - 1 => ["pipe", "w"], // stdout - 2 => ["pipe", "w"], // stderr + 1 => ['pipe', 'w'], // stdout + 2 => ['pipe', 'w'], // stderr ]; if ($stderrToFile) { self::logMessage('Running command ' . implode(' ', $command)); - $descriptorSpec[2] = ["file", self::LOG_FILE, 'a']; + $descriptorSpec[2] = ['file', self::LOG_FILE, 'a']; } // PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually diff --git a/app/Lib/Tools/PubSubTool.php b/app/Lib/Tools/PubSubTool.php index ef4cef1d3..6a17a020a 100644 --- a/app/Lib/Tools/PubSubTool.php +++ b/app/Lib/Tools/PubSubTool.php @@ -62,9 +62,13 @@ class PubSubTool if ($response === null) { throw new Exception("No response from status command returned after 5 seconds."); } - return json_decode(trim($response[1]), true); + return JsonTool::decode(trim($response[1])); } + /** + * @return bool + * @throws ProcessException + */ public function checkIfPythonLibInstalled() { $script = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py'; @@ -143,6 +147,12 @@ class PubSubTool return $this->pushToRedis('data:misp_json_warninglist', $warninglist); } + public function workflow_push(array $data) + { + $topic = 'data:misp_json_workflow'; + return $this->pushToRedis($topic, $data); + } + /** * @param array $data * @param string $type @@ -301,7 +311,7 @@ class PubSubTool $pluginConfig = Configure::read('Plugin'); foreach ($settings as $key => $setting) { - $temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null; + $temp = $pluginConfig['ZeroMQ_' . $key] ?? null; if ($temp) { $settings[$key] = $temp; } diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index 4168c6574..95bef7f36 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -431,7 +431,7 @@ class SecurityAudit App::uses('CakeEmail', 'Network/Email'); $email = new CakeEmail(); $emailConfig = $email->config(); - if ($emailConfig['transport'] === 'Smtp' && $emailConfig['port'] == 25 && !$emailConfig['tls']) { + if ($emailConfig['transport'] === 'Smtp' && $emailConfig['port'] == 25 && empty($emailConfig['tls'])) { $output['Email'][] = [ 'warning', __('STARTTLS is not enabled.'), diff --git a/app/Lib/Tools/SendEmail.php b/app/Lib/Tools/SendEmail.php index 32ed8bded..aaec3a05b 100644 --- a/app/Lib/Tools/SendEmail.php +++ b/app/Lib/Tools/SendEmail.php @@ -484,6 +484,10 @@ class SendEmail ]); } + if ($body instanceof SendEmailTemplate && $body->listUnsubscribe()) { + $email->addHeaders(['List-Unsubscribe' => "<{$body->listUnsubscribe()}>"]); + } + $signed = false; if (Configure::read('GnuPG.sign')) { if (!$this->gpg) { diff --git a/app/Lib/Tools/SendEmailTemplate.php b/app/Lib/Tools/SendEmailTemplate.php index 3c385cc22..c405e39f7 100644 --- a/app/Lib/Tools/SendEmailTemplate.php +++ b/app/Lib/Tools/SendEmailTemplate.php @@ -10,6 +10,9 @@ class SendEmailTemplate /** @var string|null */ private $referenceId; + /** @var string */ + private $listUnsubscribe; + /** @var string|null */ private $subject; @@ -31,6 +34,18 @@ class SendEmailTemplate $this->referenceId = $referenceId; } + /** + * @param string|null $listUnsubscribe + * @return string|void + */ + public function listUnsubscribe($listUnsubscribe = null) + { + if ($listUnsubscribe === null) { + return $this->listUnsubscribe; + } + $this->listUnsubscribe = $listUnsubscribe; + } + /** * Get subject from template. Must be called after render method. * @param string|null $subject diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index cff26a9f4..9cdbaaca5 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -1,5 +1,6 @@ get($url); } + /** + * @param array $events + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function filterEventIdsForPush(array $events) + { + return $this->post('/events/filterEventIdsForPush', $events); + } + /** * @param array $event * @return HttpSocketResponseExtended @@ -164,6 +179,39 @@ class ServerSyncTool return $this->post('/attributes/restSearch.json', $rules); } + /** + * @param array $rules + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function galaxyClusterSearch(array $rules) + { + return $this->post('/galaxy_clusters/restSearch', $rules); + } + + /** + * @param int|string $galaxyClusterId Galaxy Cluster ID or UUID + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + */ + public function fetchGalaxyCluster($galaxyClusterId) + { + return $this->get('/galaxy_clusters/view/' . $galaxyClusterId); + } + + /** + * @param array $cluster + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function pushGalaxyCluster(array $cluster) + { + $logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}"; + return $this->post('/galaxies/pushCluster', [$cluster], $logMessage); + } + /** * @param array $params * @return HttpSocketResponseExtended @@ -332,6 +380,12 @@ class ServerSyncTool case self::FEATURE_PROTECTED_EVENT: $version = explode('.', $info['version']); return $version[0] == 2 && $version[1] == 4 && $version[2] > 155; + case self::FEATURE_EDIT_OF_GALAXY_CLUSTER: + return isset($info['perm_galaxy_editor']); + case self::PERM_SYNC: + return isset($info['perm_sync']) && $info['perm_sync']; + case self::PERM_GALAXY_EDITOR: + return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor']; default: throw new InvalidArgumentException("Invalid flag `$flag` provided"); } @@ -373,7 +427,7 @@ class ServerSyncTool private function post($url, $data, $logMessage = null) { $protectedMode = !empty($data['Event']['protected']); - $data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + $data = JsonTool::encode($data); if ($logMessage && !empty(Configure::read('Security.sync_audit'))) { $pushLogEntry = sprintf( diff --git a/app/Lib/Tools/WorkflowFormatConverterTool.php b/app/Lib/Tools/WorkflowFormatConverterTool.php new file mode 100644 index 000000000..6e8782d77 --- /dev/null +++ b/app/Lib/Tools/WorkflowFormatConverterTool.php @@ -0,0 +1,142 @@ + ['perm_site_admin' => true]]; + + public static function convert(array $data, $scope=''): array + { + if (empty($scope)) { + $scope = self::__guessScopeFromData($data); + } + $converted = []; + switch ($scope) { + case 'event': + $converted = self::__convertEvent($data); + break; + case 'attribute': + $converted = self::__convertAttribute($data); + break; + case 'object': + $converted = self::__convertObject($data); + break; + default: + break; + } + $converted = self::__includeFlattenedAttributes($converted); + return $converted; + } + + private static function __convertEvent(array $event): array + { + $converted = []; + $converted = JSONConverterTool::convert($event, false, true); + return $converted; + } + + private static function __convertObject(array $object): array + { + $converted = []; + $convertedObject = JSONConverterTool::convertObject($object, false, true); + $convertedObject = ['Object' => $convertedObject['Object']]; + $converted = self::__encapsulateEntityWithEvent($convertedObject); + return $converted; + } + + /** + * __convertAttribute Convert and clean an attribute. May also transform the attribute into an Object if applicable. + * However, the object will not be full and will only contain the attribute + * + * @param array $attribute + * @return array + */ + private static function __convertAttribute(array $attribute): array + { + $allTags = []; + if (!empty($attribute['EventTag'])) { + foreach ($attribute['AttributeTag'] as $attributeTag) { + $attributeTag['Tag']['inherited'] = false; + $allTags[] = $attributeTag['Tag']; + } + foreach ($attribute['EventTag'] as $eventTag) { + $eventTag['Tag']['inherited'] = true; + $allTags[] = $eventTag['Tag']; + } + } + $convertedAttribute = JSONConverterTool::convertAttribute($attribute, true); + $convertedAttribute['Attribute']['_allTags'] = $allTags; + if ($convertedAttribute['Attribute']['object_id'] != 0) { + $objectModel = ClassRegistry::init('MispObject'); + $object = $objectModel->fetchObjectSimple(self::$fakeSiteAdminUser, [ + 'conditions' => [ + 'Object.id' => $convertedAttribute['Attribute']['object_id'], + ], + ]); + if (!empty($object)) { + $object = $object[0]['Object']; + $object['Attribute'][] = $convertedAttribute['Attribute']; + $convertedAttribute = ['Object' => $object]; + } else { + $convertedAttribute = ['Attribute' => $convertedAttribute['Attribute']]; + } + } else { + $convertedAttribute = ['Attribute' => $convertedAttribute['Attribute']]; + } + $converted = self::__encapsulateEntityWithEvent($convertedAttribute); + return $converted; + } + + private static function __encapsulateEntityWithEvent(array $data): array + { + $eventModel = ClassRegistry::init('Event'); + $event = $eventModel->fetchSimpleEvent(self::$fakeSiteAdminUser, $data['Attribute']['event_id'] ?? $data['Object']['event_id'], [ + 'contain' => [ + 'EventTag' => ['Tag'] + ] + ]); + if (empty($event)) { + return []; + } + $event = self::__convertEvent($event); + $event = $event['Event']; + reset($data); + $entityType = key($data); + $event[$entityType][] = $data[$entityType]; + return ['Event' => $event]; + } + + private static function __includeFlattenedAttributes(array $event): array + { + $attributes = $event['Event']['Attribute'] ?? []; + $objectAttributes = Hash::extract($event['Event']['Object'] ?? [], '{n}.Attribute.{n}'); + $event['Event']['_AttributeFlattened'] = array_merge($attributes, $objectAttributes); + return $event; + } + + private static function __guessScopeFromData(array $data) + { + if (isset($data['Object']) && !isset($data['Attribute'])) { + return 'object'; + } + if (!isset($data['Attribute'])) { + return 'event'; + } + if (!isset($data['Event'])) { + return 'attribute'; + } + if (isset($data['RelatedEvent']) || isset($data['Orgc']) || isset($data['Org'])) { + return 'event'; + } + if (!empty($data['Attribute'])) { + return 'attribute'; + } + } +} diff --git a/app/Lib/Tools/WorkflowGraphTool.php b/app/Lib/Tools/WorkflowGraphTool.php new file mode 100644 index 000000000..5762d10bc --- /dev/null +++ b/app/Lib/Tools/WorkflowGraphTool.php @@ -0,0 +1,421 @@ +graph = $graphData; + $this->numberNodes = count($this->graph); + $this->edgeList = $this->_buildEdgeList($graphData); + $this->properties = []; + } + + private function _buildEdgeList($graphData): array + { + $list = []; + foreach ($graphData as $node) { + $list[(int)$node['id']] = []; + foreach (($node['outputs'] ?? []) as $output_id => $outputs) { + foreach ($outputs as $connections) { + foreach ($connections as $connection) { + $list[$node['id']][] = (int)$connection['node']; + } + } + } + } + return $list; + } + + private function _DFSUtil($node_id, &$color): bool + { + $color[$node_id] = 'GRAY'; + foreach ($this->edgeList[$node_id] as $i) { + if ($color[$i] == 'GRAY') { + $this->loopNode = $i; + $this->properties[] = [$node_id, $i, __('Cycle')]; + return true; + } + if ($color[$i] == 'WHITE' && $this->_DFSUtil($i, $color)) { + if (!is_null($this->loopNode)) { + $this->properties[] = [$node_id, $i, __('Cycle')]; + if ($this->loopNode == $node_id) { + $this->loopNode = null; + } + } + return true; + } + } + $color[$node_id] = 'BLACK'; + return false; + } + + /** + * isCyclic Return is the graph is cyclic, so if it contains a cycle. + * + * A directed graph G is acyclic if and only if a depth-first search of G yields no back edges. + * Introduction to Algorithms, third edition By Thomas H. Cormen, Charles E. Leiserson, Ronald L. Rivest, Clifford Stein + * + * @return array + */ + public function isCyclic(): array + { + $this->properties = []; + $color = []; + foreach (array_keys($this->edgeList) as $node_id) { + $color[$node_id] = 'WHITE'; + } + + $this->loopNode = null; + foreach (array_keys($this->edgeList) as $node_id) { + if ($color[$node_id] == 'WHITE') { + if ($this->_DFSUtil($node_id, $color)) { + return [true, $this->properties]; + } + } + } + return [false, []]; + } + + public function hasMultipleOutputConnection(): array + { + $edges = []; + foreach ($this->graph as $node) { + foreach (($node['outputs'] ?? []) as $output_id => $outputs) { + foreach ($outputs as $connections) { + if (count($connections) > 1 && empty($node['data']['multiple_output_connection'])) { + $edges[$node['id']] = array_map(function ($connection) { + return intval($connection['node']); + }, $connections); + } + } + } + } + return [!empty($edges), $edges]; + } +} + +class GraphWalker +{ + private $graph; + private $WorkflowModel; + private $startNodeID; + private $for_path; + private $cursor; + + const PATH_TYPE_BLOCKING = 'blocking'; + const PATH_TYPE_NON_BLOCKING = 'non-blocking'; + const PATH_TYPE_INCLUDE_LOGIC = 'include-logic'; + const ALLOWED_PATH_TYPES = [GraphWalker::PATH_TYPE_BLOCKING, GraphWalker::PATH_TYPE_NON_BLOCKING, GraphWalker::PATH_TYPE_INCLUDE_LOGIC]; + + public function __construct(array $graphData, $WorkflowModel, $startNodeID, $for_path=null) + { + $this->graph = $graphData; + $this->WorkflowModel = $WorkflowModel; + $this->startNodeID = $startNodeID; + $this->for_path = $for_path; + $this->triggersByNodeID = []; + if (empty($this->graph[$startNodeID])) { + throw new Exception(__('Could not find start node %s', $startNodeID)); + } + $this->cursor = $startNodeID; + } + + private function getModuleClass($node) + { + $moduleClass = $this->loaded_classes[$node['data']['module_type']][$node['data']['id']] ?? null; + return $moduleClass; + } + + private function _getPathType($node_id, $path_type) + { + $node = $this->graph[$node_id]; + if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') { + return self::PATH_TYPE_NON_BLOCKING; + } + return $path_type; + } + + + private function _evaluateOutputs($node, WorkflowRoamingData $roamingData, $shouldExecuteLogicNode=true) + { + $allowed_outputs = ($node['outputs'] ?? []); + if ($shouldExecuteLogicNode && $node['data']['module_type'] == 'logic') { + $allowed_outputs = $this->_executeModuleLogic($node, $roamingData); + } + return $allowed_outputs; + } + + /** + * _executeModuleLogic function + * + * @param array $node + * @return array + */ + private function _executeModuleLogic(array $node, WorkflowRoamingData $roamingData): array + { + $outputs = ($node['outputs'] ?? []); + if ($node['data']['id'] == 'if') { + $useFirstOutput = $this->_evaluateIFCondition($node, $roamingData); + return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']]; + } else if ($node['data']['id'] == 'concurrent-task') { + $this->_evaluateConcurrentTask($node, $roamingData, $outputs['output_1']); + return ['output_1' => []]; + } else { + $useFirstOutput = $this->_evaluateCustomLogicCondition($node, $roamingData); + return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']]; + } + return $outputs; + } + + private function _evaluateIFCondition($node, WorkflowRoamingData $roamingData): bool + { + $result = $this->WorkflowModel->executeNode($node, $roamingData); + return $result; + } + + private function _evaluateCustomLogicCondition($node, WorkflowRoamingData $roamingData): bool + { + $result = $this->WorkflowModel->executeNode($node, $roamingData); + return $result; + } + + private function _evaluateConcurrentTask($concurrent_node, WorkflowRoamingData $roamingData, array $connections) + { + foreach ($connections['connections'] as $connection) { + $node_id_to_exec = (int)$connection['node']; + $data = $roamingData->getData(); + $data['__node_id_to_exec'] = $node_id_to_exec; + $data = $roamingData->setData($data); + $this->WorkflowModel->executeNode($concurrent_node, $roamingData); + } + } + + public function _walk($node_id, $path_type=null, array $path_list=[], WorkflowRoamingData $roamingData) + { + $this->cursor = $node_id; + $node = $this->graph[$node_id]; + $shouldExecuteLogicNode = $path_type != self::PATH_TYPE_INCLUDE_LOGIC; + if (!$shouldExecuteLogicNode) { + yield ['node' => $node, 'path_type' => $path_type, 'path_list' => $path_list]; + } else if ($node['data']['module_type'] != 'trigger' && $node['data']['module_type'] != 'logic') { // trigger and logic nodes should not be returned as they are "control" nodes + yield ['node' => $node, 'path_type' => $path_type, 'path_list' => $path_list]; + } + $allowedOutputs = $this->_evaluateOutputs($node, $roamingData, $shouldExecuteLogicNode); + foreach ($allowedOutputs as $output_id => $outputs) { + if ($shouldExecuteLogicNode) { + $path_type = $this->_getPathType($node_id, $path_type); + } + if (is_null($this->for_path) || $path_type == $this->for_path) { + foreach ($outputs as $connections) { + foreach ($connections as $connection_id => $connection) { + $next_node_id = (int)$connection['node']; + $current_path = $this->__genPathList($node_id, $output_id, $connection_id, $next_node_id); + if (in_array($current_path, $path_list)) { // avoid loops + continue; + } + $next_path_list = $path_list; + $next_path_list[] = $current_path; + yield from $this->_walk($next_node_id, $path_type, $next_path_list, $roamingData); + } + } + } + } + } + + public function walk(WorkflowRoamingData $roamingData) + { + return $this->_walk($this->cursor, $this->for_path, [], $roamingData); + } + + private function __genPathList($source_id, $output_id, $connection_id, $next_node_id) + { + return sprintf('%s:%s:%s:%s', $source_id, $output_id, $connection_id, $next_node_id); + } + + public static function parsePathList($pathList): array + { + return array_map(function($path) { + $split = explode(':', $path); + return [ + 'source_id' => $split[0], + 'output_id' => $split[1], + 'connection_id' => $split[2], + 'next_node_id' => $split[3], + ]; + }, $pathList); + } +} + +class WorkflowRoamingData +{ + private $workflow_user; + private $data; + private $workflow; + private $current_node; + + public function __construct(array $workflow_user, array $data, array $workflow, int $current_node) + { + $this->workflow_user = $workflow_user; + $this->data = $data; + $this->workflow = $workflow; + $this->current_node = $current_node; + } + + public function getUser(): array + { + return $this->workflow_user; + } + + public function getData(): array + { + return $this->data; + } + + public function getWorkflow(): array + { + return $this->workflow; + } + + public function getCurrentNode(): int + { + return $this->current_node; + } + + public function setData(array $data) + { + $this->data = $data; + } + + public function setCurrentNode(int $current_node) + { + $this->current_node = $current_node; + } +} + +class WorkflowGraphTool +{ + /** + * extractTriggerFromWorkflow Return the trigger id (or full module) that are specified in the workflow + * + * @param array $workflow + * @param bool $fullNode + * @return int|array|null + */ + public static function extractTriggerFromWorkflow(array $graphData, bool $fullNode = false) + { + $triggers = self::extractTriggersFromWorkflow($graphData, $fullNode); + if (empty($triggers)) { + return null; + } + $node = $triggers[0]; + return $node; + } + + /** + * extractTriggersFromWorkflow Return the list of triggers id (or full module) that are specified in the workflow + * + * @param array $workflow + * @param bool $fullNode + * @return array + */ + public static function extractTriggersFromWorkflow(array $graphData, bool $fullNode = false): array + { + $triggers = []; + foreach ($graphData as $node) { + if ($node['data']['module_type'] == 'trigger') { + if (!empty($fullNode)) { + $triggers[] = $node; + } else { + $triggers[] = $node['data']['id']; + } + } + } + return $triggers; + } + + /** + * extractConcurrentTasksFromWorkflow Return the list of concurrent-tasks's id (or full module) that are included in the workflow + * + * @param array $workflow + * @param bool $fullNode + * @return array + */ + public static function extractConcurrentTasksFromWorkflow(array $graphData, bool $fullNode = false): array + { + $nodes = []; + foreach ($graphData as $node) { + if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') { + if (!empty($fullNode)) { + $nodes[] = $node; + } else { + $nodes[] = $node['data']['id']; + } + } + } + return $nodes; + } + + /** + * isAcyclic Return if the graph contains a cycle + * + * @param array $graphData + * @param array $cycles Get a list of cycle + * @return boolean + */ + public static function isAcyclic(array $graphData, array &$cycles=[]): bool + { + $graphUtil = new GraphUtil($graphData); + $result = $graphUtil->isCyclic(); + $isCyclic = $result[0]; + $cycles = $result[1]; + return !$isCyclic; + } + + /** + * hasMultipleOutputConnection Return if the graph has multiple connection from a node output + * + * @param array $graphData + * @param array $edges Get a list of edges from the same output + * @return boolean + */ + public static function hasMultipleOutputConnection(array $graphData, array &$edges=[]): bool + { + $graphUtil = new GraphUtil($graphData); + $result = $graphUtil->hasMultipleOutputConnection(); + $hasMultipleOutputConnection = $result[0]; + $edges = $result[1]; + return $hasMultipleOutputConnection; + } + + /** + * Undocumented getNodeIdForTrigger + * + * @param array $graphData + * @param string $trigger_id + * @return integer Return the ID of the node for the provided trigger and -1 if no nodes with this id was found. + */ + public static function getNodeIdForTrigger(array $graphData, $trigger_id): int + { + $trigger_node = WorkflowGraphTool::extractTriggerFromWorkflow($graphData, true); + if ($trigger_node['data']['id'] == $trigger_id) { + return $trigger_node['id']; + } + return -1; + } + + public static function getRoamingData(array $user=[], array $data=[], array $workflow=[], int $node_id=-1) + { + return new WorkflowRoamingData($user, $data, $workflow, $node_id); + } + + public static function getWalkerIterator(array $graphData, $WorkflowModel, $startNodeID, $path_type=null, WorkflowRoamingData $roamingData) + { + if (!in_array($path_type, GraphWalker::ALLOWED_PATH_TYPES)) { + return []; + } + $graphWalker = new GraphWalker($graphData, $WorkflowModel, $startNodeID, $path_type); + return $graphWalker->walk($roamingData); + } +} diff --git a/app/Lib/WorkflowModules/action/Module_blueprint_action_module.php b/app/Lib/WorkflowModules/action/Module_blueprint_action_module.php new file mode 100644 index 000000000..c7e718b8a --- /dev/null +++ b/app/Lib/WorkflowModules/action/Module_blueprint_action_module.php @@ -0,0 +1,23 @@ +blocking == true, returning `false` will stop the execution. + $errors[] = __('Execution stopped'); + return false; + } +} diff --git a/app/Lib/WorkflowModules/logic/Module_blueprint_logic_module.php b/app/Lib/WorkflowModules/logic/Module_blueprint_logic_module.php new file mode 100644 index 000000000..464c610fc --- /dev/null +++ b/app/Lib/WorkflowModules/logic/Module_blueprint_logic_module.php @@ -0,0 +1,23 @@ +getParamsWithValues($node); + $data = $roamingData->getData(); + // Returning true will make the execution flow take the first output of this module. Otherwise, the second output will be used. + return true; + } +} diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 85e9616b1..265d70070 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -25,6 +25,7 @@ App::uses('LogableBehavior', 'Assets.models/behaviors'); App::uses('RandomTool', 'Tools'); App::uses('FileAccessTool', 'Tools'); App::uses('JsonTool', 'Tools'); +App::uses('BetterCakeEventManager', 'Tools'); class AppModel extends Model { @@ -47,15 +48,6 @@ class AppModel extends Model /** @var AttachmentTool|null */ private $attachmentTool; - public function __construct($id = false, $table = null, $ds = null) - { - parent::__construct($id, $table, $ds); - $this->findMethods['column'] = true; - if (in_array('phar', stream_get_wrappers())) { - stream_wrapper_unregister('phar'); - } - } - // deprecated, use $db_changes // major -> minor -> hotfix -> requires_logout const OLD_DB_CHANGES = array( @@ -89,10 +81,11 @@ class AppModel extends Model 63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false, 69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false, 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false, - 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false + 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, + 87 => false, 88 => false, 89 => false, 90 => false, ); - public $advanced_updates_description = array( + const ADVANCED_UPDATES_DESCRIPTION = array( 'seenOnAttributeAndObject' => array( 'title' => 'First seen/Last seen Attribute table', 'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.', @@ -106,6 +99,15 @@ class AppModel extends Model ), ); + public function __construct($id = false, $table = null, $ds = null) + { + parent::__construct($id, $table, $ds); + $this->findMethods['column'] = true; + if (in_array('phar', stream_get_wrappers(), true)) { + stream_wrapper_unregister('phar'); + } + } + public function isAcceptedDatabaseError($errorMessage) { if ($this->isMysql()) { @@ -135,8 +137,9 @@ class AppModel extends Model switch ($command) { case '2.4.20': $dbUpdateSuccess = $this->updateDatabase($command); - $this->ShadowAttribute = ClassRegistry::init('ShadowAttribute'); - $this->ShadowAttribute->upgradeToProposalCorrelation(); + //deprecated + //$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute'); + //$this->ShadowAttribute->upgradeToProposalCorrelation(); break; case '2.4.25': $dbUpdateSuccess = $this->updateDatabase($command); @@ -225,6 +228,15 @@ class AppModel extends Model case 48: $dbUpdateSuccess = $this->__generateCorrelations(); break; + case 89: + $this->__retireOldCorrelationEngine(); + $dbUpdateSuccess = true; + break; + case 90: + $dbUpdateSuccess = $this->updateDatabase($command); + $this->Workflow = Classregistry::init('Workflow'); + $this->Workflow->enableDefaultModules(); + break; default: $dbUpdateSuccess = $this->updateDatabase($command); break; @@ -274,9 +286,9 @@ class AppModel extends Model $liveOff = false; $exitOnError = false; - if (isset($this->advanced_updates_description[$command])) { - $liveOff = isset($this->advanced_updates_description[$command]['liveOff']) ? $this->advanced_updates_description[$command]['liveOff'] : $liveOff; - $exitOnError = isset($this->advanced_updates_description[$command]['exitOnError']) ? $this->advanced_updates_description[$command]['exitOnError'] : $exitOnError; + if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command])) { + $liveOff = isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['liveOff']) ? self::ADVANCED_UPDATES_DESCRIPTION[$command]['liveOff'] : $liveOff; + $exitOnError = isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['exitOnError']) ? self::ADVANCED_UPDATES_DESCRIPTION[$command]['exitOnError'] : $exitOnError; } $sqlArray = array(); @@ -1684,6 +1696,118 @@ class AppModel extends Model case 86: $this->__addIndex('attributes', 'timestamp'); break; + case 87: + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `no_acl_correlations` ( + `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `attribute_id` int(10) UNSIGNED NOT NULL, + `1_attribute_id` int(10) UNSIGNED NOT NULL, + `event_id` int(10) UNSIGNED NOT NULL, + `1_event_id` int(10) UNSIGNED NOT NULL, + `value_id` int(10) UNSIGNED NOT NULL, + PRIMARY KEY (`id`), + INDEX `event_id` (`event_id`), + INDEX `1_event_id` (`1_event_id`), + INDEX `attribute_id` (`attribute_id`), + INDEX `1_attribute_id` (`1_attribute_id`), + INDEX `value_id` (`value_id`) + ) ENGINE=InnoDB;"; + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `default_correlations` ( + `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `attribute_id` int(10) UNSIGNED NOT NULL, + `object_id` int(10) UNSIGNED NOT NULL, + `event_id` int(10) UNSIGNED NOT NULL, + `org_id` int(10) UNSIGNED NOT NULL, + `distribution` tinyint(4) NOT NULL, + `object_distribution` tinyint(4) NOT NULL, + `event_distribution` tinyint(4) NOT NULL, + `sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `object_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `event_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `1_attribute_id` int(10) UNSIGNED NOT NULL, + `1_object_id` int(10) UNSIGNED NOT NULL, + `1_event_id` int(10) UNSIGNED NOT NULL, + `1_org_id` int(10) UNSIGNED NOT NULL, + `1_distribution` tinyint(4) NOT NULL, + `1_object_distribution` tinyint(4) NOT NULL, + `1_event_distribution` tinyint(4) NOT NULL, + `1_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `1_object_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `1_event_sharing_group_id` int(10) UNSIGNED NOT NULL DEFAULT 0, + `value_id` int(10) UNSIGNED NOT NULL, + PRIMARY KEY (`id`), + INDEX `event_id` (`event_id`), + INDEX `attribute_id` (`attribute_id`), + INDEX `object_id` (`object_id`), + INDEX `org_id` (`org_id`), + INDEX `distribution` (`distribution`), + INDEX `object_distribution` (`object_distribution`), + INDEX `event_distribution` (`event_distribution`), + INDEX `sharing_group_id` (`sharing_group_id`), + INDEX `object_sharing_group_id` (`object_sharing_group_id`), + INDEX `event_sharing_group_id` (`event_sharing_group_id`), + INDEX `1_event_id` (`1_event_id`), + INDEX `1_attribute_id` (`1_attribute_id`), + INDEX `1_object_id` (`1_object_id`), + INDEX `1_org_id` (`1_org_id`), + INDEX `1_distribution` (`1_distribution`), + INDEX `1_object_distribution` (`1_object_distribution`), + INDEX `1_event_distribution` (`1_event_distribution`), + INDEX `1_sharing_group_id` (`1_sharing_group_id`), + INDEX `1_object_sharing_group_id` (`1_object_sharing_group_id`), + INDEX `1_event_sharing_group_id` (`1_event_sharing_group_id`), + INDEX `value_id` (`value_id`) + ) ENGINE=InnoDB;"; + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `correlation_values` ( + `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `value` varchar(191) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `value` (`value`(191)) + ) ENGINE=InnoDB CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `over_correlating_values` ( + `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `value` text, + `occurrence` int(10) UNSIGNED NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `value` (`value`(191)), + INDEX `occurrence` (`occurrence`) + ) ENGINE=InnoDB CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + break; + case 88: + $sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_required` tinyint(1) NOT NULL DEFAULT 0;'; + $sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_key` text COLLATE utf8_bin;'; + break; + case 90: + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `workflows` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) COLLATE utf8_bin NOT NULL , + `name` varchar(191) NOT NULL, + `description` varchar(191) NOT NULL, + `timestamp` int(11) NOT NULL DEFAULT 0, + `enabled` tinyint(1) NOT NULL DEFAULT 0, + `counter` int(11) NOT NULL DEFAULT 0, + `trigger_id` varchar(191) COLLATE utf8_bin NOT NULL, + `debug_enabled` tinyint(1) NOT NULL DEFAULT 0, + `data` text, + PRIMARY KEY (`id`), + INDEX `uuid` (`uuid`), + INDEX `name` (`name`), + INDEX `timestamp` (`timestamp`), + INDEX `trigger_id` (`trigger_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `workflow_blueprints` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) COLLATE utf8_bin NOT NULL , + `name` varchar(191) NOT NULL, + `description` varchar(191) NOT NULL, + `timestamp` int(11) NOT NULL DEFAULT 0, + `default` tinyint(1) NOT NULL DEFAULT 0, + `data` text, + PRIMARY KEY (`id`), + INDEX `uuid` (`uuid`), + INDEX `name` (`name`), + INDEX `timestamp` (`timestamp`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -1765,7 +1889,7 @@ class AppModel extends Model $total_update_count = $sql_update_count + $index_update_count; $this->__setUpdateProgress(0, $total_update_count, $command); $str_index_array = array(); - foreach($indexArray as $toIndex) { + foreach ($indexArray as $toIndex) { $str_index_array[] = __('Indexing %s -> %s', $toIndex[0], $toIndex[1]); } $this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array)); @@ -1773,8 +1897,8 @@ class AppModel extends Model $errorCount = 0; // execute test before update. Exit if it fails - if (isset($this->advanced_updates_description[$command]['preUpdate'])) { - $function_name = $this->advanced_updates_description[$command]['preUpdate']; + if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['preUpdate'])) { + $function_name = self::ADVANCED_UPDATES_DESCRIPTION[$command]['preUpdate']; try { $this->{$function_name}(); } catch (Exception $e) { @@ -1910,10 +2034,15 @@ class AppModel extends Model $this->Server->serverSettingsSaveValue('MISP.live', $isLive); } - // check whether the adminSetting should be updated after the update - private function __postUpdate($command) { - if (isset($this->advanced_updates_description[$command]['record'])) { - if($this->advanced_updates_description[$command]['record']) { + /** + * Check whether the adminSetting should be updated after the update. + * @param string $command + * @return void + */ + private function __postUpdate($command) + { + if (isset(self::ADVANCED_UPDATES_DESCRIPTION[$command]['record'])) { + if (self::ADVANCED_UPDATES_DESCRIPTION[$command]['record']) { $this->AdminSetting->changeSetting($command, 1); } } @@ -2104,7 +2233,7 @@ class AppModel extends Model 'fields' => ['id', 'value'], ]); if (count($db_version) > 1) { - // we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57 + // we ran into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57 foreach ($db_version as $k => $v) { if ($k > 0) { $this->AdminSetting->delete($v['AdminSetting']['id']); @@ -2822,7 +2951,7 @@ class AppModel extends Model * * @return false|string */ - protected function checkMIPSCommit() + public function checkMIPSCommit() { static $commit; if ($commit === null) { @@ -3383,4 +3512,133 @@ class AppModel extends Model $dataSourceName = $dataSource->config['datasource']; return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql; } + + public function getCorrelationModelName() + { + if (!empty(Configure::read('MISP.correlation_engine'))) { + return Configure::read('MISP.correlation_engine'); + } + return 'Default'; + } + + public function loadCorrelationModel() + { + if (!empty(Configure::read('MISP.correlation_engine'))) { + return ClassRegistry::init(Configure::read('MISP.correlation_engine')); + } + return ClassRegistry::init('Correlation'); + } + + /** + * executeTrigger + * + * @param string $trigger_id + * @param array $data Data to be passed to the workflow + * @param array $blockingErrors Errors will be appened if any + * @param array $logging If the execution failure should be logged + * @return boolean If the execution for the blocking path was a success + */ + public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool + { + if ($this->Workflow === null) { + $this->Workflow = ClassRegistry::init('Workflow'); + } + if ($this->isTriggerCallable($trigger_id)) { + $success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging); + if (!empty($logging) && empty($success)) { + $logging['message'] = !empty($logging['message']) ? $logging['message'] : __('Error while executing workflow.'); + $errorMessage = implode(', ', $blockingErrors); + $this->loadLog()->createLogEntry('SYSTEM', $logging['action'], $logging['model'], $logging['id'], $logging['message'], __('Returned message: %s', $errorMessage)); + } + return $success; + } + return true; + } + + public function isTriggerCallable($trigger_id): bool + { + if ($this->Workflow === null) { + $this->Workflow = ClassRegistry::init('Workflow'); + } + return $this->Workflow->checkTriggerEnabled($trigger_id) && + $this->Workflow->checkTriggerListenedTo($trigger_id); + } + + public function addPendingLogEntry($logEntry) + { + $logEntries = Configure::read('pendingLogEntries'); + $logEntries[] = $logEntry; + Configure::write('pendingLogEntries', $logEntries); + } + + /** + * Use different CakeEventManager to fix memory leak + * @return CakeEventManager + */ + public function getEventManager() + { + if (empty($this->_eventManager)) { + $this->_eventManager = new BetterCakeEventManager(); + $this->_eventManager->attach($this->Behaviors); + $this->_eventManager->attach($this); + } + return $this->_eventManager; + } + + private function __retireOldCorrelationEngine($user = null) { + if ($user === null) { + $user = [ + 'id' => 0, + 'email' => 'SYSTEM', + 'Organisation' => [ + 'name' => 'SYSTEM' + ] + ]; + } + $this->Correlation = ClassRegistry::init('Correlation'); + $this->Attribute = ClassRegistry::init('Attribute'); + if (!Configure::read('MISP.background_jobs')) { + $this->Correlation->truncate($user, 'Legacy'); + $this->Attribute->generateCorrelation(); + } else { + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'truncate table', + $this->Correlation->validEngines['Legacy'], + 'Job created.' + ); + $this->Correlation->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'truncateTable', + 0, + 'Legacy', + $jobId + ], + true, + $jobId + ); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate correlation', + 'All attributes', + 'Job created.' + ); + + $this->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateCorrelation', + $jobId + ], + true, + $jobId + ); + } + } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index e56c9928b..7b03ae603 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -416,6 +416,7 @@ class Attribute extends AppModel // update correlation... if (isset($attribute['deleted']) && $attribute['deleted']) { $this->Correlation->beforeSaveCorrelation($attribute); + $this->Correlation->advancedCorrelationsUpdate($attribute); if (isset($attribute['event_id'])) { $this->__alterAttributeCount($attribute['event_id'], false); } @@ -438,9 +439,11 @@ class Attribute extends AppModel ) { $this->Correlation->beforeSaveCorrelation($attribute); $this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent); + $this->Correlation->advancedCorrelationsUpdate($attribute); } } else { $this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent); + $this->Correlation->advancedCorrelationsUpdate($attribute); } } $result = true; @@ -457,7 +460,8 @@ class Attribute extends AppModel } $pubToZmq = $this->pubToZmq('attribute'); $kafkaTopic = $this->kafkaTopic('attribute'); - if ($pubToZmq || $kafkaTopic) { + $isTriggerCallable = $this->isTriggerCallable('attribute-after-save'); + if ($pubToZmq || $kafkaTopic || $isTriggerCallable) { $attributeForPublish = $this->fetchAttribute($this->id); if (!empty($attributeForPublish)) { $user = array( @@ -486,11 +490,16 @@ class Attribute extends AppModel $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $attributeForPublish, $action); } + $workflowErrors = []; + $logging = [ + 'model' => 'Attribute', + 'action' => $action, + 'id' => $attributeForPublish['Attribute']['id'], + ]; + $triggerData = $attributeForPublish; + $this->executeTrigger('attribute-after-save', $triggerData, $workflowErrors, $logging); } } - if (Configure::read('MISP.enable_advanced_correlations') && in_array($attribute['type'], ['ip-src', 'ip-dst'], true) && strpos($attribute['value'], '/')) { - $this->Correlation->updateCidrList(); - } if ($created && isset($attribute['event_id']) && empty($attribute['skip_auto_increment'])) { $this->__alterAttributeCount($attribute['event_id']); } @@ -784,90 +793,6 @@ class Attribute extends AppModel return $output; } - public function getRelatedAttributes($user, $attribute, $fields=array(), $includeEventData = false) - { - // LATER getRelatedAttributes($attribute) this might become a performance bottleneck - - // exclude these specific categories from being linked - switch ($attribute['category']) { - case 'Antivirus detection': - return null; - } - // exclude these specific types from being linked - switch ($attribute['type']) { - case 'other': - case 'comment': - return null; - } - - // prepare the conditions - $conditions = array( - 'Attribute.event_id !=' => $attribute['event_id'], - 'Attribute.deleted !=' => 1, - ); - - // prevent issues with empty fields - if (empty($attribute['value1'])) { - return null; - } - - if (empty($attribute['value2'])) { - // no value2, only search for value 1 - $conditions['OR'] = array( - 'Attribute.value1' => $attribute['value1'], - 'Attribute.value2' => $attribute['value1'], - ); - } else { - // value2 also set, so search for both - $conditions['AND'] = array( // TODO was OR - 'Attribute.value1' => array($attribute['value1'],$attribute['value2']), - 'Attribute.value2' => array($attribute['value1'],$attribute['value2']), - ); - } - $baseConditions = $this->buildConditions($user); - $baseConditions['AND'][] = $conditions; - // do the search - if (empty($fields)) { - $fields = array('Attribute.*'); - } - $params = array( - 'conditions' => $baseConditions, - 'fields' => $fields, - 'recursive' => 0, - 'group' => array('Attribute.id', 'Attribute.event_id', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.uuid', 'Attribute.timestamp', 'Attribute.distribution', 'Attribute.sharing_group_id', 'Attribute.to_ids', 'Attribute.comment', 'Event.id', 'Event.uuid', 'Event.threat_level_id', 'Event.analysis', 'Event.info', 'Event.extends_uuid', 'Event.distribution', 'Event.sharing_group_id', 'Event.published', 'Event.date', 'Event.orgc_id', 'Event.org_id', 'Object.id', 'Object.uuid', 'Object.distribution', 'Object.name', 'Object.template_uuid', 'Object.distribution', 'Object.sharing_group_id'), - 'order' => 'Attribute.event_id DESC' - ); - if (!empty($includeEventData)) { - $params['contain'] = array( - 'Event' => array( - 'fields' => array( - 'Event.id', 'Event.uuid', 'Event.threat_level_id', 'Event.analysis', 'Event.info', 'Event.extends_uuid', 'Event.distribution', 'Event.sharing_group_id', 'Event.published', 'Event.date', 'Event.orgc_id', 'Event.org_id' - ) - ), - 'Object' => array( - 'fields' => array( - 'Object.id', 'Object.uuid', 'Object.distribution', 'Object.name', 'Object.template_uuid', 'Object.distribution', 'Object.sharing_group_id' - ) - ) - ); - } - $similarEvents = $this->find( - 'all', - $params - ); - if (!empty($includeEventData)) { - foreach ($similarEvents as $k => $similarEvent) { - $similarEvents[$k] = array_merge( - $similarEvent['Attribute'], - array( - 'Event' => $similarEvent['Event'] - ) - ); - } - } - return $similarEvents; - } - public function typeIsMalware($type) { return in_array($type, self::ZIPPED_DEFINITION, true); @@ -1611,65 +1536,24 @@ class Attribute extends AppModel $attributeCount = 0; if (Configure::read('MISP.background_jobs') && $jobId) { $this->Job = ClassRegistry::init('Job'); - $eventCount = count($eventIds); } else { $jobId = false; } - foreach ($eventIds as $j => $eventId) { - if ($jobId) { - $message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024)); - $this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100); + if (!empty($eventIds)) { + $eventCount = count($eventIds); + foreach ($eventIds as $j => $currentEventId) { + $attributeCount = $this->__iteratedCorrelation( + $jobId, + $full, + $attributeCount, + $attributeId, + $eventCount, + $currentEventId, + $j + ); } - $event = $this->Event->find('first', array( - 'recursive' => -1, - 'fields' => ['Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'], - 'conditions' => array('id' => $eventId), - 'order' => false, - )); - $attributeConditions = [ - 'Attribute.event_id' => $eventId, - 'Attribute.deleted' => 0, - 'Attribute.disable_correlation' => 0, - 'NOT' => [ - 'Attribute.type' => Attribute::NON_CORRELATING_TYPES, - ], - ]; - if ($attributeId) { - $attributeConditions['Attribute.id'] = $attributeId; - } - $query = [ - 'recursive' => -1, - 'conditions' => $attributeConditions, - // fetch just necessary fields to save memory - 'fields' => [ - 'Attribute.id', - 'Attribute.type', - 'Attribute.value1', - 'Attribute.value2', - 'Attribute.distribution', - 'Attribute.sharing_group_id', - ], - 'order' => 'Attribute.id', - 'limit' => 5000, - 'callbacks' => false, // memory leak fix - ]; - do { - $attributes = $this->find('all', $query); - foreach ($attributes as $attribute) { - $attribute['Attribute']['event_id'] = $eventId; - $this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full, $event); - } - $fetchedAttributes = count($attributes); - unset($attributes); - $attributeCount += $fetchedAttributes; - if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop - $query['conditions']['Attribute.id >'] = $attribute['Attribute']['id']; - } else { - break; - } - } while (true); - // Generating correlations can take long time, so clear CIDR cache after each event to refresh cache - $this->Correlation->clearCidrCache(); + } else { + $attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount); } if ($jobId) { $this->Job->saveStatus($jobId, true); @@ -1677,18 +1561,68 @@ class Attribute extends AppModel return $attributeCount; } + private function __iteratedCorrelation( + $jobId = false, + $full = false, + $attributeCount = 0, + $attributeId = null, + $eventCount = null, + $eventId = null, + $j = 0 + ) + { + if ($jobId) { + $message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024)); + $this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100); + } + $attributeConditions = [ + 'Attribute.deleted' => 0, + 'Attribute.disable_correlation' => 0, + 'NOT' => [ + 'Attribute.type' => Attribute::NON_CORRELATING_TYPES, + ], + ]; + if ($eventId) { + $attributeConditions['Attribute.event_id'] = $eventId; + } + if ($attributeId) { + $attributeConditions['Attribute.id'] = $attributeId; + } + $query = [ + 'recursive' => -1, + 'conditions' => $attributeConditions, + // fetch just necessary fields to save memory + 'fields' => $this->Correlation->getFieldRules(), + 'order' => 'Attribute.id', + 'limit' => 5000, + 'callbacks' => false, // memory leak fix + ]; + do { + $attributes = $this->find('all', $query); + foreach ($attributes as $attribute) { + $attribute['Attribute']['event_id'] = $eventId; + if ($full) { + $this->Correlation->beforeSaveCorrelation($attribute['Attribute']); + } + $this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full); + } + $fetchedAttributes = count($attributes); + unset($attributes); + $attributeCount += $fetchedAttributes; + if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop + $query['conditions']['Attribute.id >'] = $attribute['Attribute']['id']; + } else { + break; + } + } while (true); + // Generating correlations can take long time, so clear CIDR cache after each event to refresh cache + $this->Correlation->clearCidrCache(); + return $attributeCount; + } + public function purgeCorrelations($eventId = false) { - if (!$eventId) { - $this->query('TRUNCATE TABLE correlations;'); - } else { - $this->Correlation->deleteAll([ - 'OR' => array( - 'Correlation.1_event_id' => $eventId, - 'Correlation.event_id' => $eventId, - ) - ], false); - } + $this->Correlation->purgeCorrelations($eventId); } public function reportValidationIssuesAttributes($eventId) @@ -1725,8 +1659,13 @@ class Attribute extends AppModel return $result; } - // This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays - // array 1 will have all of the non negated terms and array 2 all the negated terms + /** + * This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays + * array 1 will have all of the non negated terms and array 2 all the negated terms + * + * @param string|array $args + * @return array[] + */ public function dissectArgs($args) { $result = array(0 => array(), 1 => array(), 2 => array()); @@ -1748,7 +1687,7 @@ class Attribute extends AppModel } } else { foreach ($args as $arg) { - if ($arg[0] === '!') { + if (is_string($arg) && $arg[0] === '!') { $result[1][] = substr($arg, 1); } else { $result[0][] = $arg; @@ -2039,7 +1978,6 @@ class Attribute extends AppModel $params = array( 'conditions' => $this->buildConditions($user), 'fields' => array(), - 'recursive' => -1, 'contain' => ['Event', 'Object'], // by default include Event and Object, because it is required for conditions ); if (isset($options['conditions'])) { @@ -2278,7 +2216,7 @@ class Attribute extends AppModel $this->attachTagsToAttributes($results, $options); $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); - + $sgids = $this->SharingGroup->authorizedIds($user); foreach ($results as &$attribute) { if (!empty($options['includeContext'])) { $attribute['Event'] = $eventsById[$attribute['Attribute']['event_id']]; @@ -2290,7 +2228,7 @@ class Attribute extends AppModel } if (!empty($options['includeCorrelations'])) { $attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment'); - $attribute['Attribute']['RelatedAttribute'] = $this->getRelatedAttributes($user, $attribute['Attribute'], $attributeFields, true); + $attribute['Attribute']['RelatedAttribute'] = $this->Correlation->getRelatedAttributes($user, $sgids, $attribute['Attribute'], $attributeFields, true); } if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) { continue; @@ -2467,27 +2405,77 @@ class Attribute extends AppModel } $temp = $this->Event->EventTag->find('all', array( 'recursive' => -1, - 'contain' => array('Tag'), + 'contain' => ['Tag' => ['fields' => ['id', 'name', 'colour', 'numerical_value']]], 'conditions' => $tagConditions, )); if (empty($temp)) { $eventTags[$eventId] = []; } else { foreach ($temp as $tag) { + $tag['Tag']['inherited'] = true; $tag['EventTag']['Tag'] = $tag['Tag']; - unset($tag['Tag']); $eventTags[$eventId][] = $tag['EventTag']; } } } - if (!empty($eventTags)) { - foreach ($eventTags[$eventId] as $eventTag) { - $attribute['EventTag'][] = $eventTag; - } - } + $attribute['EventTag'] = $eventTags[$eventId]; return $attribute; } + public function touch($attribute_id) + { + $attribute = $this->find('first', [ + 'conditions' => ['Attribute.id' => $attribute_id], + 'recursive' => -1, + ]); + $event = $this->Event->find('first', [ + 'conditions' => ['Event.id' => $attribute['Attribute']['event_id']], + 'recursive' => -1, + ]); + $timestamp = (new DateTime())->getTimestamp(); + $event['Event']['published'] = 0; + $event['Event']['timestamp'] = $timestamp; + $attribute['Attribute']['timestamp'] = $timestamp; + $saveSucces = true; + if ($attribute['Attribute']['object_id'] != 0) { + $saveSucces = $this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $timestamp); + } + $saveSucces = $saveSucces && $this->save($attribute['Attribute'], true, ['timestamp']); + return $saveSucces && $this->Event->save($event, true, ['timestamp', 'published']); + } + + public function attachTagsFromAttributeAndTouch($attribute_id, $event_id, $tags) + { + $touchAttribute = false; + $success = false; + foreach ($tags as $tag_id) { + $nothingToChange = false; + $saveSuccess = $this->AttributeTag->attachTagToAttribute($attribute_id, $event_id, $tag_id, false, $nothingToChange); + $success = $success || !empty($saveSuccess); + $touchAttribute = $touchAttribute || !$nothingToChange; + } + if ($touchAttribute) { + return $this->touch($attribute_id); + } + return $success; + } + + public function detachTagsFromAttributeAndTouch($attribute_id, $event_id, $tags) + { + $touchAttribute = false; + $success = false; + foreach ($tags as $tag_id) { + $nothingToChange = false; + $saveSuccess = $this->AttributeTag->detachTagFromAttribute($attribute_id, $event_id, $tag_id, $nothingToChange); + $success = $success || !empty($saveSuccess); + $touchAttribute = $touchAttribute || !$nothingToChange; + } + if ($touchAttribute) { + return $this->touch($attribute_id); + } + return $success; + } + private function __blockAttributeViaProposal($attribute) { if (!empty($attribute['ShadowAttribute'])) { @@ -2681,8 +2669,12 @@ class Attribute extends AppModel public function setTimestampConditions($timestamp, $conditions, $scope = 'Event.timestamp', $returnRaw = false) { if (is_array($timestamp)) { - $timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0])); - $timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1])); + if (count($timestamp) !== 2) { + throw new InvalidArgumentException('Invalid date specification, must be string or array with two elements'); + } + + $timestamp[0] = intval($this->resolveTimeDelta($timestamp[0])); + $timestamp[1] = intval($this->resolveTimeDelta($timestamp[1])); if ($timestamp[0] > $timestamp[1]) { $temp = $timestamp[0]; $timestamp[0] = $timestamp[1]; @@ -2691,7 +2683,7 @@ class Attribute extends AppModel $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]); $conditions['AND'][] = array($scope . ' <=' => $timestamp[1]); } else { - $timestamp = intval($this->Event->resolveTimeDelta($timestamp)); + $timestamp = intval($this->resolveTimeDelta($timestamp)); $conditions['AND'][] = array($scope . ' >=' => $timestamp); } if ($returnRaw) { @@ -2703,8 +2695,8 @@ class Attribute extends AppModel public function setTimestampSeenConditions($timestamp, $conditions, $scope = 'Attribute.first_seen', $returnRaw = false) { if (is_array($timestamp)) { - $timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB - $timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB + $timestamp[0] = intval($this->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB + $timestamp[1] = intval($this->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB if ($timestamp[0] > $timestamp[1]) { $temp = $timestamp[0]; $timestamp[0] = $timestamp[1]; @@ -2713,7 +2705,7 @@ class Attribute extends AppModel $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]); $conditions['AND'][] = array($scope . ' <=' => $timestamp[1]); } else { - $timestamp = intval($this->Event->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB + $timestamp = intval($this->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB if ($scope == 'Attribute.first_seen') { $conditions['AND'][] = array($scope . ' >=' => $timestamp); } else { @@ -3237,7 +3229,6 @@ class Attribute extends AppModel $filters['wildcard'] = $filters['searchall']; } } - $subqueryElements = $this->Event->harvestSubqueryElements($filters); $filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements, $user); $filters = $this->Event->addFiltersFromUserSettings($user, $filters); diff --git a/app/Model/AttributeTag.php b/app/Model/AttributeTag.php index 58793fc2f..ecc79cd6d 100644 --- a/app/Model/AttributeTag.php +++ b/app/Model/AttributeTag.php @@ -135,7 +135,7 @@ class AttributeTag extends AppModel * @return bool * @throws Exception */ - public function attachTagToAttribute($attribute_id, $event_id, $tag_id, $local = false) + public function attachTagToAttribute($attribute_id, $event_id, $tag_id, $local = false, &$nothingToChange = false) { $existingAssociation = $this->hasAny([ 'tag_id' => $tag_id, @@ -152,11 +152,13 @@ class AttributeTag extends AppModel if (!$this->save($data)) { return false; } + } else { + $nothingToChange = true; } return true; } - public function detachTagFromAttribute($attribute_id, $event_id, $tag_id) + public function detachTagFromAttribute($attribute_id, $event_id, $tag_id, &$nothingToChange = false) { $existingAssociation = $this->find('first', array( 'recursive' => -1, @@ -173,6 +175,8 @@ class AttributeTag extends AppModel if ($result) { return true; } + } else { + $nothingToChange = true; } return false; } diff --git a/app/Model/AuditLog.php b/app/Model/AuditLog.php index 0e702aba8..ad8f64dbd 100644 --- a/app/Model/AuditLog.php +++ b/app/Model/AuditLog.php @@ -33,6 +33,7 @@ class AuditLog extends AppModel public $actsAs = [ 'Containable', + 'LightPaginator' ]; /** @var array|null */ diff --git a/app/Model/Behavior/DefaultCorrelationBehavior.php b/app/Model/Behavior/DefaultCorrelationBehavior.php new file mode 100644 index 000000000..1d3699008 --- /dev/null +++ b/app/Model/Behavior/DefaultCorrelationBehavior.php @@ -0,0 +1,594 @@ + [ + 'fields' => [ + 'Attribute.event_id', + 'Attribute.object_id', + 'Attribute.id', + 'Attribute.type', + 'Attribute.distribution', + 'Attribute.sharing_group_id', + 'Attribute.value1', + 'Attribute.value2', + ], + 'contain' => [ + 'Event' => [ + 'fields' => [ + 'Event.id', + 'Event.org_id', + 'Event.distribution', + 'Event.sharing_group_id', + 'Event.disable_correlation', + ] + ], + 'Object' => [ + 'fields' => [ + 'Object.id', + 'Object.distribution', + 'Object.sharing_group_id', + ] + ] + ], + ] + ]; + + public $Correlation = null; + + private $deadlockAvoidance = false; + + public function setup(Model $Model, $settings = []) { + $Model->useTable = $this->__tableName; + $this->Correlation = $Model; + $this->deadlockAvoidance = $settings['deadlockAvoidance']; + } + + public function getTableName(Model $Model) + { + return $this->__tableName; + } + + public function createCorrelationEntry(Model $Model, $value, $a, $b) { + $value_id = $this->Correlation->CorrelationValue->getValueId($value); + if ($this->deadlockAvoidance) { + return [ + 'value_id' => $value_id, + '1_event_id' => $a['Event']['id'], + '1_object_id' => $a['Attribute']['object_id'], + '1_attribute_id' => $a['Attribute']['id'], + '1_org_id' => $a['Event']['org_id'], + '1_distribution' => $a['Attribute']['distribution'], + '1_event_distribution' => $a['Event']['distribution'], + '1_object_distribution' => empty($a['Attribute']['object_id']) ? 0 : $a['Object']['distribution'], + '1_sharing_group_id' => $a['Attribute']['sharing_group_id'], + '1_event_sharing_group_id' => $a['Event']['sharing_group_id'], + '1_object_sharing_group_id' => empty($a['Attribute']['object_id']) ? 0 : $a['Object']['sharing_group_id'], + 'event_id' => $b['Event']['id'], + 'object_id' => $b['Attribute']['object_id'], + 'attribute_id' => $b['Attribute']['id'], + 'org_id' => $b['Event']['org_id'], + 'distribution' => $b['Attribute']['distribution'], + 'event_distribution' => $b['Event']['distribution'], + 'object_distribution' => empty($b['Attribute']['object_id']) ? 0 : $b['Object']['distribution'], + 'sharing_group_id' => $b['Attribute']['sharing_group_id'], + 'event_sharing_group_id' => $b['Event']['sharing_group_id'], + 'object_sharing_group_id' => empty($b['Attribute']['object_id']) ? 0 : $b['Object']['sharing_group_id'], + ]; + } else { + return [ + (int) $value_id, + (int) $a['Event']['id'], + (int) $a['Attribute']['object_id'], + (int) $a['Attribute']['id'], + (int) $a['Event']['org_id'], + (int) $a['Attribute']['distribution'], + (int) $a['Event']['distribution'], + (int) empty($a['Attribute']['object_id']) ? 0 : $a['Object']['distribution'], + (int) $a['Attribute']['sharing_group_id'], + (int) $a['Event']['sharing_group_id'], + (int) empty($a['Attribute']['object_id']) ? 0 : $a['Object']['sharing_group_id'], + (int) $b['Event']['id'], + (int) $b['Attribute']['object_id'], + (int) $b['Attribute']['id'], + (int) $b['Event']['org_id'], + (int) $b['Attribute']['distribution'], + (int) $b['Event']['distribution'], + (int) empty($b['Attribute']['object_id']) ? 0 : $b['Object']['distribution'], + (int) $b['Attribute']['sharing_group_id'], + (int) $b['Event']['sharing_group_id'], + (int) empty($b['Attribute']['object_id']) ? 0 : $b['Object']['sharing_group_id'] + ]; + } + } + + public function saveCorrelations(Model $Model, $correlations) + { + $fields = [ + 'value_id', + '1_event_id', + '1_object_id', + '1_attribute_id', + '1_org_id', + '1_distribution', + '1_event_distribution', + '1_object_distribution', + '1_sharing_group_id', + '1_event_sharing_group_id', + '1_object_sharing_group_id', + 'event_id', + 'object_id', + 'attribute_id', + 'org_id', + 'distribution', + 'event_distribution', + 'object_distribution', + 'sharing_group_id', + 'event_sharing_group_id', + 'object_sharing_group_id' + ]; + + if ($this->deadlockAvoidance) { + return $this->Correlation->saveMany($correlations, array( + 'atomic' => false, + 'callbacks' => false, + 'deep' => false, + 'validate' => false, + 'fieldList' => $fields + )); + } else { + $db = $this->Correlation->getDataSource(); + // Split to chunks datasource is is enabled + if (count($correlations) > 100) { + foreach (array_chunk($correlations, 100) as $chunk) { + $db->insertMulti('default_correlations', $fields, $chunk); + } + return true; + } else { + return $db->insertMulti('default_correlations', $fields, $correlations); + } + } + } + + public function runBeforeSaveCorrelation(Model $Model, $attribute) { + // (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id + // ==> DELETE FROM default_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */ + // first check if it's an update + if (isset($attribute['id'])) { + $Model->deleteAll([ + 'OR' => [ + '1_attribute_id' => $attribute['id'], + 'attribute_id' => $attribute['id'] + ], + ], false); + } + if ($attribute['type'] === 'ssdeep') { + $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); + $this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']); + } + } + + public function getContainRules(Model $Model, $filter = null) + { + if (empty($filter)) { + return $this->__config['AttributeFetcher']['contain']; + } else { + return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter]; + } + } + + public function getFieldRules(Model $Model) + { + return $this->__config['AttributeFetcher']['fields']; + } + + private function __collectCorrelations($user, $id, $sgids, $primary) + { + $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; + $source = $primary ? '' : '1_'; + $prefix = $primary ? '1_' : ''; + $correlations = $this->Correlation->find('all', array( + 'fields' => [ + $source . 'attribute_id', + $prefix . 'attribute_id', + $prefix . 'org_id', + $prefix . 'event_id', + $prefix . 'event_distribution', + $prefix . 'event_sharing_group_id', + $prefix . 'object_id', + $prefix . 'object_distribution', + $prefix . 'object_sharing_group_id', + $prefix . 'distribution', + $prefix . 'sharing_group_id' + ], + 'conditions' => [ + 'OR' => [ + $source . 'event_id' => $id + ], + 'AND' => [ + [ + 'CorrelationValue.value NOT IN (select value from correlation_exclusions)' + ], + [ + 'CorrelationValue.value NOT IN (select value from over_correlating_values)' + ] + ] + ], + 'recursive' => -1, + 'contain' => [ + 'CorrelationValue' => [ + 'fields' => [ + 'CorrelationValue.id', + 'CorrelationValue.value' + ] + ] + ], + 'order' => false, + 'limit' => $max_correlations + )); + foreach ($correlations as $k => &$correlation) { + if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) { + unset($correlations[$k]); + } + } + $correlations = array_values($correlations); + return $correlations; + } + + public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids) + { + $temp_correlations = $this->__collectCorrelations($user, $id, $sgids, false); + $temp_correlations_1 = $this->__collectCorrelations($user, $id, $sgids, true); + $correlations = []; + $event_ids = []; + foreach ($temp_correlations as $temp_correlation) { + $correlations[] = [ + 'id' => $temp_correlation['Correlation']['event_id'], + 'attribute_id' => $temp_correlation['Correlation']['attribute_id'], + 'parent_id' => $temp_correlation['Correlation']['1_attribute_id'], + 'value' => $temp_correlation['CorrelationValue']['value'] + ]; + $event_ids[$temp_correlation['Correlation']['event_id']] = true; + } + foreach ($temp_correlations_1 as $temp_correlation) { + $correlations[] = [ + 'id' => $temp_correlation['Correlation']['1_event_id'], + 'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'], + 'parent_id' => $temp_correlation['Correlation']['attribute_id'], + 'value' => $temp_correlation['CorrelationValue']['value'] + ]; + $event_ids[$temp_correlation['Correlation']['1_event_id']] = true; + } + if (empty($correlations)) { + return []; + } + $conditions = $Model->Event->createEventConditions($user); + $conditions['Event.id'] = array_keys($event_ids); + $events = $Model->Event->find('all', [ + 'recursive' => -1, + 'conditions' => $conditions, + 'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'], + ]); + + $events = array_column(array_column($events, 'Event'), null, 'id'); + $relatedAttributes = []; + foreach ($correlations as $correlation) { + $eventId = $correlation['id']; + if (!isset($events[$eventId])) { + continue; + } + $event = $events[$eventId]; + $correlation['org_id'] = $events[$eventId]['orgc_id']; + $correlation['info'] = $events[$eventId]['info']; + $correlation['date'] = $events[$eventId]['date']; + $parentId = $correlation['parent_id']; + unset($correlation['parent_id']); + $relatedAttributes[$parentId][] = $correlation; + } + return $relatedAttributes; + } + + public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false) + { + // LATER getRelatedAttributes($attribute) this might become a performance bottleneck + // prepare the conditions + $conditions = [ + [ + 'Correlation.1_event_id !=' => $attribute['event_id'], + 'Correlation.attribute_id' => $attribute['id'] + ], + [ + 'Correlation.event_id !=' => $attribute['event_id'], + 'Correlation.1_attribute_id' => $attribute['id'] + ] + ]; + $corr_fields = [ + [ + '1_attribute_id', + '1_object_id', + '1_event_id', + '1_distribution', + '1_object_distribution', + '1_event_distribution', + '1_sharing_group_id', + '1_object_sharing_group_id', + '1_event_sharing_group_id', + '1_org_id', + 'value_id' + ], + [ + 'attribute_id', + 'object_id', + 'event_id', + 'distribution', + 'object_distribution', + 'event_distribution', + 'sharing_group_id', + 'object_sharing_group_id', + 'event_sharing_group_id', + 'org_id', + 'value_id' + ] + ]; + $prefixes = ['1_', '']; + $correlated_attribute_ids = []; + foreach ($conditions as $k => $condition) { + $temp_correlations = $Model->find('all', [ + 'recursive' => -1, + 'conditions' => $condition, + 'fields' => $corr_fields[$k] + ]); + if (!empty($temp_correlations)) { + foreach ($temp_correlations as $temp_correlation) { + if (empty($user['Role']['perm_site_admin'])) { + if (!$this->checkCorrelationACL($user, $temp_correlation, $sgids, $prefixes[$k])) { + continue; + } + } + $correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id']; + } + } + } + $contain = []; + if (!empty($includeEventData)) { + $contain['Event'] = [ + 'fields' => [ + 'Event.id', + 'Event.uuid', + 'Event.threat_level_id', + 'Event.analysis', + 'Event.info', + 'Event.extends_uuid', + 'Event.distribution', + 'Event.sharing_group_id', + 'Event.published', + 'Event.date', + 'Event.orgc_id', + 'Event.org_id' + ] + ]; + } + $relatedAttributes = $Model->Attribute->find('all', [ + 'recursive' => -1, + 'conditions' => [ + 'Attribute.id' => $correlated_attribute_ids + ], + 'fields' => $fields, + 'contain' => $contain + ]); + if (!empty($includeEventData)) { + $results = []; + foreach ($relatedAttributes as $k => $attribute) { + $temp = $attribute['Attribute']; + $temp['Event'] = $attribute['Event']; + $results[] = $temp; + } + return $results; + } else { + return $relatedAttributes; + } + } + + public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids) + { + // search the correlation table for the event ids of the related events + // Rules: + // 1. Event is owned by the user (org_id matches) + // 2. User is allowed to see both the event and the org: + // a. Event: + // i. Event has a distribution between 1-3 (community only, connected communities, all orgs) + // ii. Event has a sharing group that the user is accessible to view + // b. Attribute: + // i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway) + // ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) + // iii. Attribute has a sharing group that the user is accessible to view + $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true); + $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false); + return array_unique(array_merge($primaryEventIds,$secondaryEventIds)); + + } + + private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary) + { + $current = $primary ? '' : '1_'; + $prefix = $primary ? '1_' : ''; + $correlations = $Model->find('all', [ + 'recursive' => -1, + 'fields' => [ + $prefix . 'org_id', + $prefix . 'event_id', + $prefix . 'event_distribution', + $prefix . 'event_sharing_group_id', + $prefix . 'object_id', + $prefix . 'object_distribution', + $prefix . 'object_sharing_group_id', + $prefix . 'distribution', + $prefix . 'sharing_group_id' + ], + 'conditions' => [ + $current . 'event_id' => $eventId + ], + 'unique' => true, + ]); + $eventIds = []; + if (empty($user['Role']['perm_site_admin'])) { + foreach ($correlations as $k => $correlation) { + // if we have already added this event as a valid target, no need to check again. + if (isset($eventIds[$correlation['Correlation'][$prefix . 'event_id']])) { + continue; + } + $correlation = $correlation['Correlation']; + if (!$this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) { + unset($correlations[$k]); + continue; + } + $eventIds[$correlation[$prefix . 'event_id']] = true; + } + return array_keys($eventIds); + } else { + $eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id'); + return $eventIds; + } + } + + private function checkCorrelationACL($user, $correlation, $sgids, $prefix) + { + if ($user['Role']['perm_site_admin']) { + return true; + } + // check if user can see the event + if (isset($correlation['Correlation'])) { + $correlation = $correlation['Correlation']; + } + if ( + $correlation[$prefix . 'org_id'] != $user['org_id'] && + ( + $correlation[$prefix . 'event_distribution'] == 0 || + ( + $correlation[$prefix . 'event_distribution'] == 4 && + !in_array($correlation[$prefix . 'event_sharing_group_id'], $sgids) + ) + ) + ) { + return false; + } + + //check if the user can see the object, if we're looking at an object attribute + if ( + $correlation[$prefix . 'object_id'] && + ( + $correlation[$prefix . 'object_distribution'] == 0 || + $correlation[$prefix . 'object_distribution'] == 5 || + ( + $correlation[$prefix . 'object_distribution'] == 4 && + !in_array($correlation[$prefix . 'object_sharing_group_id'], $sgids) + ) + ) + ) { + return false; + } + + //check if the user can see the attribute + if ( + ( + $correlation[$prefix . 'distribution'] == 0 || + ( + $correlation[$prefix . 'distribution'] == 4 && + !in_array($correlation[$prefix . 'sharing_group_id'], $sgids) + ) + ) + ) { + return false; + } + return true; + } + + public function updateContainedCorrelations( + Model $Model, + array $data, + string $type = 'event', + array $options = [] + ): bool + { + $updateCorrelation = []; + $updateFields = [ + 'Correlation.' . $type . '_id', + 'Correlation.1_' . $type . '_id' + ]; + if ( + isset($data['distribution']) && + ( + empty($options['fieldList']) || + in_array('distribution', $options['fieldList']) + ) + ) { + $updateCorrelation[0]['Correlation.' . $type . '_distribution'] = (int)$data['distribution']; + $updateCorrelation[1]['Correlation.1_' . $type . '_distribution'] = (int)$data['distribution']; + } + if ( + isset($data['sharing_group_id']) && + ( + empty($options['fieldList']) || + in_array('sharing_group_id', $options['fieldList']) + ) + ) { + $updateCorrelation[0]['Correlation.' . $type . '_sharing_group_id'] = (int)$data['sharing_group_id']; + $updateCorrelation[1]['Correlation.1_' . $type . '_sharing_group_id'] = (int)$data['sharing_group_id']; + } + if (!empty($updateCorrelation)) { + foreach ($updateCorrelation as $k => $side) { + $Model->updateAll( + $side, + [ + $updateFields[$k] => (int)$data['id']] + ); + } + } + return true; + } + + public function purgeCorrelations(Model $Model, $eventId = null): void + { + if (!$eventId) { + $Model->query('TRUNCATE TABLE default_correlations;'); + //$Model->query('TRUNCATE TABLE correlation_values;'); + //$Model->query('TRUNCATE TABLE over_correlating_values;'); + } else { + $Model->deleteAll([ + 'OR' => array( + 'Correlation.1_event_id' => $eventId, + 'Correlation.event_id' => $eventId, + ) + ], false); + } + } + + public function purgeByValue(Model $Model, string $value): void + { + $valueIds = $Model->CorrelationValue->find('column', [ + 'recursive' => -1, + 'conditions' => [ + 'OR' => [ + 'CorrelationValue.value LIKE' => '%' . $value, + 'CorrelationValue.value LIKE' => $value . '%' + ] + ], + 'fields' => [ + 'CorrelationValue.id' + ] + ]); + $Model->deleteAll([ + 'Correlation.value_id' => $valueIds + ]); + } +} diff --git a/app/Model/Behavior/LightPaginatorBehavior.php b/app/Model/Behavior/LightPaginatorBehavior.php new file mode 100644 index 000000000..6bf97cbb1 --- /dev/null +++ b/app/Model/Behavior/LightPaginatorBehavior.php @@ -0,0 +1,19 @@ + [ + 'fields' => [ + 'Attribute.event_id', + 'Attribute.id', + 'Attribute.type', + 'Attribute.value1', + 'Attribute.value2', + ], + 'contain' => [ + 'Event' => [ + 'fields' => [ + 'Event.id', + 'Event.disable_correlation', + ] + ] + ], + ] + ]; + + public $Correlation = null; + + private $deadlockAvoidance = false; + + public function setup(Model $Model, $settings = []) { + $Model->useTable = $this->__tableName; + $this->Correlation = $Model; + $this->deadlockAvoidance = $settings['deadlockAvoidance']; + } + + public function getTableName(Model $Model) + { + return $this->__tableName; + } + + public function createCorrelationEntry(Model $Model, $value, $a, $b) { + $value_id = $this->Correlation->CorrelationValue->getValueId($value); + if ($this->deadlockAvoidance) { + return [ + 'value_id' => $value_id, + '1_event_id' => $a['Event']['id'], + '1_attribute_id' => $a['Attribute']['id'], + 'event_id' => $b['Event']['id'], + 'attribute_id' => $b['Attribute']['id'] + ]; + } else { + return [ + (int) $value_id, + (int) $a['Event']['id'], + (int) $a['Attribute']['id'], + (int) $b['Event']['id'], + (int) $b['Attribute']['id'] + ]; + } + } + + public function saveCorrelations(Model $Model, $correlations) + { + $fields = [ + 'value_id', + '1_event_id', + '1_attribute_id', + 'event_id', + 'attribute_id' + ]; + + if ($this->deadlockAvoidance) { + return $this->Correlation->saveMany($correlations, array( + 'atomic' => false, + 'callbacks' => false, + 'deep' => false, + 'validate' => false, + 'fieldList' => $fields + )); + } else { + $db = $this->Correlation->getDataSource(); + // Split to chunks datasource is is enabled + if (count($correlations) > 100) { + foreach (array_chunk($correlations, 100) as $chunk) { + $db->insertMulti('no_acl_correlations', $fields, $chunk); + } + return true; + } else { + return $db->insertMulti('no_acl_correlations', $fields, $correlations); + } + } + } + + public function runBeforeSaveCorrelation(Model $Model, $attribute) { + // (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id + // ==> DELETE FROM no_acl_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */ + // first check if it's an update + if (isset($attribute['id'])) { + $Model->deleteAll([ + 'OR' => [ + '1_attribute_id' => $attribute['id'], + 'attribute_id' => $attribute['id'] + ], + ], false); + } + if ($attribute['type'] === 'ssdeep') { + $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); + $this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']); + } + } + + public function getContainRules(Model $Model, $filter = null) + { + if (empty($filter)) { + return $this->__config['AttributeFetcher']['contain']; + } else { + return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter]; + } + } + + public function getFieldRules(Model $Model) + { + return $this->__config['AttributeFetcher']['fields']; + } + + private function __collectCorrelations($user, $id, $primary) + { + $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; + $source = $primary ? '' : '1_'; + $prefix = $primary ? '1_' : ''; + $correlations = $this->Correlation->find('all', [ + 'fields' => [ + $source . 'attribute_id', + $prefix . 'attribute_id', + $prefix . 'event_id' + ], + 'conditions' => [ + 'OR' => [ + $source . 'event_id' => $id + ], + 'AND' => [ + [ + 'CorrelationValue.value NOT IN (select value from correlation_exclusions)' + ], + [ + 'CorrelationValue.value NOT IN (select value from over_correlating_values)' + ] + ] + ], + 'recursive' => -1, + 'contain' => [ + 'CorrelationValue' => [ + 'fields' => [ + 'CorrelationValue.id', + 'CorrelationValue.value' + ] + ] + ], + 'order' => false, + 'limit' => $max_correlations + ]); + return $correlations; + } + + public function runGetAttributesRelatedToEvent(Model $Model, $user, $id) + { + $temp_correlations = $this->__collectCorrelations($user, $id, false); + $temp_correlations_1 = $this->__collectCorrelations($user, $id, true); + $correlations = []; + $event_ids = []; + foreach ($temp_correlations as $temp_correlation) { + $correlations[] = [ + 'id' => $temp_correlation['Correlation']['event_id'], + 'attribute_id' => $temp_correlation['Correlation']['attribute_id'], + 'parent_id' => $temp_correlation['Correlation']['1_attribute_id'], + 'value' => $temp_correlation['CorrelationValue']['value'] + ]; + $event_ids[$temp_correlation['Correlation']['event_id']] = true; + } + foreach ($temp_correlations_1 as $temp_correlation) { + $correlations[] = [ + 'id' => $temp_correlation['Correlation']['1_event_id'], + 'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'], + 'parent_id' => $temp_correlation['Correlation']['attribute_id'], + 'value' => $temp_correlation['CorrelationValue']['value'] + ]; + $event_ids[$temp_correlation['Correlation']['1_event_id']] = true; + } + if (empty($correlations)) { + return []; + } + $conditions = [ + 'Event.id' => array_keys($event_ids) + ]; + $events = $Model->Event->find('all', array( + 'recursive' => -1, + 'conditions' => $conditions, + 'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'], + )); + + $events = array_column(array_column($events, 'Event'), null, 'id'); + $relatedAttributes = []; + foreach ($correlations as $correlation) { + $eventId = $correlation['id']; + if (!isset($events[$eventId])) { + continue; + } + $event = $events[$eventId]; + $correlation['org_id'] = $events[$eventId]['orgc_id']; + $correlation['info'] = $events[$eventId]['info']; + $correlation['date'] = $events[$eventId]['date']; + $parentId = $correlation['parent_id']; + unset($correlation['parent_id']); + $relatedAttributes[$parentId][] = $correlation; + } + return $relatedAttributes; + } + + public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false) + { + // LATER getRelatedAttributes($attribute) this might become a performance bottleneck + // prepare the conditions + $conditions = [ + [ + 'Correlation.1_event_id !=' => $attribute['event_id'], + 'Correlation.attribute_id' => $attribute['id'] + ], + [ + 'Correlation.event_id !=' => $attribute['event_id'], + 'Correlation.1_attribute_id' => $attribute['id'] + ] + ]; + $corr_fields = [ + [ + '1_attribute_id', + '1_event_id', + 'value_id' + ], + [ + 'attribute_id', + 'event_id', + 'value_id' + ] + ]; + $prefixes = ['1_', '']; + $correlated_attribute_ids = []; + foreach ($conditions as $k => $condition) { + $temp_correlations = $Model->find('all', [ + 'recursive' => -1, + 'conditions' => $condition, + 'fields' => $corr_fields[$k] + ]); + if (!empty($temp_correlations)) { + foreach ($temp_correlations as $temp_correlation) { + $correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id']; + } + } + } + $contain = []; + if (!empty($includeEventData)) { + $contain['Event'] = [ + 'fields' => [ + 'Event.id', + 'Event.uuid', + 'Event.threat_level_id', + 'Event.analysis', + 'Event.info', + 'Event.extends_uuid', + 'Event.distribution', + 'Event.sharing_group_id', + 'Event.published', + 'Event.date', + 'Event.orgc_id', + 'Event.org_id' + ] + ]; + } + $relatedAttributes = $Model->Attribute->find('all', [ + 'recursive' => -1, + 'conditions' => [ + 'Attribute.id' => $correlated_attribute_ids + ], + 'fields' => $fields, + 'contain' => $contain + ]); + if (!empty($includeEventData)) { + $results = []; + foreach ($relatedAttributes as $k => $attribute) { + $temp = $attribute['Attribute']; + $temp['Event'] = $attribute['Event']; + $results[] = $temp; + } + return $results; + } else { + return $relatedAttributes; + } + } + + public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids) + { + // search the correlation table for the event ids of the related events + // Rules: + // 1. Event is owned by the user (org_id matches) + // 2. User is allowed to see both the event and the org: + // a. Event: + // i. Event has a distribution between 1-3 (community only, connected communities, all orgs) + // ii. Event has a sharing group that the user is accessible to view + // b. Attribute: + // i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway) + // ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) + // iii. Attribute has a sharing group that the user is accessible to view + $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, true); + $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, false); + return array_unique(array_merge($primaryEventIds,$secondaryEventIds)); + + } + + private function __filterRelatedEvents(Model $Model, array $user, int $eventId, bool $primary) + { + $current = $primary ? '' : '1_'; + $prefix = $primary ? '1_' : ''; + $correlations = $Model->find('all', [ + 'recursive' => -1, + 'fields' => [ + $prefix . 'event_id' + ], + 'conditions' => [ + $current . 'event_id' => $eventId + ], + 'unique' => true, + ]); + $eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id'); + return $eventIds; + } + + public function updateContainedCorrelations( + Model $Model, + array $data, + string $type = 'event', + array $options = [] + ): bool + { + // We don't care. No ACL means nothing to change. + return true; + } + + public function purgeCorrelations(Model $Model, $eventId = null): void + { + if (!$eventId) { + $Model->query('TRUNCATE TABLE no_acl_correlations;'); + //$Model->query('TRUNCATE TABLE correlation_values;'); + //$Model->query('TRUNCATE TABLE over_correlating_values;'); + } else { + $Model->deleteAll([ + 'OR' => array( + 'Correlation.1_event_id' => $eventId, + 'Correlation.event_id' => $eventId, + ) + ], false); + } + } + + public function purgeByValue(Model $Model, string $value): void + { + $valueIds = $Model->CorrelationValue->find('column', [ + 'recursive' => -1, + 'conditions' => [ + 'OR' => [ + 'CorrelationValue.value LIKE' => '%' . $value, + 'CorrelationValue.value LIKE' => $value . '%' + ] + ], + 'fields' => [ + 'CorrelationValue.id' + ] + ]); + $Model->deleteAll([ + 'Correlation.value_id' => $valueIds + ]); + } +} diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 9b536ba04..bb1025f20 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -10,6 +10,8 @@ class Correlation extends AppModel const CACHE_NAME = 'misp:top_correlations', CACHE_AGE = 'misp:top_correlations_age'; + private $__compositeTypes = []; + public $belongsTo = array( 'Attribute' => [ 'className' => 'Attribute', @@ -18,7 +20,25 @@ class Correlation extends AppModel 'Event' => array( 'className' => 'Event', 'foreignKey' => 'event_id' - ) + ), + 'Object' => array( + 'className' => 'Object', + 'foreignKey' => 'object_id' + ), + 'CorrelationValue' => [ + 'className' => 'CorrelationValue', + 'foreignKey' => 'value_id' + ] + ); + + public $validEngines = [ + 'Default' => 'default_correlations', + 'NoAcl' => 'no_acl_correlations', + 'Legacy' => 'correlations' + ]; + + public $actsAs = array( + 'Containable' ); /** @var array */ @@ -39,12 +59,26 @@ class Correlation extends AppModel /** @var array */ private $cidrListCache; + private $__correlationEngine = 'DefaultCorrelation'; + + protected $_config = []; + + private $__tempContainCache = []; + + public $OverCorrelatingValue = null; + public function __construct($id = false, $table = null, $ds = null) { parent::__construct($id, $table, $ds); - $this->oldSchema = $this->schema('date') !== null; - $this->deadlockAvoidance = Configure::read('MISP.deadlock_avoidance'); + $this->__correlationEngine = $this->getCorrelationModelName(); + $this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false; + // load the currently used correlation engine + $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]); + // getTableName() needs to be implemented by the engine - this points us to the table to be used + $this->useTable = $this->getTableName(); $this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations'); + // load the overcorrelatingvalue model for chaining + $this->OverCorrelatingValue = ClassRegistry::init('OverCorrelatingValue'); } public function correlateValueRouter($value) @@ -89,9 +123,9 @@ class Correlation extends AppModel return null; } - if (in_array($attribute['type'], ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], true)) { + if (in_array($attribute['Attribute']['type'], ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], true)) { return $this->cidrCorrelation($attribute); - } else if ($attribute['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) { + } else if ($attribute['Attribute']['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) { return $this->ssdeepCorrelation($attribute); } return null; @@ -102,7 +136,7 @@ class Correlation extends AppModel if (!$this->advancedCorrelationEnabled) { return []; } - $extraConditions = $this->__buildAdvancedCorrelationConditions($correlatingAttribute['Attribute']); + $extraConditions = $this->__buildAdvancedCorrelationConditions($correlatingAttribute); if (empty($extraConditions)) { return []; } @@ -117,25 +151,16 @@ class Correlation extends AppModel 'Attribute.deleted' => 0 ], 'recursive' => -1, - 'fields' => [ - 'Attribute.event_id', - 'Attribute.id', - 'Attribute.distribution', - 'Attribute.sharing_group_id', - 'Attribute.value1', - 'Attribute.value2', - ], - 'contain' => [ - 'Event' => [ - 'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'] - ] - ], + 'fields' => $this->getFieldRules(), + 'contain' => $this->getContainRules(), 'order' => [], ]); } private function __getMatchingAttributes($value) { + // stupid hack to allow statically retrieving the constants + ClassRegistry::init('Attribute'); $conditions = [ 'OR' => [ 'Attribute.value1' => $value, @@ -154,20 +179,8 @@ class Correlation extends AppModel $correlatingAttributes = $this->Attribute->find('all', [ 'conditions' => $conditions, 'recursive' => -1, - 'fields' => [ - 'Attribute.event_id', - 'Attribute.id', - 'Attribute.type', - 'Attribute.distribution', - 'Attribute.sharing_group_id', - 'Attribute.value1', - 'Attribute.value2', - ], - 'contain' => [ - 'Event' => [ - 'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'] - ] - ], + 'fields' => $this->getFieldRules(), + 'contain' => $this->getContainRules(), 'order' => [], ]); return $correlatingAttributes; @@ -179,35 +192,9 @@ class Correlation extends AppModel * @param array $b Attribute B * @return array */ - private function __addCorrelationEntry($value, $a, $b) + private function __createCorrelationEntry($value, $a, $b) { - if ($this->deadlockAvoidance) { - return [ - 'value' => $value, - '1_event_id' => $a['Event']['id'], - '1_attribute_id' => $a['Attribute']['id'], - 'event_id' => $b['Event']['id'], - 'attribute_id' => $b['Attribute']['id'], - 'org_id' => $b['Event']['org_id'], - 'distribution' => $b['Event']['distribution'], - 'a_distribution' => $b['Attribute']['distribution'], - 'sharing_group_id' => $b['Event']['sharing_group_id'], - 'a_sharing_group_id' => $b['Attribute']['sharing_group_id'], - ]; - } else { - return [ - $value, - (int) $a['Event']['id'], - (int) $a['Attribute']['id'], - (int) $b['Event']['id'], - (int) $b['Attribute']['id'], - (int) $b['Event']['org_id'], - (int) $b['Event']['distribution'], - (int) $b['Attribute']['distribution'], - (int) $b['Event']['sharing_group_id'], - (int) $b['Attribute']['sharing_group_id'], - ]; - } + return $this->createCorrelationEntry($value, $a, $b); } public function correlateValue($value, $jobId = false) @@ -232,7 +219,7 @@ class Correlation extends AppModel if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) { continue; } - $correlations[] = $this->__addCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2); + $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2); } $extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute); if (!empty($extraCorrelations)) { @@ -240,8 +227,8 @@ class Correlation extends AppModel if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) { continue; } - $correlations[] = $this->__addCorrelationEntry($value, $correlatingAttribute, $extraCorrelation); - //$correlations = $this->__addCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations); + $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation); + //$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations); } } if ($jobId && $k % 100 === 0) { @@ -258,74 +245,38 @@ class Correlation extends AppModel * @param array $correlations * @return array|bool|bool[]|mixed */ - private function __saveCorrelations($correlations) + private function __saveCorrelations(array $correlations) { - $fields = [ - 'value', '1_event_id', '1_attribute_id', 'event_id', 'attribute_id', 'org_id', - 'distribution', 'a_distribution', 'sharing_group_id', 'a_sharing_group_id', - ]; - - // In older MISP instances, correlations table contains also date and info columns, that stores information - // about correlated event title and date. But because this information can be fetched directly from Event table, - // it is not necessary to keep them there. The problem is that these columns are marked as not null, so they must - // be filled with value and removing these columns can take long time for big instances. So for new installation - // these columns doesn't exists anymore and we don't need to save dummy value into them. Also feel free to remove - // them from your instance. - if ($this->oldSchema) { - $fields[] = 'date'; - $fields[] = 'info'; - } - - if ($this->deadlockAvoidance) { - if ($this->oldSchema) { - foreach ($correlations as &$correlation) { - $correlation['date'] = '1000-01-01'; // Dummy value - $correlation['info'] = ''; // Dummy value - } - } - return $this->saveMany($correlations, array( - 'atomic' => false, - 'callbacks' => false, - 'deep' => false, - 'validate' => false, - 'fieldList' => $fields - )); - } else { - if ($this->oldSchema) { - foreach ($correlations as &$correlation) { - $correlation[] = '1000-01-01'; // Dummy value - $correlation[] = ''; // Dummy value - } - } - $db = $this->getDataSource(); - // Split to chunks datasource is is enabled - if (count($correlations) > 100) { - foreach (array_chunk($correlations, 100) as $chunk) { - $db->insertMulti('correlations', $fields, $chunk); - } - return true; - } else { - return $db->insertMulti('correlations', $fields, $correlations); - } - } + return $this->saveCorrelations($correlations); } - public function beforeSaveCorrelation($attribute) + public function correlateAttribute(array $attribute): void { - // (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id - // ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */ - // first check if it's an update - if (isset($attribute['id'])) { - $this->deleteAll([ - 'OR' => [ - 'Correlation.1_attribute_id' => $attribute['id'], - 'Correlation.attribute_id' => $attribute['id'] - ], - ], false); - } - if ($attribute['type'] === 'ssdeep') { - $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); - $this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']); + $this->runBeforeSaveCorrelation($attribute); + $this->afterSaveCorrelation($attribute); + } + + public function beforeSaveCorrelation(array $attribute): void + { + $this->runBeforeSaveCorrelation($attribute); + } + + private function __cachedGetContainData($scope, $id) + { + if (!empty($this->getContainRules($scope))) { + if (empty($this->__tempContainCache[$scope][$id])) { + $temp = $this->Attribute->$scope->find('first', array( + 'recursive' => -1, + 'fields' => $this->getContainRules($scope)['fields'], + 'conditions' => ['id' => $id], + 'order' => array(), + )); + $temp = empty($temp) ? false : $temp[$scope]; + $this->__tempContainCache[$scope][$id] = $temp; + return $temp; + } else { + return $this->__tempContainCache[$scope][$id]; + } } } @@ -337,42 +288,48 @@ class Correlation extends AppModel */ public function afterSaveCorrelation($a, $full = false, $event = false) { - if (!empty($a['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) { + $a = ['Attribute' => $a]; + if (!empty($a['Attribute']['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) { return true; } // Don't do any correlation if the type is a non correlating type - if (in_array($a['type'], Attribute::NON_CORRELATING_TYPES, true)) { + if (in_array($a['Attribute']['type'], Attribute::NON_CORRELATING_TYPES, true)) { return true; } if (!$event) { - $event = $this->Attribute->Event->find('first', array( - 'recursive' => -1, - 'fields' => array('Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'), - 'conditions' => array('id' => $a['event_id']), - 'order' => array(), - )); + $a['Event'] = $this->__cachedGetContainData('Event', $a['Attribute']['event_id']); + if (!$a['Event']) { + // orphaned attribute, do not correlate + return true; + } + } else { + $a['Event'] = $event['Event']; } - - if (!empty($event['Event']['disable_correlation'])) { + if (!empty($a['Event']['disable_correlation'])) { return true; } + + if (!empty($a['Attribute']['object_id'])) { + $a['Object'] = $this->__cachedGetContainData('Object', $a['Attribute']['object_id']); + if (!$a['Object']) { + // orphaned attribute, do not correlate + return true; + } + } // generate additional correlating attribute list based on the advanced correlations - if (!$this->__preventExcludedCorrelations($a['value1'])) { + if (!$this->__preventExcludedCorrelations($a['Attribute']['value1'])) { $extraConditions = $this->__buildAdvancedCorrelationConditions($a); - $correlatingValues = [$a['value1']]; + $correlatingValues = [$a['Attribute']['value1']]; } else { $extraConditions = null; - $correlatingValues = [null]; + $correlatingValues = []; } - if (!empty($a['value2']) && !in_array($a['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true) && !$this->__preventExcludedCorrelations($a['value2'])) { - $correlatingValues[] = $a['value2']; + if (!empty($a['Attribute']['value2']) && !in_array($a['Attribute']['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true) && !$this->__preventExcludedCorrelations($a['Attribute']['value2'])) { + $correlatingValues[] = $a['Attribute']['value2']; } - if (empty($correlatingValues)) { return true; } - - $attributeToProcess = ['Attribute' => $a, 'Event' => $event['Event']]; $correlations = []; foreach ($correlatingValues as $k => $cV) { if ($cV === null) { @@ -387,42 +344,48 @@ class Correlation extends AppModel ], ], 'NOT' => [ - 'Attribute.event_id' => $a['event_id'], + 'Attribute.event_id' => $a['Attribute']['event_id'], 'Attribute.type' => Attribute::NON_CORRELATING_TYPES, ], 'Attribute.disable_correlation' => 0, 'Event.disable_correlation' => 0, 'Attribute.deleted' => 0, ]; - $fields = ['Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id']; - if ($k === 0 && !empty($extraConditions)) { - $conditions['OR'][] = $extraConditions; - // Fetch value field just when fetching attributes also by extra conditions, because then it can be - // not exact match - $fields[] = 'Attribute.value1'; - $fields[] = 'Attribute.value2'; - } - if ($full) { - $conditions['Attribute.id > '] = $a['id']; - } + $correlationLimit = $this->OverCorrelatingValue->getLimit(); + $correlatingAttributes = $this->Attribute->find('all', [ 'conditions' => $conditions, 'recursive' => -1, - 'fields' => $fields, - 'contain' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'], + 'fields' => $this->getFieldRules(), + 'contain' => $this->getContainRules(), 'order' => [], 'callbacks' => 'before', // memory leak fix + // let's fetch the limit +1 - still allows us to detect overcorrelations, but we'll also never need more + 'limit' => empty($correlationLimit) ? null : ($correlationLimit+1) ]); - foreach ($correlatingAttributes as $corr) { - if (isset($corr['Attribute']['value1'])) { + // Let's check if we don't have a case of an over-correlating attribute + $count = count($correlatingAttributes); + if ($count > $correlationLimit) { + // If we have more correlations for the value than the limit, set the block entry and stop the correlation process + $this->OverCorrelatingValue->block($cV, $count); + return true; + } else { + // If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers + $this->OverCorrelatingValue->unblock($cV); + } + foreach ($correlatingAttributes as $b) { + if (isset($b['Attribute']['value1'])) { // TODO: Currently it is hard to check if value1 or value2 correlated, so we check value2 and if not, it is value1 - $value = $cV === $corr['Attribute']['value2'] ? $corr['Attribute']['value2'] : $corr['Attribute']['value1']; + $value = $cV === $b['Attribute']['value2'] ? $b['Attribute']['value2'] : $b['Attribute']['value1']; } else { $value = $cV; } - $correlations[] = $this->__addCorrelationEntry($value, $attributeToProcess, $corr); - $correlations[] = $this->__addCorrelationEntry($cV, $corr, $attributeToProcess); + if ($a['Attribute']['id'] > $b['Attribute']['id']) { + $correlations[] = $this->__createCorrelationEntry($value, $a, $b); + } else { + $correlations[] = $this->__createCorrelationEntry($value, $b, $a); + } } } if (empty($correlations)) { @@ -486,8 +449,8 @@ class Correlation extends AppModel if (!isset($this->FuzzyCorrelateSsdeep)) { $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); } - $value = $attribute['value1']; - $fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($value, $attribute['id']); + $value = $attribute['Attribute']['value1']; + $fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($value, $attribute['Attribute']['id']); if (!empty($fuzzyIds)) { $ssdeepIds = $this->Attribute->find('list', array( 'recursive' => -1, @@ -517,7 +480,7 @@ class Correlation extends AppModel private function cidrCorrelation($attribute) { $ipValues = array(); - $ip = $attribute['value1']; + $ip = $attribute['Attribute']['value1']; if (strpos($ip, '/') !== false) { // IP is CIDR list($networkIp, $mask) = explode('/', $ip); $ip_version = filter_var($networkIp, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6; @@ -703,7 +666,7 @@ class Correlation extends AppModel $maxPage = ceil($maxId / $chunkSize); for ($page = 0; $page < $maxPage; $page++) { $correlations = $this->find('column', [ - 'fields' => ['value'], + 'fields' => ['value_id'], 'conditions' => [ 'id >' => $page * $chunkSize, 'id <=' => ($page + 1) * $chunkSize @@ -724,7 +687,7 @@ class Correlation extends AppModel return true; } - public function findTop(array $query) + public function findTop(array $query): array { try { $redis = $this->setupRedisWithException(); @@ -732,13 +695,20 @@ class Correlation extends AppModel return false; } $start = $query['limit'] * ($query['page'] -1); - $end = $query['limit'] * $query['page']; + $end = $query['limit'] * $query['page'] - 1; $list = $redis->zRevRange(self::CACHE_NAME, $start, $end, true); $results = []; foreach ($list as $value => $count) { + $realValue = $this->CorrelationValue->find('first', + [ + 'recursive' => -1, + 'conditions' => ['CorrelationValue.id' => $value], + 'fields' => 'CorrelationValue.value' + ] + ); $results[] = [ 'Correlation' => [ - 'value' => $value, + 'value' => $realValue['CorrelationValue']['value'], 'count' => $count, 'excluded' => $this->__preventExcludedCorrelations($value), ] @@ -757,11 +727,22 @@ class Correlation extends AppModel return $redis->get(self::CACHE_AGE); } + /** + * @param array $attribute + * @return void + */ + public function advancedCorrelationsUpdate(array $attribute) + { + if ($this->advancedCorrelationEnabled && in_array($attribute['type'], ['ip-src', 'ip-dst'], true) && strpos($attribute['value'], '/')) { + $this->updateCidrList(); + } + } + /** * Get list of all CIDR for correlation from database * @return array */ - private function getCidrListFromDatabase() + private function getCidrListFromDatabase(): array { return $this->Attribute->find('column', [ 'conditions' => [ @@ -779,9 +760,9 @@ class Correlation extends AppModel /** * @return array */ - public function updateCidrList() + public function updateCidrList(): array { - $redis = $this->setupRedis(); + $redis = $this->setupRedisWithException(); $cidrList = []; $this->cidrListCache = null; if ($redis) { @@ -804,7 +785,7 @@ class Correlation extends AppModel /** * @return void */ - public function clearCidrCache() + public function clearCidrCache(): void { $this->cidrListCache = null; } @@ -812,13 +793,13 @@ class Correlation extends AppModel /** * @return array */ - public function getCidrList() + public function getCidrList(): array { if ($this->cidrListCache !== null) { return $this->cidrListCache; } - $redis = $this->setupRedis(); + $redis = $this->setupRedisWithException(); if ($redis) { if (!$redis->exists('misp:cidr_cache_list')) { $cidrList = $this->updateCidrList(); @@ -831,4 +812,155 @@ class Correlation extends AppModel $this->cidrListCache = $cidrList; return $cidrList; } + + /** + * @param array $user User array + * @param int $eventIds List of event IDs + * @param array $sgids List of sharing group IDs + * @return array + */ + public function getAttributesRelatedToEvent(array $user, $eventIds, array $sgids): array + { + return $this->runGetAttributesRelatedToEvent($user, $eventIds, $sgids); + } + + + /** + * @param array $user User array + * @param array $attribute Attribute Array + * @param array $fields List of fields to include + * @param bool $includeEventData Flag to include the event data in the response + * @return array + */ + public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false): array + { + if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES)) { + return []; + } + return $this->runGetRelatedAttributes($user, $sgids, $attribute, $fields, $includeEventData); + } + + /** + * @param array $user User array + * @param int $eventIds List of event IDs + * @param array $sgids List of sharing group IDs + * @return array + */ + public function getRelatedEventIds(array $user, int $eventId, array $sgids): array + { + $relatedEventIds = $this->fetchRelatedEventIds($user, $eventId, $sgids); + if (empty($relatedEventIds)) { + return []; + } + return $relatedEventIds; + } + + public function attachExclusionsToOverCorrelations($data) + { + foreach ($data as $k => $v) { + $data[$k]['OverCorrelatingValue']['excluded'] = $this->__preventExcludedCorrelations($data[$k]['OverCorrelatingValue']['value']); + } + return $data; + } + + public function setCorrelationExclusion($attribute) + { + if (empty($this->__compositeTypes)) { + $this->__compositeTypes = $this->Attribute->getCompositeTypes(); + } + $values = [$attribute['value']]; + if (in_array($attribute['type'], $this->__compositeTypes)) { + $values = explode('|', $attribute['value']); + } + if ($this->__preventExcludedCorrelations($values[0])) { + $attribute['correlation_exclusion'] = true; + } + if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) { + $attribute['correlation_exclusion'] = true; + } + if ($this->OverCorrelatingValue->checkValue($values[0])) { + $attribute['over_correlation'] = true; + } + if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) { + $attribute['over_correlation'] = true; + } + return $attribute; + } + + public function collectMetrics() + { + $results['engine'] = $this->getCorrelationModelName(); + $results['db'] = [ + 'Default' => [ + 'name' => __('Default correlation engine'), + 'tables' => [ + 'default_correlations' => [ + 'id_limit' => 4294967295 + ], + 'correlation_values' => [ + 'id_limit' => 4294967295 + ] + ] + ], + 'NoAcl' => [ + 'name' => __('No ACL correlation engine'), + 'tables' => [ + 'no_acl_correlations' => [ + 'id_limit' => 4294967295 + ], + 'correlation_values' => [ + 'id_limit' => 4294967295 + ] + ] + ], + 'Legacy' => [ + 'name' => __('Legacy correlation engine (< 2.4.160)'), + 'tables' => [ + 'correlations' => [ + 'id_limit' => 2147483647 + ] + ] + ] + ]; + $results['over_correlations'] = $this->OverCorrelatingValue->find('count'); + $this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion'); + $results['excluded_correlations'] = $this->CorrelationExclusion->find('count'); + foreach ($results['db'] as &$result) { + foreach ($result['tables'] as $table_name => &$table_data) { + $size_metrics = $this->query(sprintf('show table status like \'%s\';', $table_name)); + if (!empty($size_metrics)) { + $table_data['size_on_disk'] = $this->query( + //'select FILE_SIZE from information_schema.innodb_sys_tablespaces where FILENAME like \'%/' . $table_name . '.ibd\';' + sprintf( + 'select TABLE_NAME, ROUND((DATA_LENGTH + INDEX_LENGTH)) AS size FROM information_schema.TABLES where TABLE_SCHEMA="%s" AND TABLE_NAME="%s"', + $this->getDataSource()->config['database'], + $table_name + ) + )[0][0]['size']; + $last_id = $this->query(sprintf('select max(id) as max_id from %s;', $table_name)); + $table_data['row_count'] = $size_metrics[0]['TABLES']['Rows']; + $table_data['last_id'] = $last_id[0][0]['max_id']; + $table_data['id_saturation'] = round(100 * $table_data['last_id'] / $table_data['id_limit'], 2); + } + } + } + return $results; + } + + public function truncate(array $user, string $engine) + { + $table = $this->validEngines[$engine]; + $result = $this->query('truncate table ' . $table); + if ($result !== true) { + $this->loadLog()->createLogEntry( + $user, + 'truncate', + 'Correlation', + 0, + 'Could not truncate table ' . $table, + 'Errors: ' . json_encode($result) + ); + } + return $result === true; + } } diff --git a/app/Model/CorrelationExclusion.php b/app/Model/CorrelationExclusion.php index ca2303a6e..7d1bc890d 100644 --- a/app/Model/CorrelationExclusion.php +++ b/app/Model/CorrelationExclusion.php @@ -16,6 +16,8 @@ class CorrelationExclusion extends AppModel 'Containable', ); + private $__redis = null; + public $validate = [ 'value' => [ 'uniqueValue' => [ @@ -52,15 +54,15 @@ class CorrelationExclusion extends AppModel public function cacheValues() { try { - $redis = $this->setupRedisWithException(); + $this->__redis = $this->setupRedisWithException(); } catch (Exception $e) { return false; } - $redis->del($this->key); + $this->__redis->del($this->key); $exclusions = $this->find('column', [ 'fields' => ['value'] ]); - $redis->sAddArray($this->key, $exclusions); + $this->__redis->sAddArray($this->key, $exclusions); } public function cleanRouter($user) @@ -97,19 +99,16 @@ class CorrelationExclusion extends AppModel $this->Job = ClassRegistry::init('Job'); $this->Job->id = $jobId; } - $query = sprintf( - 'DELETE FROM correlations where (%s) or (%s);', - sprintf( - 'value IN (%s)', - 'SELECT correlation_exclusions.value FROM correlation_exclusions WHERE correlations.value = correlation_exclusions.value' - ), - sprintf( - 'EXISTS (SELECT NULL FROM correlation_exclusions WHERE (%s) OR (%s))', - "correlations.value LIKE CONCAT('%', correlation_exclusions.value)", - "correlations.value LIKE CONCAT(correlation_exclusions.value, '%')" - ) - ); - $this->query($query); - $this->Job->saveProgress($jobId, 'Job done.', 100); + $values = $this->find('column', [ + 'recursive' => -1, + 'fields' => ['value'] + ]); + $this->Correlation = ClassRegistry::init('Correlation'); + foreach ($values as $value) { + $this->Correlation->purgeByValue($value); + } + if ($jobId) { + $this->Job->saveProgress($jobId, 'Job done.', 100); + } } } diff --git a/app/Model/CorrelationValue.php b/app/Model/CorrelationValue.php new file mode 100644 index 000000000..726ce6510 --- /dev/null +++ b/app/Model/CorrelationValue.php @@ -0,0 +1,59 @@ +find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'value' => $value + ] + ]); + if (empty($existingValue)) { + $this->create(); + try { + $this->save(['value' => $value]); + return $this->id; + } catch (Exception $e) { + $existingValue = $this->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'value' => $value + ] + ]); + return $existingValue['ExistingValue']['id']; + } + } else { + return $existingValue['CorrelationValue']['id']; + } + return false; + } + + public function getValue($id) + { + $existingValue = $this->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'id' => $id + ] + ]); + if (!empty($existingValue)) { + return $existingValue['CorrelationValue']['value']; + } + return false; + } +} diff --git a/app/Model/Event.php b/app/Model/Event.php index 2077d26f8..f2c5f7eda 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -17,6 +17,7 @@ App::uses('ProcessTool', 'Tools'); * @property ThreatLevel $ThreatLevel * @property Sighting $Sighting * @property Organisation $Org + * @property Organisation $Orgc * @property CryptographicKey $CryptographicKey */ class Event extends AppModel @@ -301,6 +302,9 @@ class Event extends AppModel private $assetCache = []; + /** @var array|null */ + private $eventBlockRule; + public function beforeDelete($cascade = true) { // blocklist the event UUID if the feature is enabled @@ -434,16 +438,7 @@ class Event extends AppModel { $event = $this->data['Event']; if (!Configure::read('MISP.completely_disable_correlation') && !$created) { - $updateCorrelation = []; - if (isset($event['distribution']) && (empty($options['fieldList']) || in_array('distribution', $options['fieldList']))) { - $updateCorrelation['Correlation.distribution'] = (int)$event['distribution']; - } - if (isset($event['sharing_group_id']) && (empty($options['fieldList']) || in_array('sharing_group_id', $options['fieldList']))) { - $updateCorrelation['Correlation.sharing_group_id'] = (int)$event['sharing_group_id']; - } - if (!empty($updateCorrelation)) { - $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]); - } + $this->Attribute->Correlation->updateContainedCorrelations($event, 'event'); } if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && $this->pubToZmq('event')) { $pubSubTool = $this->getPubSubTool(); @@ -455,6 +450,17 @@ class Event extends AppModel if (empty($event['unpublishAction']) && empty($event['skip_kafka'])) { $this->publishKafkaNotification('event', $this->quickFetchEvent($event['id']), $created ? 'add' : 'edit'); } + if ($this->isTriggerCallable('event-after-save')) { + $event = $this->quickFetchEvent($event['id']); + $workflowErrors = []; + $logging = [ + 'model' => 'Event', + 'action' => $created ? 'add' : 'edit', + 'id' => $event['Event']['id'], + ]; + $triggerData = $event; + $this->executeTrigger('event-after-save', $triggerData, $workflowErrors, $logging); + } } public function attachTagsToEvents(array $events) @@ -483,6 +489,47 @@ class Event extends AppModel return $events; } + public function touch($event_id) + { + $event = $this->find('first', [ + 'conditions' => ['Event.id' => $event_id], + 'recursive' => -1, + ]); + $event['Event']['published'] = 0; + $event['Event']['timestamp'] = (new DateTime())->getTimestamp(); + return $this->save($event, true, ['timestamp', 'published']); + } + + public function attachTagsToEventAndTouch($event_id, $tags) + { + $touchEvent = false; + $success = false; + foreach ($tags as $tagId) { + $nothingToChange = false; + $success = $success || $this->EventTag->attachTagToEvent($event_id, ['id' => $tagId], $nothingToChange); + $touchEvent = $touchEvent || !$nothingToChange; + } + if ($touchEvent) { + return $this->touch($event_id); + } + return $success; + } + + public function detachTagsFromEventAndTouch($event_id, $tags) + { + $touchEvent = false; + $success = false; + foreach ($tags as $tagId) { + $nothingToChange = false; + $success = $success || $this->EventTag->detachTagFromEvent($event_id, $tagId, $nothingToChange); + $touchEvent = $touchEvent || !$nothingToChange; + } + if ($touchEvent) { + return $this->touch($event_id); + } + return $success; + } + /** * Gets the logged in user + an array of events, attaches the correlation count to each * @param array $user @@ -492,16 +539,8 @@ class Event extends AppModel public function attachCorrelationCountToEvents(array $user, array $events) { $sgids = $this->SharingGroup->authorizedIds($user); - $eventIds = array_column(array_column($events, 'Event'), 'id'); - $conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventIds, $sgids); - $this->Attribute->Correlation->virtualFields['count'] = 'count(distinct(Correlation.event_id))'; - $correlations = $this->Attribute->Correlation->find('list', array( - 'fields' => array('Correlation.1_event_id', 'Correlation.count'), - 'conditions' => $conditionsCorrelation, - 'group' => array('Correlation.1_event_id'), - )); foreach ($events as &$event) { - $event['Event']['correlation_count'] = isset($correlations[$event['Event']['id']]) ? $correlations[$event['Event']['id']] : 0; + $event['Event']['correlation_count'] = $this->getRelatedEventCount($user, $event['Event']['id'], $sgids); } return $events; } @@ -569,61 +608,12 @@ class Event extends AppModel return $events; } - private function __buildEventConditionsCorrelation($user, $eventIds, $sgids) + public function getRelatedEventCount($user, $eventId, $sgids) { - if (!is_array($eventIds)) { - $eventIds = array($eventIds); + if (!isset($sgids) || empty($sgids)) { + $sgids = array(-1); } - if (!$user['Role']['perm_site_admin']) { - $conditionsCorrelation = array( - 'AND' => array( - 'Correlation.1_event_id' => $eventIds, - array( - 'OR' => array( - 'Correlation.org_id' => $user['org_id'], - 'AND' => array( - array( - 'OR' => array( - array( - 'AND' => array( - 'Correlation.distribution >' => 0, - 'Correlation.distribution <' => 4, - ), - ), - array( - 'AND' => array( - 'Correlation.distribution' => 4, - 'Correlation.sharing_group_id' => $sgids - ), - ), - ), - ), - array( - 'OR' => array( - 'Correlation.a_distribution' => 5, - array( - 'AND' => array( - 'Correlation.a_distribution >' => 0, - 'Correlation.a_distribution <' => 4, - ), - ), - array( - 'AND' => array( - 'Correlation.a_distribution' => 4, - 'Correlation.a_sharing_group_id' => $sgids - ), - ), - ), - ), - ), - ), - ), - ), - ); - } else { - $conditionsCorrelation = array('Correlation.1_event_id' => $eventIds); - } - return $conditionsCorrelation; + return count($this->Attribute->Correlation->getRelatedEventIds($user, $eventId, $sgids)); } private function getRelatedEvents($user, $eventId, $sgids) @@ -631,48 +621,31 @@ class Event extends AppModel if (!isset($sgids) || empty($sgids)) { $sgids = array(-1); } - // search the correlation table for the event ids of the related events - // Rules: - // 1. Event is owned by the user (org_id matches) - // 2. User is allowed to see both the event and the org: - // a. Event: - // i. Event has a distribution between 1-3 (community only, connected communities, all orgs) - // ii. Event has a sharing group that the user is accessible to view - // b. Attribute: - // i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway) - // ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) - // iii. Attribute has a sharing group that the user is accessible to view - $conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventId, $sgids); - $relatedEventIds = $this->Attribute->Correlation->find('column', array( - 'fields' => array('Correlation.event_id'), - 'conditions' => $conditionsCorrelation, - 'unique' => true, - )); - + $relatedEventIds = $this->Attribute->Correlation->getRelatedEventIds($user, $eventId, $sgids); if (empty($relatedEventIds)) { return []; } - // now look up the event data for these attributes - $conditions = $this->createEventConditions($user); - $conditions['AND'][] = array('Event.id' => $relatedEventIds); - $fields = array('id', 'date', 'threat_level_id', 'info', 'published', 'uuid', 'analysis', 'timestamp', 'distribution', 'org_id', 'orgc_id'); - $orgfields = array('id', 'name', 'uuid'); - $relatedEvents = $this->find( + $relatedEvents = $this->find( 'all', - array('conditions' => $conditions, + [ + 'conditions' => [ + 'Event.id' => $relatedEventIds + ], 'recursive' => -1, - 'order' => 'Event.date DESC', - 'fields' => $fields, - 'contain' => array( - 'Org' => array( - 'fields' => $orgfields - ), - 'Orgc' => array( - 'fields' => $orgfields - ) - ) - ) + 'order' => 'date DESC', + 'fields' => [ + 'id', 'date', 'threat_level_id', 'info', 'published', 'uuid', 'analysis', 'timestamp', 'distribution', 'org_id', 'orgc_id' + ], + 'contain' => [ + 'Org' => [ + 'fields' => ['id', 'name', 'uuid'] + ], + 'Orgc' => [ + 'fields' => ['id', 'name', 'uuid'] + ] + ] + ] ); $fieldsToRearrange = array('Org', 'Orgc'); foreach ($relatedEvents as $k => $relatedEvent) { @@ -696,8 +669,8 @@ class Event extends AppModel public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event') { if ($shadowAttribute) { - $parentIdField = '1_shadow_attribute_id'; - $correlationModelName = 'ShadowAttributeCorrelation'; + // no longer supported + return []; } else { $parentIdField = '1_attribute_id'; $correlationModelName = 'Correlation'; @@ -705,102 +678,8 @@ class Event extends AppModel if (!isset($this->{$correlationModelName})) { $this->{$correlationModelName} = ClassRegistry::init($correlationModelName); } - if (!$user['Role']['perm_site_admin']) { - $sgids = $this->SharingGroup->authorizedIds($user); - $conditionsCorrelation = array( - 'AND' => array( - $correlationModelName . '.1_' . $scope . '_id' => $id, - array( - 'OR' => array( - $correlationModelName . '.org_id' => $user['org_id'], - 'AND' => array( - array( - 'OR' => array( - array( - 'AND' => array( - $correlationModelName . '.distribution >' => 0, - $correlationModelName . '.distribution <' => 4, - ), - ), - array( - 'AND' => array( - $correlationModelName . '.distribution' => 4, - $correlationModelName . '.sharing_group_id' => $sgids - ), - ), - ), - ), - array( - 'OR' => array( - $correlationModelName . '.a_distribution' => 5, - array( - 'AND' => array( - $correlationModelName . '.a_distribution >' => 0, - $correlationModelName . '.a_distribution <' => 4, - ), - ), - array( - 'AND' => array( - $correlationModelName . '.a_distribution' => 4, - $correlationModelName . '.a_sharing_group_id' => $sgids - ), - ), - ), - ), - ), - ) - ) - - ) - ); - } else { - $conditionsCorrelation = array($correlationModelName . '.1_' . $scope . '_id' => $id); - } - $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; - $correlations = $this->{$correlationModelName}->find('all', array( - 'fields' => ['event_id', 'attribute_id', 'value', $parentIdField], - 'conditions' => $conditionsCorrelation, - 'recursive' => -1, - 'order' => false, - 'limit' => $max_correlations - )); - if (empty($correlations)) { - return array(); - } - - $correlations = array_column($correlations, $correlationModelName); - $eventIds = array_unique(array_column($correlations, 'event_id')); - - $conditions = $this->createEventConditions($user); - $conditions['Event.id'] = $eventIds; - $events = $this->find('all', array( - 'recursive' => -1, - 'conditions' => $conditions, - 'fields' => ['Event.id', 'Event.orgc_id', 'Event.info', 'Event.date'], - )); - - $events = array_column(array_column($events, 'Event'), null, 'id'); - - $relatedAttributes = []; - foreach ($correlations as $correlation) { - // User don't have access to correlated attribute event, skip. - $eventId = $correlation['event_id']; - if (!isset($events[$eventId])) { - continue; - } - - $event = $events[$eventId]; - $current = array( - 'id' => $eventId, - 'attribute_id' => $correlation['attribute_id'], - 'value' => $correlation['value'], - 'org_id' => $event['orgc_id'], - 'info' => $event['info'], - 'date' => $event['date'], - ); - $parentId = $correlation[$parentIdField]; - $relatedAttributes[$parentId][] = $current; - } + $sgids = $this->SharingGroup->authorizedIds($user); + $relatedAttributes = $this->{$correlationModelName}->getAttributesRelatedToEvent($user, $id, $sgids); return $relatedAttributes; } @@ -836,12 +715,13 @@ class Event extends AppModel /** * @param array $event * @param array $server + * @param ServerSyncTool $serverSync * @return false|string * @throws HttpSocketJsonException * @throws JsonException * @throws Exception */ - public function uploadEventToServer(array $event, array $server) + public function uploadEventToServer(array $event, array $server, ServerSyncTool $serverSync) { $this->Server = ClassRegistry::init('Server'); @@ -852,8 +732,6 @@ class Event extends AppModel return 'The distribution level of this event blocks it from being pushed.'; } - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); - $push = $this->Server->checkVersionCompatibility($server, false, $serverSync); if (empty($push['canPush'])) { return 'The remote user is not a sync user - the upload of the event has been blocked.'; @@ -2101,7 +1979,7 @@ class Event extends AppModel if (!empty($options['includeRelatedTags'])) { $event = $this->includeRelatedTags($event, $options); } - $event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true); + //$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true); } $shadowAttributeByOldId = []; if (!empty($event['ShadowAttribute'])) { @@ -2147,6 +2025,7 @@ class Event extends AppModel unset($event['Attribute'][$key]); continue; } + $attribute = $this->Attribute->Correlation->setCorrelationExclusion($attribute); if ($attribute['category'] === 'Financial fraud') { $attribute = $this->Attribute->attachValidationWarnings($attribute); } @@ -2753,7 +2632,7 @@ class Event extends AppModel public function set_filter_published(&$params, $conditions, $options) { - if (isset($params['published'])) { + if (isset($params['published']) && $params['published'] !== [true, false]) { $conditions['AND']['Event.published'] = $params['published']; } return $conditions; @@ -3131,8 +3010,8 @@ class Event extends AppModel { if (Configure::read('MISP.extended_alert_subject')) { $subject = preg_replace("/\r|\n/", "", $event['Event']['info']); - if (strlen($subject) > 58) { - $subject = substr($subject, 0, 55) . '... - '; + if (mb_strlen($subject) > 58) { + $subject = mb_substr($subject, 0, 55) . '... - '; } else { $subject .= " - "; } @@ -3159,6 +3038,10 @@ class Event extends AppModel $template->set('tlp', $subjMarkingString); $template->subject($subject); $template->referenceId("event-alert|{$event['Event']['id']}"); + + $unsubscribeLink = $this->__getAnnounceBaseurl() . '/users/unsubscribe/' . $this->User->unsubscribeCode($user); + $template->set('unsubscribe', $unsubscribeLink); + $template->listUnsubscribe($unsubscribeLink); return $template; } @@ -3433,17 +3316,18 @@ class Event extends AppModel return $attributes; } - public function checkEventBlockRules($event) + /** + * @param array $event + * @return bool + */ + private function checkEventBlockRules(array $event) { - if (!isset($this->AdminSetting)) { + if (!isset($this->eventBlockRule)) { $this->AdminSetting = ClassRegistry::init('AdminSetting'); + $setting = $this->AdminSetting->getSetting('eventBlockRule'); + $this->eventBlockRule = $setting ? json_decode($setting, true) : false; } - $setting = $this->AdminSetting->getSetting('eventBlockRule'); - if (empty($setting)) { - return true; - } - $rules = json_decode($setting, true); - if (empty($rules)) { + if (empty($this->eventBlockRule)) { return true; } if (!empty($rules['tags'])) { @@ -3544,7 +3428,9 @@ class Event extends AppModel public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = array()) { if (Configure::read('MISP.enableEventBlocklisting') !== false && isset($data['Event']['uuid'])) { - $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); + if (!isset($this->EventBlocklist)) { + $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); + } if ($this->EventBlocklist->isBlocked($data['Event']['uuid'])) { return 'Blocked by blocklist'; } @@ -4215,8 +4101,15 @@ class Event extends AppModel $failedServers = []; foreach ($servers as $server) { + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); try { - $this->pushSightingsToServer($server, $event, $sightingsUuidsToPush); + try { + if ($serverSync->eventExists($event) === false) { + continue; // skip if event not exists on remote server + } + } catch (Exception $e) {} + + $this->pushSightingsToServer($serverSync, $event, $sightingsUuidsToPush); } catch (Exception $e) { $this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e); $failedServers[] = $server['Server']['url']; @@ -4229,25 +4122,16 @@ class Event extends AppModel } /** - * @param array $server + * @param ServerSyncTool $serverSync * @param array $event * @param array $sightingsUuidsToPush * @throws HttpSocketJsonException - * @throws JsonException * @throws Exception */ - private function pushSightingsToServer(array $server, array $event, array $sightingsUuidsToPush = []) + private function pushSightingsToServer(ServerSyncTool $serverSync, array $event, array $sightingsUuidsToPush = []) { - App::uses('ServerSyncTool', 'Tools'); - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); - try { - if ($serverSync->eventExists($event) === false) { - return; // skip if event not exists on remote server - } - } catch (Exception $e) {} - $fakeSyncUser = [ - 'org_id' => $server['Server']['remote_org_id'], + 'org_id' => $serverSync->server()['Server']['remote_org_id'], 'Role' => [ 'perm_site_admin' => 0, ], @@ -4316,8 +4200,6 @@ class Event extends AppModel } $uploaded = true; $failedServers = []; - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); foreach ($servers as $server) { if ( @@ -4327,7 +4209,7 @@ class Event extends AppModel } // Skip servers where the event has come from. if ($passAlong != $server['Server']['id']) { - $HttpSocket = $syncTool->setupHttpSocket($server); + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); $params = [ 'eventid' => $id, 'includeAttachments' => true, @@ -4355,17 +4237,17 @@ class Event extends AppModel 'perm_site_admin' => 0 ) ); - $this->Server->syncGalaxyClusters($HttpSocket, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event); - $thisUploaded = $this->uploadEventToServer($event, $server); + $this->Server->syncGalaxyClusters($serverSync, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event); + $thisUploaded = $this->uploadEventToServer($event, $server, $serverSync); if ($thisUploaded === 'Success') { try { - $this->pushSightingsToServer($server, $event); // push sighting by method that check for duplicates + $this->pushSightingsToServer($serverSync, $event); // push sighting by method that check for duplicates } catch (Exception $e) { $this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e); } } if (isset($this->data['ShadowAttribute'])) { - $this->Server->syncProposals($HttpSocket, $server, null, $id, $this); + $this->Server->syncProposals(null, $server, null, $id, $this); } if (!$thisUploaded) { $uploaded = !$uploaded ? $uploaded : $thisUploaded; @@ -4438,7 +4320,6 @@ class Event extends AppModel $jobId ); } - return $this->publish($id, $passAlong); } @@ -4483,9 +4364,59 @@ class Event extends AppModel 'recursive' => -1, 'conditions' => array('Event.id' => $id) )); + if (empty($event)) { return false; } + $hostOrg = $this->Org->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'id' => Configure::read('MISP.host_org_id') + ], + ]); + if (empty($hostOrg)) { + $hostOrg = $this->Org->find('first', [ + 'recursive' => -1, + 'order' => ['id ASC'] + ]); + } + $userForPubSub = [ + 'id' => 0, + 'org_id' => $hostOrg['Org']['id'], + 'Role' => ['perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 1], + 'Organisation' => $hostOrg['Org'] + ]; + $allowZMQ = Configure::read('Plugin.ZeroMQ_enable'); + $kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic'); + $allowKafka = Configure::read('Plugin.Kafka_enable') && + Configure::read('Plugin.Kafka_event_publish_notifications_enable') && + !empty($kafkaTopic); + $triggerCallable = $this->isTriggerCallable('event-publish'); + + if ($allowZMQ || $allowKafka || $triggerCallable) { + $currentUserId = Configure::read('CurrentUserId'); + $userForWorkflow = $this->User->getAuthUser($currentUserId, true); + $userForWorkflow['Role']['perm_site_admin'] = 1; + $fullEvent = $this->fetchEvent($userForWorkflow, [ + 'eventid' => $id, + 'includeAttachments' => 1 + ]); + } + if ($triggerCallable) { + $workflowErrors = []; + $logging = [ + 'model' => 'Event', + 'action' => 'publish', + 'id' => $id, + 'message' => __('Publishing stopped by a blocking workflow.'), + ]; + $success = $this->executeTrigger('event-publish', $fullEvent[0], $workflowErrors, $logging); + if (empty($success)) { + $errorMessage = implode(', ', $workflowErrors); + + return $errorMessage; + } + } if ($jobId) { $this->Behaviors->unload('SysLogLogable.SysLogLogable'); } else { @@ -4498,50 +4429,11 @@ class Event extends AppModel $event['Event']['skip_kafka'] = 1; $this->save($event, array('fieldList' => $fieldList)); } - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_event_publish_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_event_publish_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { - $hostOrgId = Configure::read('MISP.host_org_id'); - if (!empty($hostOrgId)) { - $hostOrg = $this->Org->find('first', [ - 'recursive' => -1, - 'conditions' => [ - 'id' => $hostOrgId - ] - ] - ); - } - if (empty($hostOrg)) { - $hostOrg = $this->Org->find('first', [ - 'recursive' => -1, - 'order' => ['id ASC'] - ]); - $hostOrgId = $hostOrg['Org']['id']; - } - $user = array('org_id' => $hostOrgId, 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']); - if ($pubToZmq) { - $params = array('eventid' => $id); - if (Configure::read('Plugin.ZeroMQ_include_attachments')) { - $params['includeAttachments'] = 1; - } - $fullEvent = $this->fetchEvent($user, $params); - if (!empty($fullEvent)) { - $pubSubTool = $this->getPubSubTool(); - $pubSubTool->publishEvent($fullEvent[0], 'publish'); - } - } - if ($pubToKafka) { - $params = array('eventid' => $id); - if (Configure::read('Plugin.Kafka_include_attachments')) { - $params['includeAttachments'] = 1; - } - $fullEvent = $this->fetchEvent($user, $params); - if (!empty($fullEvent)) { - $kafkaPubTool = $this->getKafkaPubTool(); - $kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish'); - } - } + if ($allowZMQ) { + $this->publishEventToZmq($id, $userForPubSub, $fullEvent); + } + if ($allowKafka) { + $this->publishEventToKafka($id, $userForPubSub, $fullEvent, $kafkaTopic); } return $this->uploadEventToServersRouter($id, $passAlong); } @@ -4753,30 +4645,6 @@ class Event extends AppModel return $xmlArray; } - public function removeOlder(array &$events, $scope = 'events') - { - $field = $scope === 'sightings' ? 'sighting_timestamp' : 'timestamp'; - $localEvents = $this->find('all', [ - 'recursive' => -1, - 'fields' => ['Event.uuid', 'Event.' . $field, 'Event.locked'], - ]); - $localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid'); - foreach ($events as $k => $event) { - // remove all events for the sighting sync if the remote is not aware of the new field yet - if (!isset($event[$field])) { - unset($events[$k]); - } else { - $uuid = $event['uuid']; - if (isset($localEvents[$uuid]) - && ($localEvents[$uuid][$field] >= $event[$field] - || ($scope === 'events' && !$localEvents[$uuid]['locked']))) - { - unset($events[$k]); - } - } - } - } - public function sharingGroupRequired($field) { if ($this->data[$this->alias]['distribution'] == 4) { @@ -4813,7 +4681,6 @@ class Event extends AppModel if ($filterType) { $include = true; - /* proposal */ if ($filterType['proposal'] == 0) { // `both` // pass, do not consider as `both` is selected @@ -5459,7 +5326,7 @@ class Event extends AppModel 'Object.event_id' => $event_id, 'Object.deleted' => 0), 'recursive' => -1, - 'fields' => array('Object.id', 'Object.uuid', 'Object.name') + 'fields' => array('Object.id', 'Object.uuid', 'Object.name', 'Object.distribution', 'Object.sharing_group_id') )); if (!empty($initial_object)) { $initial_attributes = $this->Attribute->find('all', array( @@ -5923,6 +5790,8 @@ class Event extends AppModel public function enrichmentRouter($options) { + $result = $this->enrichment($options); + return __('#' . $result . ' attributes have been created during the enrichment process.'); if (Configure::read('MISP.background_jobs')) { /** @var Job $job */ @@ -5964,10 +5833,30 @@ class Event extends AppModel throw new MethodNotAllowedException(__('%s not set', $option_field)); } } - $event = $this->fetchEvent($params['user'], array('eventid' => $params['event_id'], 'includeAttachments' => 1, 'flatten' => 1)); + $event = []; + if (!empty($params['attribute_uuids'])) { + $attributes = $this->Attribute->fetchAttributes($params['user'], [ + 'conditions' => [ + 'Attribute.uuid' => $params['attribute_uuids'], + ], + 'withAttachments' => 1, + ]); + $event = [ + [ + 'Event' => ['id' => $params['event_id']], + 'Attribute' => Hash::extract($attributes, '{n}.Attribute') + ] + ]; + } else { + $event = $this->fetchEvent($params['user'], [ + 'eventid' => $params['event_id'], + 'includeAttachments' => 1, + 'flatten' => 1, + ]); + } $this->Module = ClassRegistry::init('Module'); $enabledModules = $this->Module->getEnabledModules($params['user']); - if (empty($enabledModules)) { + if (empty($enabledModules) || is_string($enabledModules)) { return true; } $options = array(); @@ -5988,6 +5877,9 @@ class Event extends AppModel $event_id = $event[0]['Event']['id']; foreach ($event[0]['Attribute'] as $attribute) { $object_id = $attribute['object_id']; + if ($object_id != '0' && empty($initial_objects[$object_id])) { + $initial_objects[$object_id] = $this->fetchInitialObject($event_id, $object_id); + } foreach ($enabledModules['modules'] as $module) { if (in_array($module['name'], $params['modules'])) { if (in_array($attribute['type'], $module['mispattributes']['input'])) { @@ -5997,15 +5889,19 @@ class Event extends AppModel } if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') { $data['attribute'] = $attribute; - if ($object_id != '0' && empty($initial_objects[$object_id])) { - $initial_objects[$object_id] = $this->fetchInitialObject($event_id, $object_id); - } } else { $data[$attribute['type']] = $attribute['value']; } - $result = $this->Module->queryModuleServer($data, false, 'Enrichment'); - if (!$result) { + if ($object_id != '0' && !empty($initial_objects[$object_id])) { + $attribute['Object'] = $initial_objects[$object_id]['Object']; + } + $triggerData = $event[0]; + $triggerData['Attribute'] = [$attribute]; + $result = $this->Module->queryModuleServer($data, false, 'Enrichment', false, $triggerData); + if ($result === false) { throw new MethodNotAllowedException(h($module['name']) . ' service not reachable.'); + } else if (!is_array($result)) { + continue 2; } //if (isset($result['error'])) $this->Session->setFlash($result['error']); if (!is_array($result)) { @@ -7636,4 +7532,51 @@ class Event extends AppModel ), ); } + + private function __prepareEventForPubSub($id, $user, &$fullEvent) + { + if ($fullEvent) { + if (empty(Configure::read('Plugin.ZeroMQ_include_attachments'))) { + foreach ($fullEvent[0]['Attribute'] as $k => $attribute) { + if (isset($attribute['data'])) { + unset($fullEvent[0]['Attribute'][$k]['data']); + } + } + foreach ($fullEvent[0]['Object'] as $k => $object) { + foreach ($object['Attribute'] as $k2 => $attribute) { + if (isset($attribute['data'])) { + unset($fullEvent[0]['Object'][$k]['Attribute'][$k2]['data']); + } + } + } + } + } else { + $params = [ + 'eventid' => $id + ]; + if (Configure::read('Plugin.ZeroMQ_include_attachments')) { + $params['includeAttachments'] = 1; + } + $fullEvent = $this->fetchEvent($user, $params); + } + return $fullEvent; + } + + public function publishEventToZmq($id, $user, &$fullEvent) + { + $fullEvent = $this->__prepareEventForPubSub($id, $user, $fullEvent); + if (!empty($fullEvent)) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->publishEvent($fullEvent[0], 'publish'); + } + } + + public function publishEventToKafka($id, $user, &$fullEvent, $kafkaTopic) + { + $fullEvent = $this->__prepareEventForPubSub($id, $user, $fullEvent); + if (!empty($fullEvent)) { + $kafkaPubTool = $this->getKafkaPubTool(); + $kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish'); + } + } } diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 84e368fb2..73d7d5a32 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -628,13 +628,11 @@ class GalaxyCluster extends AppModel $elevatedUser = array( 'Role' => array( 'perm_site_admin' => 1, - 'perm_sync' => 1 + 'perm_sync' => 1, + 'perm_audit' => 0, ), 'org_id' => $clusterOrgcId['GalaxyCluster']['orgc_id'] ); - $elevatedUser['Role']['perm_site_admin'] = 1; - $elevatedUser['Role']['perm_sync'] = 1; - $elevatedUser['Role']['perm_audit'] = 0; $cluster = $this->fetchGalaxyClusters($elevatedUser, array('minimal' => true, 'conditions' => array('id' => $clusterId)), $full=false); if (empty($cluster)) { return true; @@ -655,14 +653,10 @@ class GalaxyCluster extends AppModel return true; } $uploaded = false; - $failedServers = array(); - App::uses('SyncTool', 'Tools'); - foreach ($servers as &$server) { + foreach ($servers as $server) { if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $cluster['GalaxyCluster']['distribution'] < 2) { continue; } - $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket($server); $fakeSyncUser = array( 'id' => 0, 'email' => 'fakeSyncUser@user.test', @@ -678,14 +672,13 @@ class GalaxyCluster extends AppModel ); $cluster = $this->fetchGalaxyClusters($fakeSyncUser, array('conditions' => array('GalaxyCluster.id' => $clusterId)), $full=true); if (empty($cluster)) { - return true; + continue; } $cluster = $cluster[0]; - $result = $this->uploadClusterToServer($cluster, $server, $HttpSocket, $fakeSyncUser); - if ($result == 'Success') { + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + $result = $this->uploadClusterToServer($cluster, $server, $serverSync, $fakeSyncUser); + if ($result === 'Success') { $uploaded = true; - } else { - $failedServers[] = $server; } } return $uploaded; @@ -1124,7 +1117,7 @@ class GalaxyCluster extends AppModel if (!empty($tagsToFetch)) { $tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [ - 'conditions' => ['id' => array_unique($tagsToFetch)], + 'conditions' => ['id' => array_unique($tagsToFetch, SORT_REGULAR)], 'recursive' => -1, ]); $tags = array_column(array_column($tags, 'Tag'), null, 'id'); @@ -1672,79 +1665,29 @@ class GalaxyCluster extends AppModel /** * @return string|bool The result of the upload. True if success, a string otherwise + * @throws Exception */ - public function uploadClusterToServer($cluster, $server, $HttpSocket, $user) - { - $this->Server = ClassRegistry::init('Server'); - $this->Log = ClassRegistry::init('Log'); - $push = $this->Server->checkVersionCompatibility($server, false); - if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) { - return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); - } - $updated = null; - $newLocation = $newTextBody = ''; - $result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user); - if ($result !== true) { - return $result; - } - if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http:// - $result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user); - if ($result !== true) { - return $result; - } - } - $uploadFailed = false; - try { - $json = json_decode($newTextBody, true); - } catch (Exception $e) { - $uploadFailed = true; - } - if (!is_array($json) || $uploadFailed) { - $this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody); - } - return 'Success'; - } - - private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user) - { - $result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket); - if (is_numeric($result)) { - $error = $this->__resolveErrorCode($result, $cluster, $server, $user); - if ($error) { - return $error . ' Error code: ' . $result; - } - } - return true; - } - - /** - * @return string|bool|int The result of the upload. - */ - public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null) + public function uploadClusterToServer(array $cluster, array $server, ServerSyncTool $serverSync, array $user) { $cluster = $this->__prepareForPushToServer($cluster, $server); if (is_numeric($cluster)) { return $cluster; } - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $scope = 'galaxies/pushCluster'; - $uri = $url . '/' . $scope; - $clusters = array($cluster); - $data = json_encode($clusters); - if (!empty(Configure::read('Security.sync_audit'))) { - $pushLogEntry = sprintf( - "==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n", - date("Y-m-d H:i:s"), - $cluster['GalaxyCluster']['id'], - $server['Server']['id'], - $data - ); - file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND); + + try { + if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || $serverSync->isSupported(ServerSyncTool::PERM_GALAXY_EDITOR)) { + return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); + } + $serverSync->pushGalaxyCluster($cluster)->json(); + } catch (Exception $e) { + $title = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']); + $this->loadLog()->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], $title, $e->getMessage()); + + $this->logException("Could not push galaxy cluster to remote server {$serverSync->serverId()}", $e); + return $e->getMessage(); } - $response = $HttpSocket->post($uri, $data, $request); - return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody); + + return 'Success'; } /** @@ -1752,7 +1695,7 @@ class GalaxyCluster extends AppModel * * @param array $cluster * @param array $server - * @return array The cluster ready to be pushed + * @return array|int The cluster ready to be pushed */ private function __prepareForPushToServer(array $cluster, array $server) { @@ -1773,11 +1716,9 @@ class GalaxyCluster extends AppModel } $this->Event = ClassRegistry::init('Event'); if ($this->Event->checkDistributionForPush($cluster, $server, 'GalaxyCluster')) { - $cluster = $this->__updateClusterForSync($cluster, $server); - } else { - return 403; + return $this->__updateClusterForSync($cluster, $server); } - return $cluster; + return 403; } /** @@ -1886,90 +1827,39 @@ class GalaxyCluster extends AppModel return $relation; } - /** - * @return string|bool|int The result of the upload. - */ - private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody) - { - switch ($response->code) { - case '200': // 200 (OK) + entity-action-result - if ($response->isOk()) { - $newTextBody = $response->body(); - return true; - } else { - try { - $jsonArray = json_decode($response->body, true); - } catch (Exception $e) { - return true; - } - return $jsonArray['name']; - } - // no break - case '302': // Found - $newLocation = $response->headers['Location']; - $newTextBody = $response->body(); - return true; - case '404': // Not Found - $newLocation = $response->headers['Location']; - $newTextBody = $response->body(); - return 404; - case '405': - return 405; - case '403': // Not authorised - return 403; - } - } - - private function __resolveErrorCode($code, &$cluster, &$server, $user) - { - $this->Log = ClassRegistry::init('Log'); - $error = false; - switch ($code) { - case 403: - return __('The distribution level of the cluster blocks it from being pushed.'); - case 405: - $error = __('The sync user on the remote instance does not have the required privileges to handle this cluster.'); - break; - } - if ($error) { - $newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')'; - $newTextBody = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']); - $this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody); - } - return $error; - } - /** * pullGalaxyClusters * - * @param array $user - * @param array $server - * @param string|int $technique The technique startegy used for pulling + * @param array $user + * @param ServerSyncTool $serverSync + * @param string|int $technique The technique startegy used for pulling * allowed: * - int event containing the clusters to pulled * - string pull everything * - string pull updates of cluster present locally * - string pull clusters based on tags present locally * @return int The number of pulled clusters + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException */ - public function pullGalaxyClusters(array $user, array $server, $technique = 'full') + public function pullGalaxyClusters(array $user, ServerSyncTool $serverSync, $technique = 'full') { - $this->Server = ClassRegistry::init('Server'); - $compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster']; + $compatible = $serverSync->isSupported(ServerSyncTool::FEATURE_EDIT_OF_GALAXY_CLUSTER); if (!$compatible) { return 0; } - $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $server); - $successes = array(); - $fails = array(); + $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync); + $successes = 0; // now process the $clusterIds to pull each of the events sequentially if (!empty($clusterIds)) { // download each cluster - foreach ($clusterIds as $k => $clusterId) { - $this->__pullGalaxyCluster($clusterId, $successes, $fails, $server, $user); + foreach ($clusterIds as $clusterId) { + if ($this->__pullGalaxyCluster($clusterId, $serverSync, $user)) { + $successes++; + } } } - return count($successes); + return $successes; } /** @@ -1977,16 +1867,16 @@ class GalaxyCluster extends AppModel * * @param array $user * @param string|int $technique - * @param array $server + * @param ServerSyncTool $serverSync * @return array cluster ID list to be pulled */ - private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server) + private function getClusterIdListBasedOnPullTechnique(array $user, $technique, ServerSyncTool $serverSync) { $this->Server = ClassRegistry::init('Server'); try { if ("update" === $technique) { $localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate); } elseif ("pull_relevant_clusters" === $technique) { // Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end $tagNames = $this->Tag->find('column', array( @@ -2005,55 +1895,39 @@ class GalaxyCluster extends AppModel } $localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user); $conditions = array('uuid' => array_keys($clusterUUIDs)); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions); } elseif (is_numeric($technique)) { $conditions = array('eventid' => $technique); - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions); } else { - $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false); + $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false); } } catch (HttpSocketHttpException $e) { if ($e->getCode() === 403) { return array('error' => array(1, null)); } else { - $this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e); + $this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e); return array('error' => array(2, $e->getMessage())); } } catch (Exception $e) { - $this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e); + $this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e); return array('error' => array(2, $e->getMessage())); } return $clusterIds; } - private function __pullGalaxyCluster($clusterId, &$successes, &$fails, $server, $user) + private function __pullGalaxyCluster($clusterId, ServerSyncTool $serverSync, array $user) { - $cluster = $this->downloadGalaxyClusterFromServer($clusterId, $server); - if (!empty($cluster)) { - $cluster = $this->updatePulledClusterBeforeInsert($cluster, $server, $user); - $result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$server['Server']['org_id']); - if ($result['success']) { - $successes[] = $clusterId; - } else { - $fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']); - } - } else { - $fails[$clusterId] = __('failed downloading the galaxy cluster'); + try { + $cluster = $serverSync->fetchGalaxyCluster($clusterId)->json(); + } catch (Exception $e) { + $this->logException("Could not fetch galaxy cluster $clusterId from server {$serverSync->serverId()}", $e); + return false; } - return true; - } - public function downloadGalaxyClusterFromServer($clusterId, $server, $HttpSocket=null) - { - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $uri = $url . '/galaxy_clusters/view/' . $clusterId; - $response = $HttpSocket->get($uri, $data = '', $request); - if ($response->isOk()) { - return json_decode($response->body, true); - } - return null; + $cluster = $this->updatePulledClusterBeforeInsert($cluster, $serverSync->server(), $user); + $result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$serverSync->server()['Server']['org_id']); + return $result['success']; } private function updatePulledClusterBeforeInsert($cluster, $server, $user) diff --git a/app/Model/Log.php b/app/Model/Log.php index 7461e0c95..4c8d14bed 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -39,11 +39,14 @@ class Log extends AppModel 'enrichment', 'error', 'execute_blueprint', + 'execute_workflow', + 'exec_module', 'export', 'fetchEvent', 'file_upload', 'galaxy', 'include_formula', + 'load_module', 'login', 'login_fail', 'logout', @@ -104,6 +107,8 @@ class Log extends AppModel 'email' => array('values' => array('admin_email')) ); + public $actsAs = ['LightPaginator']; + /** * Null when not defined, false when not enabled * @var Syslog|null|false @@ -202,7 +207,7 @@ class Log extends AppModel */ public function createLogEntry($user, $action, $model, $modelId = 0, $title = '', $change = '') { - if (in_array($action, ['tag', 'galaxy', 'publish', 'publish_sightings', 'enable'], true) && Configure::read('MISP.log_new_audit')) { + if (in_array($action, ['tag', 'galaxy', 'publish', 'publish_sightings', 'enable', 'edit'], true) && Configure::read('MISP.log_new_audit')) { return; // Do not store tag changes when new audit is enabled } if ($user === 'SYSTEM') { diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index b927c9548..578ec7638 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -300,6 +300,31 @@ class MispObject extends AppModel public function afterSave($created, $options = array()) { + if (!Configure::read('MISP.completely_disable_correlation') && !$created) { + $object = $this->data['Object']; + $this->Attribute->Correlation->updateContainedCorrelations($object, 'object'); + } + if (!empty($this->data['Object']['deleted']) && !$created) { + $attributes_to_delete = $this->Attribute->find('all', [ + 'recursive' => -1, + 'conditions' => [ + 'Attribute.object_id' => $this->id, + 'Attribute.deleted' => 0 + ] + ]); + foreach ($attributes_to_delete as &$attribute_to_delete) { + $attribute_to_delete['Attribute']['deleted'] = 1; + unset($attribute_to_delete['Attribute']['timestamp']); + } + $this->Attribute->saveMany($attributes_to_delete); + } + $workflowErrors = []; + $logging = [ + 'model' => 'Object', + 'action' => $created ? 'add' : 'edit', + 'id' => $this->data['Object']['id'], + ]; + $this->executeTrigger('object-after-save', $this->data, $workflowErrors, $logging); $pubToZmq = $this->pubToZmq('object') && empty($this->data['Object']['skip_zmq']); $kafkaTopic = $this->kafkaTopic('object'); $pubToKafka = $kafkaTopic && empty($this->data['Object']['skip_kafka']); @@ -1004,6 +1029,9 @@ class MispObject extends AppModel $objectId = $this->id; if (!empty($object['Object']['Attribute'])) { foreach ($object['Object']['Attribute'] as $attribute) { + if (!empty($object['Object']['deleted'])) { + $attribute['deleted'] = 1; + } $this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent); } } @@ -1088,6 +1116,9 @@ class MispObject extends AppModel } if (!empty($object['Attribute'])) { foreach ($object['Attribute'] as $attribute) { + if (!empty($object['deleted'])) { + $attribute['deleted'] = 1; + } $result = $this->Attribute->editAttribute($attribute, $event, $user, $object['id'], false, $force); } } diff --git a/app/Model/Module.php b/app/Model/Module.php index 003e050cd..858967814 100644 --- a/app/Model/Module.php +++ b/app/Model/Module.php @@ -10,12 +10,14 @@ class Module extends AppModel 'Enrichment' => array('hover', 'expansion'), 'Import' => array('import'), 'Export' => array('export'), + 'Action' => array('action'), 'Cortex' => array('cortex') ); private $__typeToFamily = array( 'Import' => 'Import', 'Export' => 'Export', + 'Action' => 'Action', 'hover' => 'Enrichment', 'expansion' => 'Enrichment', 'Cortex' => 'Cortex' @@ -131,6 +133,8 @@ class Module extends AppModel $output['Import'] = $temp['name']; } elseif (isset($temp['meta']['module-type']) && in_array('export', $temp['meta']['module-type'])) { $output['Export'] = $temp['name']; + } elseif (isset($temp['meta']['module-type']) && in_array('action', $temp['meta']['module-type'])) { + $output['Action'] = $temp['name']; } else { foreach ($temp['mispattributes']['input'] as $input) { if (!isset($temp['meta']['module-type']) || (in_array('expansion', $temp['meta']['module-type']) || in_array('cortex', $temp['meta']['module-type']))) { @@ -199,6 +203,53 @@ class Module extends AppModel return "$url:$port"; } + private function __prepareAndExectureForTrigger($postData, $triggerData=[]): bool + { + $this->Workflow = ClassRegistry::init('Workflow'); + $trigger_id = 'enrichment-before-query'; + $workflowErrors = []; + $logging = [ + 'model' => 'Workflow', + 'action' => 'execute_workflow', + 'id' => 0, + ]; + if (empty($triggerData) && $this->Workflow->isTriggerCallable($trigger_id) && !empty($postData['attribute_uuid'])) { + $this->User = ClassRegistry::init('User'); + $this->Attribute = ClassRegistry::init('Attribute'); + $user = $this->User->getAuthUser(Configure::read('CurrentUserId'), true); + $options = [ + 'conditions' => [ + 'Attribute.uuid' => $postData['attribute_uuid'], + ], + 'includeAllTags' => true, + 'includeAttributeUuid' => true, + 'flatten' => true, + 'deleted' => [0, 1], + 'withAttachments' => true, + 'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]], + ]; + $attributes = $this->Attribute->fetchAttributes($user, $options); + $triggerData = !empty($attributes) ? $attributes[0] : []; + $logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`', $trigger_id, $postData['attribute_uuid'], $triggerData['Attribute']['event_id'], $postData['module']); + } else { + if (isset($triggerData['Attribute'])) { + $logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`', + $trigger_id, + $triggerData['Attribute']['id'] ?? $triggerData['Attribute'][0]['id'], + $triggerData['Attribute']['event_id'] ?? $triggerData['Attribute'][0]['event_id'], + $postData['module'] + ); + } else { + $logging['message'] = __('The workflow `%s` prevented attribute `%s` (from event `%s`) to query the module `%s`', $trigger_id, $triggerData['Event']['Attribute'][0]['id'], $triggerData['Event']['id'], $postData['module']); + } + } + if (empty($triggerData)) { + return false; + } + $success = $this->executeTrigger($trigger_id, $triggerData, $workflowErrors, $logging); + return !empty($success); + } + /** * Send request to `/query` module endpoint. * @@ -209,8 +260,15 @@ class Module extends AppModel * @return array|false * @throws JsonException */ - public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false) + public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false, $triggerData=[]) { + if ($moduleFamily == 'Enrichment') { + $success = $this->__prepareAndExectureForTrigger($postData, $triggerData); + if (!$success) { + $trigger_id = 'enrichment-before-query'; + return __('Trigger `%s` blocked enrichment', $trigger_id); + } + } if ($hover) { $timeout = Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5; } else { @@ -290,15 +348,42 @@ class Module extends AppModel foreach ($modules as $module) { if (array_intersect($this->__validTypes[$moduleFamily], $module['meta']['module-type'])) { $moduleSettings = [ - array('name' => 'enabled', 'type' => 'boolean'), - array('name' => 'restrict', 'type' => 'orgs') + [ + 'name' => 'enabled', + 'type' => 'boolean', + 'description' => empty($module['meta']['description']) ? '' : $module['meta']['description'] + ] ]; - if (isset($module['meta']['config'])) { - foreach ($module['meta']['config'] as $key => $value) { - if (is_string($key)) { - $moduleSettings[] = array('name' => $key, 'type' => 'string', 'description' => $value); - } else { - $moduleSettings[] = array('name' => $value, 'type' => 'string'); + if ($moduleFamily !== 'Action') { + $moduleSettings[] = [ + 'name' => 'restrict', + 'type' => 'orgs', + 'description' => __('Restrict the use of this module to an organisation.') + ]; + if (isset($module['meta']['config'])) { + foreach ($module['meta']['config'] as $key => $value) { + if (is_array($value)) { + $name = is_string($key) ? $key : $value['name']; + $moduleSettings[] = [ + 'name' => $name, + 'type' => isset($value['type']) ? $value['type'] : 'string', + 'test' => isset($value['test']) ? $value['test'] : null, + 'description' => isset($value['description']) ? $value['description'] : null, + 'null' => isset($value['null']) ? $value['null'] : null, + 'test' => isset($value['test']) ? $value['test'] : null, + 'bigField' => isset($value['bigField']) ? $value['bigField'] : false, + 'cli_only' => isset($value['cli_only']) ? $value['cli_only'] : false, + 'redacted' => isset($value['redacted']) ? $value['redacted'] : false + ]; + } else if (is_string($key)) { + $moduleSettings[] = [ + 'name' => $key, + 'type' => 'string', + 'description' => $value + ]; + } else { + $moduleSettings[] = array('name' => $value, 'type' => 'string'); + } } } } diff --git a/app/Model/OverCorrelatingValue.php b/app/Model/OverCorrelatingValue.php new file mode 100644 index 000000000..11fac96a7 --- /dev/null +++ b/app/Model/OverCorrelatingValue.php @@ -0,0 +1,68 @@ +unblock($value); + $this->create(); + $this->save( + [ + 'value' => $value, + 'occurrence' => $count + ] + ); + } + + public function unBlock($value) + { + $this->deleteAll( + [ + 'OverCorrelatingValue.value' => $value + ] + ); + } + + public function getLimit() + { + return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20; + } + + public function getOverCorrelations($query) + { + $data = $this->find('all', $query); + $limit = $this->getLimit(); + foreach ($data as $k => $v) { + if ($v['OverCorrelatingValue']['occurrence'] >= $limit) { + $data[$k]['OverCorrelatingValue']['over_correlation'] = true; + } else { + $data[$k]['OverCorrelatingValue']['over_correlation'] = false; + } + } + return $data; + } + + public function checkValue($value) + { + $hit = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['value' => $value], + 'fields' => ['id'] + ]); + if (empty($hit)) { + return false; + } + return true; + } +} diff --git a/app/Model/Post.php b/app/Model/Post.php index dbcfeb4e2..9dee21bcc 100644 --- a/app/Model/Post.php +++ b/app/Model/Post.php @@ -28,6 +28,21 @@ class Post extends AppModel ), ); + public function afterSave($created, $options = array()) + { + $post = $this->data; + if ($this->isTriggerCallable('post-after-save')) { + $workflowErrors = []; + $logging = [ + 'model' => 'Post', + 'action' => $created ? 'add' : 'edit', + 'id' => $post['Post']['id'], + ]; + $triggerData = $post; + $this->executeTrigger('post-after-save', $triggerData, $workflowErrors, $logging); + } + } + public function sendPostsEmailRouter($user_id, $post_id, $event_id, $title, $message) { if (Configure::read('MISP.background_jobs')) { diff --git a/app/Model/Server.php b/app/Model/Server.php index f96d2e804..2e972f8f2 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -132,6 +132,59 @@ class Server extends AppModel public $syncTestErrorCodes = array(); + const MYSQL_RECOMMENDED_SETTINGS = [ + 'innodb_buffer_pool_size' => [ + 'default' => '134217728', + 'recommended' => '2147483648', + 'explanation' => 'The InnoDB buffer pool is the memory area where caches table and index data reside. It is the most important MySQL setting, in a dedicated server it should be around 3/4 of all the available RAM. In a shared server it should be around 1/2 of the available RAM.', + ], + 'innodb_dedicated_server' => [ + 'default' => '0', + 'recommended' => '', + 'explanation' => 'Set to `1` if the database is running in a dedicated server. The database engine will examine the available memory and dynamically set `innodb_buffer_pool_size`, `innodb_log_file_size`, `innodb_log_files_in_group` and `innodb_flush_method`. It is particularly useful in cloud enviroments that can be auto-scaled.', + ], + 'innodb_log_file_size' => [ + 'default' => '100663296', + 'recommended' => '629145600', + 'explanation' => 'This parameter determines the fixed size for MySQLs redo logs. Tuning this value affects the crash recovery time and also overall system performance.', + ], + 'innodb_log_files_in_group' => [ + 'default' => '2', + 'recommended' => '2', + 'explanation' => 'Defines the number of log files in the log group.', + ], + 'innodb_change_buffering' => [ + 'default' => 'none', + 'recommended' => 'none', + 'explanation' => 'Whether InnoDB performs change buffering, an optimization that delays write operations to secondary indexes so that the I/O operations can be performed sequentially, enabling it causes extremely long shutdown times for upgrades.', + ], + 'innodb_io_capacity' => [ + 'default' => '200', + 'recommended' => '1000', + 'explanation' => 'Defines the number of I/O operations per second (IOPS) available to InnoDB background tasks, such as flushing pages from the buffer pool and merging data from the change buffer.', + ], + 'innodb_io_capacity_max' => [ + 'default' => '2000', + 'recommended' => '2000', + 'explanation' => 'If flushing activity falls behind, InnoDB can flush more aggressively, at a higher rate of I/O operations per second (IOPS) than defined by the `innodb_io_capacity variable`.', + ], + 'innodb_stats_persistent' => [ + 'default' => 'ON', + 'recommended' => 'ON', + 'explanation' => 'Specifies whether InnoDB index statistics are persisted to disk. Otherwise, statistics may be recalculated frequently which can lead to variations in query execution plans.', + ], + 'innodb_read_io_threads' => [ + 'default' => '4', + 'recommended' => '16', + 'explanation' => 'The number of I/O threads for read operations in InnoDB.', + ], + 'innodb_write_io_threads' => [ + 'default' => '4', + 'recommended' => '4', + 'explanation' => 'The number of I/O threads for write operations in InnoDB.', + ], + ]; + public function __construct($id = false, $table = null, $ds = null) { parent::__construct($id, $table, $ds); @@ -161,7 +214,12 @@ class Server extends AppModel 'description' => 'If your your database is locked and is not updating, unlock it here.', 'ignore_disabled' => true, 'url' => '/servers/releaseUpdateLock/' - ) + ), + 'normalizeCustomTagsToTaxonomyFormat' => array( + 'title' => 'Normalize custom tags to taxonomy format', + 'description' => 'Transform all custom tags existing in a taxonomy into the taxonomy version', + 'url' => '/taxonomies/normalizeCustomTagsToTaxonomyFormat/' + ), ); public $validEventIndexFilters = array('searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute'); @@ -222,25 +280,14 @@ class Server extends AppModel throw new InvalidArgumentException("Invalid pull technique `$technique`."); } - private function __checkIfEventIsBlockedBeforePull($event) - { - if (Configure::read('MISP.enableEventBlocklisting') !== false) { - if (!isset($this->EventBlocklist)) { - $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); - } - if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) { - return true; - } - } - return false; - } - /** * @param array $event * @param array $server * @param array $user + * @param array $pullRules + * @param bool $pullRulesEmptiedEvent */ - private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false) + private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent = false) { // we have an Event array // The event came from a pull, so it should be locked. @@ -269,14 +316,12 @@ class Server extends AppModel } } + $typeFilteringEnabled = !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && + !empty($pullRules['type_attributes']['NOT']); if (isset($event['Event']['Attribute'])) { $originalCount = count($event['Event']['Attribute']); foreach ($event['Event']['Attribute'] as $key => $attribute) { - if ( - !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && - !empty($pullRules['type_attributes']['NOT']) && - in_array($attribute['type'], $pullRules['type_attributes']['NOT']) - ) { + if ($typeFilteringEnabled && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Attribute'][$key]); continue; } @@ -297,7 +342,7 @@ class Server extends AppModel } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) { + if ($typeFilteringEnabled && $originalCount > 0 && empty($event['Event']['Attribute'])) { $pullRulesEmptiedEvent = true; } } @@ -323,11 +368,7 @@ class Server extends AppModel if (isset($object['Attribute'])) { $originalAttributeCount = count($object['Attribute']); foreach ($object['Attribute'] as $j => $a) { - if ( - !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && - !empty($pullRules['type_attributes']['NOT']) && - in_array($a['type'], $pullRules['type_attributes']['NOT']) - ) { + if ($typeFilteringEnabled && in_array($a['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Object'][$i]['Attribute'][$j]); continue; } @@ -348,13 +389,13 @@ class Server extends AppModel } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) { + if ($typeFilteringEnabled && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) { unset($event['Event']['Object'][$i]); // Object is empty, get rid of it $pullRulesEmptiedEvent = true; } } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) { + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && empty($event['Event']['Object'])) { $pullRulesEmptiedEvent = true; } } @@ -466,7 +507,18 @@ class Server extends AppModel } } - private function __pullEvent($eventId, &$successes, &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false) + /** + * @param int|string $eventId Event ID or UUID + * @param array $successes + * @param array $fails + * @param Event $eventModel + * @param ServerSyncTool $serverSync + * @param array $user + * @param int $jobId + * @param bool $force + * @return bool + */ + private function __pullEvent($eventId, array &$successes, array &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false) { $params = [ 'deleted' => [0, 1], @@ -489,23 +541,16 @@ class Server extends AppModel return false; } - if (!empty($event)) { - if ($this->__checkIfEventIsBlockedBeforePull($event)) { - return false; + $pullRulesEmptiedEvent = false; + $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent); + + if (!$this->__checkIfEventSaveAble($event)) { + if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure + $fails[$eventId] = __('Empty event detected.'); } - $pullRulesEmptiedEvent = false; - $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent); - if (!$this->__checkIfEventSaveAble($event)) { - if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure - $fails[$eventId] = __('Empty event detected.'); - } - } else { - $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); - } - } else { - // error - $fails[$eventId] = __('failed downloading the event'); + return false; } + $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); return true; } @@ -551,7 +596,7 @@ class Server extends AppModel if ($jobId) { $job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.')); } - $pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique); + $pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $serverSync, $technique); if ($technique === 'pull_relevant_clusters') { if ($jobId) { $job->saveStatus($jobId, true, 'Pulling complete.'); @@ -579,8 +624,8 @@ class Server extends AppModel /** @var Event $eventModel */ $eventModel = ClassRegistry::init('Event'); - $successes = array(); - $fails = array(); + $successes = []; + $fails = []; // now process the $eventIds to pull each of the events sequentially if (!empty($eventIds)) { // download each event @@ -622,7 +667,7 @@ class Server extends AppModel count($fails) ); $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change); - return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters); + return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters]; } public function filterRuleToParameter($filter_rules) @@ -662,30 +707,20 @@ class Server extends AppModel /** * fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions * - * @param array $server - * @param HttpSocketExtended|null $HttpSocket + * @param ServerSyncTool $serverSync * @param array $conditions * @return array The list of clusters * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException */ - private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array()) + private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions = []) { - $url = $server['Server']['url']; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $uri = $url . '/galaxy_clusters/restSearch'; $filterRules = [ 'published' => 1, 'minimal' => 1, 'custom' => 1, ]; $filterRules = array_merge($filterRules, $conditions); - $response = $HttpSocket->post($uri, json_encode($filterRules), $request); - if (!$response->isOk()) { - throw new HttpSocketHttpException($response); - } - - $clusterArray = $response->json(); + $clusterArray = $serverSync->galaxyClusterSearch($filterRules)->json(); if (isset($clusterArray['response'])) { $clusterArray = $clusterArray['response']; } @@ -705,9 +740,9 @@ class Server extends AppModel * @throws HttpSocketJsonException * @throws JsonException */ - public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array()) + public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSyncTool, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array()) { - $clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions); + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSyncTool, $conditions=$conditions); if (!empty($clusterArray)) { foreach ($clusterArray as $cluster) { if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) { @@ -730,8 +765,7 @@ class Server extends AppModel /** * Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed. - * @param array $server - * @param HttpSocket|null $HttpSocket + * @param ServerSyncTool $serverSync * @param array $localClusters * @param array $conditions * @return array @@ -739,9 +773,9 @@ class Server extends AppModel * @throws HttpSocketJsonException * @throws JsonException */ - public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array()) + private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array()) { - $clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions); + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions); $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version'); if (!empty($localClusters)) { foreach ($localClusters as $k => $localCluster) { @@ -784,13 +818,35 @@ class Server extends AppModel return $eventIndex; } + /** + * @param array $events + * @return void + */ + private function removeOlderEvents(array &$events) + { + $conditions = (count($events) > 10000) ? [] : ['Event.uuid' => array_column($events, 'uuid')]; + $this->Event = ClassRegistry::init('Event'); + $localEvents = $this->Event->find('all', [ + 'recursive' => -1, + 'conditions' => $conditions, + 'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'], + ]); + $localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid'); + foreach ($events as $k => $event) { + $uuid = $event['uuid']; + if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) { + unset($events[$k]); + } + } + } + /** * Get an array of event UUIDs that are present on the remote server. * * @param ServerSyncTool $serverSync * @param bool $all - * @param bool $ignoreFilterRules - * @param bool $force + * @param bool $ignoreFilterRules Ignore defined server pull rules + * @param bool $force If true, returns all events regardless their update timestamp * @return array Array of event UUIDs. * @throws HttpSocketHttpException * @throws HttpSocketJsonException @@ -820,8 +876,7 @@ class Server extends AppModel } } if (!$force) { - $this->Event = ClassRegistry::init('Event'); - $this->Event->removeOlder($eventArray); + $this->removeOlderEvents($eventArray); } return array_column($eventArray, 'uuid'); } @@ -940,7 +995,7 @@ class Server extends AppModel // sync custom galaxy clusters if user is capable if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" == $technique) { - $clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique='full'); + $clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique='full'); } else { $clustersSuccesses = array(); } @@ -989,7 +1044,7 @@ class Server extends AppModel 'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names ); $eventIds = $this->Event->find('all', $findParams); - $eventUUIDsFiltered = $this->getEventIdsForPush($server, $HttpSocket, $eventIds); + $eventUUIDsFiltered = $this->getEventIdsForPush($server, $serverSync, $eventIds); if (!empty($eventUUIDsFiltered)) { $eventCount = count($eventUUIDsFiltered); // now process the $eventIds to push each of the events sequentially @@ -1018,11 +1073,11 @@ class Server extends AppModel $event = $event[0]; $event['Event']['locked'] = 1; if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" != $technique) { - $clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique=$event['Event']['id'], $event=$event); + $clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique=$event['Event']['id'], $event=$event); } else { $clustersSuccesses = array(); } - $result = $this->Event->uploadEventToServer($event, $server, $HttpSocket); + $result = $this->Event->uploadEventToServer($event, $server, $serverSync); if ('Success' === $result) { $successes[] = $event['Event']['id']; } else { @@ -1049,7 +1104,7 @@ class Server extends AppModel if ($push['canPush'] || $push['canSight']) { $this->Sighting = ClassRegistry::init('Sighting'); - $sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync); + $sightingSuccesses = $this->Sighting->pushSightings($user, $serverSync); } else { $sightingSuccesses = array(); } @@ -1083,23 +1138,35 @@ class Server extends AppModel return true; } - public function getEventIdsForPush(array $server, HttpSocket $HttpSocket, array $eventIds) + /** + * @param array $server + * @param ServerSyncTool $serverSync + * @param array $events + * @return array|false + */ + private function getEventIdsForPush(array $server, ServerSyncTool $serverSync, array $events) { - foreach ($eventIds as $k => $event) { - if (empty($this->eventFilterPushableServers($event, array($server)))) { - unset($eventIds[$k]); + $request = []; + foreach ($events as $event) { + if (empty($this->eventFilterPushableServers($event, [$server]))) { continue; } - unset($eventIds[$k]['Event']['id']); + $request[] = ['Event' => [ + 'uuid' => $event['Event']['uuid'], + 'timestamp' => $event['Event']['timestamp'], + ]]; } - $request = $this->setupSyncRequest($server); - $data = json_encode($eventIds); - $uri = $server['Server']['url'] . '/events/filterEventIdsForPush'; - $response = $HttpSocket->post($uri, $data, $request); - if ($response->code == '200') { - return $this->jsonDecode($response->body()); + + if (empty($request)) { + return []; + } + + try { + return $serverSync->filterEventIdsForPush($request)->json(); + } catch (Exception $e) { + $this->logException("Could not filter events for push when pushing to server {$serverSync->serverId()}", $e); + return false; } - return false; } /** @@ -1112,7 +1179,7 @@ class Server extends AppModel * @param array|bool $event * @return array List of successfully pushed clusters */ - public function syncGalaxyClusters($HttpSocket, array $server, array $user, $technique='full', $event=false) + public function syncGalaxyClusters(ServerSyncTool $serverSync, array $server, array $user, $technique='full', $event=false) { $successes = array(); if (!$server['Server']['push_galaxy_clusters']) { @@ -1120,7 +1187,6 @@ class Server extends AppModel } $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); $this->Event = ClassRegistry::init('Event'); - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $clusters = array(); if ($technique == 'full') { $clusters = $this->GalaxyCluster->getElligibleClustersToPush($user, $conditions=array(), $full=true); @@ -1135,13 +1201,13 @@ class Server extends AppModel } $localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid'); try { - $clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket = $HttpSocket, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs)); + $clustersToPush = $this->getElligibleClusterIdsFromServerForPush($serverSync, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs)); } catch (Exception $e) { $this->logException("Could not get eligible cluster IDs from server #{$server['Server']['id']} for push.", $e); return []; } foreach ($clustersToPush as $cluster) { - $result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user); + $result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $serverSync, $user); if ($result === 'Success') { $successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']); } @@ -1153,10 +1219,10 @@ class Server extends AppModel { $saModel = ClassRegistry::init('ShadowAttribute'); $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); if ($sa_id == null) { if ($event_id == null) { // event_id is null when we are doing a push + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); try { $ids = $this->getEventIdsFromServer($serverSync, true, true); } catch (Exception $e) { @@ -1226,7 +1292,7 @@ class Server extends AppModel public function getCurrentServerSettings() { $serverSettings = $this->serverSettings; - $moduleTypes = array('Enrichment', 'Import', 'Export', 'Cortex'); + $moduleTypes = array('Enrichment', 'Import', 'Export', 'Action', 'Cortex'); return $this->readModuleSettings($serverSettings, $moduleTypes); } @@ -1248,6 +1314,15 @@ class Server extends AppModel $setting['test'] = 'testBool'; $setting['type'] = 'boolean'; $setting['description'] = __('Enable or disable the %s module.', $module); + if (!empty($result['description'])) { + $setting['description'] = sprintf( + "[%s%s%s] %s", + '', + $setting['description'], + '', + $result['description'] + ); + } $setting['value'] = false; } elseif ($result['type'] === 'orgs') { $setting['description'] = __('Restrict the %s module to the given organisation.', $module); @@ -1258,15 +1333,29 @@ class Server extends AppModel return $this->loadLocalOrganisations(); }; } else { - $setting['test'] = 'testForEmpty'; - $setting['type'] = 'string'; + $setting['test'] = isset($result['test']) ? $result['test'] : 'testForEmpty'; + $setting['type'] = isset($result['type']) ? $result['type'] : 'string'; $setting['description'] = isset($result['description']) ? $result['description'] : __('Set this required module specific setting.'); - $setting['value'] = ''; + $setting['value'] = isset($result['value']) ? $result['value'] : ''; } $serverSettings['Plugin'][$moduleType . '_' . $module . '_' . $result['name']] = $setting; } } } + if (Configure::read('Plugin.Workflow_enable')) { + $this->Workflow = ClassRegistry::init('Workflow'); + $triggerModules = $this->Workflow->getModulesByType('trigger'); + foreach ($triggerModules as $triggerModule) { + $setting = [ + 'level' => 1, + 'description' => __('Enable/disable the `%s` trigger', $triggerModule['id']), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ]; + $serverSettings['Plugin']['Workflow_triggers_' . $triggerModule['id']] = $setting; + } + } } return $serverSettings; } @@ -1514,6 +1603,17 @@ class Server extends AppModel } } + public function testForCorrelationEngine($value) + { + $defaults = $this->generateServerSettings(); + $options = $defaults['MISP']['correlation_engine']['options']; + if (!empty($value) && !in_array($value, array_keys($options))) { + return __('Please select a valid option from the list of available engines: ', implode(', ', array_keys($options))); + } else { + return true; + } + } + public function testLocalOrg($value) { if ($value == 0) { @@ -2095,7 +2195,7 @@ class Server extends AppModel // This is just hack to reset opcache, so for next request cache will be reloaded. $this->opcacheResetConfig(); - if (strpos($settingName, 'Plugin.Enrichment') !== false || strpos($settingName, 'Plugin.Import') !== false || strpos($settingName, 'Plugin.Export') !== false || strpos($settingName, 'Plugin.Cortex') !== false) { + if (strpos($settingName, 'Plugin.Enrichment') !== false || strpos($settingName, 'Plugin.Import') !== false || strpos($settingName, 'Plugin.Export') !== false || strpos($settingName, 'Plugin.Cortex') !== false || strpos($settingName, 'Plugin.Action') !== false || strpos($settingName, 'Plugin.Workflow') !== false) { $serverSettings = $this->getCurrentServerSettings(); } else { $serverSettings = $this->serverSettings; @@ -2503,7 +2603,6 @@ class Server extends AppModel $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; - $supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']); $canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false; $remoteVersionString = $remoteVersion['version']; $remoteVersion = explode('.', $remoteVersion['version']); @@ -2555,7 +2654,6 @@ class Server extends AppModel 'canPush' => $canPush, 'canSight' => $canSight, 'canEditGalaxyCluster' => $canEditGalaxyCluster, - 'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster, 'version' => $remoteVersion, 'protectedMode' => $protectedMode, ]; @@ -2706,6 +2804,44 @@ class Server extends AppModel return $schemaDiagnostic; } + /* + * Get RDBMS configuration values + */ + public function dbConfiguration(): array + { + if ($this->isMysql()) { + $configuration = []; + + $dbVariables = $this->query("SHOW VARIABLES;"); + $settings = array_keys(self::MYSQL_RECOMMENDED_SETTINGS); + + foreach ($dbVariables as $dbVariable) { + // different rdbms have different casing + if (isset($dbVariable['SESSION_VARIABLES'])) { + $dbVariable = $dbVariable['SESSION_VARIABLES']; + } elseif (isset($dbVariable['session_variables'])) { + $dbVariable = $dbVariable['session_variables']; + } else { + continue; + } + + if (in_array($dbVariable['Variable_name'], $settings)) { + $configuration[] = [ + 'name' => $dbVariable['Variable_name'], + 'value' => $dbVariable['Value'], + 'default' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['default'], + 'recommended' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['recommended'], + 'explanation' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['explanation'], + ]; + } + } + + return $configuration; + } else { + return []; + } + } + /* * Work in progress, still needs DEFAULT in the schema for it to work correctly * Currently only works for missing_column and column_different @@ -3024,13 +3160,6 @@ class Server extends AppModel $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes foreach ($tableIndexDiff as $columnDiff) { $shouldBeUnique = $indexes[$columnDiff]; - if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) { - $indexDiff[$tableName][$columnDiff] = array( - 'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff), - 'sql' => '', - ); - continue; - } $message = __('Column `%s` should be indexed', $columnDiff); $indexDiff[$tableName][$columnDiff] = array( @@ -3058,15 +3187,6 @@ class Server extends AppModel 'sql' => $sql, ); } else { - if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) { - $message = __('Column `%s` should be unique index, but contains duplicate values', $column); - $indexDiff[$tableName][$column] = array( - 'message' => $message, - 'sql' => '', - ); - continue; - } - $sql = $this->generateSqlDropIndexQuery($tableName, $column); $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true); @@ -3309,7 +3429,15 @@ class Server extends AppModel return 1; } $pubSubTool = $this->getPubSubTool(); - if (!$pubSubTool->checkIfPythonLibInstalled()) { + try { + $isInstalled = $pubSubTool->checkIfPythonLibInstalled(); + } catch (Exception $e) { + $this->logException('ZMQ is not properly installed.', $e, LOG_NOTICE); + $diagnostic_errors++; + return 2; + } + + if (!$isInstalled) { $diagnostic_errors++; return 2; } @@ -3391,6 +3519,7 @@ class Server extends AppModel $sqlResult = $this->query($sql); if (isset($sqlResult[0][0])) { $sessionCount = $sqlResult[0][0]['session_count']; + $errorCode = 0; } else { $errorCode = 9; } @@ -3762,8 +3891,7 @@ class Server extends AppModel public function extensionDiagnostics() { try { - $file = new File(APP . DS . 'composer.json'); - $composer = $this->jsonDecode($file->read()); + $composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json'); $extensions = []; foreach ($composer['require'] as $require => $foo) { if (substr($require, 0, 4) === 'ext-') { @@ -4738,14 +4866,34 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true ], - 'enable_advanced_correlations' => array( + 'correlation_engine' => [ + 'level' => 0, + 'description' => __('Choose which correlation engine to use. MISP defaults to the default engine, maintaining all data in the database whilst enforcing ACL rules on any non site-admin user. This is recommended for any MISP instnace with multiple organisations. If you are an endpoint MISP, consider switching to the much leaner and faster No ACL engine.'), + 'value' => 'default', + 'test' => 'testForCorrelationEngine', + 'type' => 'string', + 'null' => true, + 'options' => [ + 'Default' => __('Default Correlation Engine'), + 'NoAcl' => __('No ACL Engine') + ], + ], + 'correlation_limit' => [ + 'level' => 0, + 'description' => __('Set a value for the maximum number of correlations a value should have before MISP will refuse to correlate it (extremely over-correlating values are rarely useful from a correlation perspective).'), + 'value' => 100, + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ], + 'enable_advanced_correlations' => [ 'level' => 0, 'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'), 'value' => false, 'test' => 'testBool', 'type' => 'boolean', 'null' => true - ), + ], 'server_settings_skip_backup_rotate' => array( 'level' => 1, 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), @@ -6973,6 +7121,34 @@ class Server extends AppModel 'test' => 'testForEmpty', 'type' => 'numeric' ), + 'Action_services_url' => array( + 'level' => 1, + 'description' => __('The url used to access the action services. By default, it is accessible at http://127.0.0.1:6666'), + 'value' => 'http://127.0.0.1', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Action_services_port' => array( + 'level' => 1, + 'description' => __('The port used to access the action services. By default, it is accessible at 127.0.0.1:6666'), + 'value' => '6666', + 'test' => 'testForPortNumber', + 'type' => 'numeric' + ), + 'Action_services_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the action services'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Action_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the action services'), + 'value' => 10, + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), 'Enrichment_hover_enable' => array( 'level' => 0, 'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'), @@ -7008,6 +7184,20 @@ class Server extends AppModel 'test' => 'testForPortNumber', 'type' => 'numeric' ), + 'Workflow_enable' => array( + 'level' => 1, + 'description' => __('Enable/disable workflow feature. [experimental]'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Workflow_debug_url' => array( + 'level' => 1, + 'description' => __('Set the debug URL where info about workflow execution will be POSTed'), + 'value' => 'http://127.0.0.1:27051', + 'test' => 'testForEmpty', + 'type' => 'string' + ), 'Cortex_services_url' => array( 'level' => 1, 'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'), @@ -7273,6 +7463,7 @@ class Server extends AppModel 'Get IPs for user ID' => 'MISP/app/Console/cake Admin UserIP [user_id]', 'Get user ID for user IP' => 'MISP/app/Console/cake Admin IPUser [ip]', 'Generate correlation' => 'MISP/app/Console/cake Admin jobGenerateCorrelation [job_id]', + 'Truncate correlation table' => 'MISP/app/Console/cake Admin truncateTable [user_id] [correlation_engine_name] [job_id]', 'Purge correlation' => 'MISP/app/Console/cake Admin jobPurgeCorrelation [job_id]', 'Generate shadow attribute correlation' => 'MISP/app/Console/cake Admin jobGenerateShadowAttributeCorrelation [job_id]', 'Update MISP' => 'MISP/app/Console/cake Admin updateMISP', diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 9d2a67fcd..3c9e1ad2a 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -190,7 +190,8 @@ class ShadowAttribute extends AppModel $this->data['ShadowAttribute']['deleted'] = 0; } if ($this->data['ShadowAttribute']['deleted']) { - $this->__beforeDeleteCorrelation($this->data['ShadowAttribute']); + // correlations for proposals are deprecated. + //$this->__beforeDeleteCorrelation($this->data['ShadowAttribute']); } // convert into utc and micro sec @@ -277,12 +278,15 @@ class ShadowAttribute extends AppModel $result = $result && $this->saveBase64EncodedAttachment($this->data['ShadowAttribute']); } } + /* + * correlations are deprecated for proposals if ((isset($this->data['ShadowAttribute']['deleted']) && $this->data['ShadowAttribute']['deleted']) || (isset($this->data['ShadowAttribute']['proposal_to_delete']) && $this->data['ShadowAttribute']['proposal_to_delete'])) { // this is a deletion // Could be a proposal to delete or flagging a proposal that it was discarded / accepted - either way, we don't want to correlate here for now } else { $this->__afterSaveCorrelation($this->data['ShadowAttribute']); } + */ if (empty($this->data['ShadowAttribute']['deleted'])) { $action = $created ? 'add' : 'edit'; $this->publishKafkaNotification('shadow_attribute', $this->data, $action); diff --git a/app/Model/Tag.php b/app/Model/Tag.php index a1b6763fd..b58fd7c75 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -583,7 +583,8 @@ class Tag extends AppModel $changedTags = $this->AttributeTag->getAffectedRows(); $this->EventTag->updateAll(['tag_id' => $destinationTag['Tag']['id']], ['tag_id' => $sourceTag['Tag']['id']]); $changedTags += $this->EventTag->getAffectedRows(); - + $this->GalaxyClusterRelationTag->updateAll(['tag_id' => $destinationTag['Tag']['id']], ['tag_id' => $sourceTag['Tag']['id']]); + $changedTags += $this->GalaxyClusterRelationTag->getAffectedRows(); $this->delete($sourceTag['Tag']['id']); return [ diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 24f29d0c9..97c40c57a 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -770,4 +770,77 @@ class Taxonomy extends AppModel } return $splits; } + + private function __craftTaxonomiesTags() + { + $taxonomies = $this->find('all', [ + 'fields' => ['namespace'], + 'contain' => ['TaxonomyPredicate' => ['TaxonomyEntry']], + ]); + $allTaxonomyTags = []; + foreach ($taxonomies as $taxonomy) { + $namespace = $taxonomy['Taxonomy']['namespace']; + foreach ($taxonomy['TaxonomyPredicate'] as $predicate) { + if (isset($predicate['TaxonomyEntry']) && !empty($predicate['TaxonomyEntry'])) { + foreach ($predicate['TaxonomyEntry'] as $entry) { + $tag = $namespace . ':' . $predicate['value'] . '="' . $entry['value'] . '"'; + $allTaxonomyTags[$tag] = true; + } + } else { + $tag = $namespace . ':' . $predicate['value']; + $allTaxonomyTags[$tag] = true; + } + } + } + return $allTaxonomyTags; + } + + /** + * normalizeCustomTagsToTaxonomyFormat Transform all custom tags into their taxonomy version. + * + * @return int The number of converted tag + */ + public function normalizeCustomTagsToTaxonomyFormat(): array + { + $tagConverted = 0; + $rowUpdated = 0; + $craftedTags = $this->__craftTaxonomiesTags(); + $allTaxonomyTagsByName = Hash::combine($this->getAllTaxonomyTags(false, false, true, false, true), '{n}.Tag.name', '{n}.Tag.id'); + $tagsToMigrate = array_diff_key($allTaxonomyTagsByName, $craftedTags); + foreach ($tagsToMigrate as $tagToMigrate_name => $tagToMigrate_id) { + foreach (array_keys($craftedTags) as $craftedTag) { + if (strcasecmp($craftedTag, $tagToMigrate_name) == 0) { + $result = $this->__updateTagToNormalized(intval($tagToMigrate_id), intval($allTaxonomyTagsByName[$craftedTag])); + $tagConverted += 1; + $rowUpdated += $result['changed']; + } + } + } + return [ + 'tag_converted' => $tagConverted, + 'row_updated' => $rowUpdated, + ]; + } + + /** + * __updateTagToNormalized Change the link of element having $source_id tag attached to them for the $target_id one. + * Updated: + * - event_tags + * - attribute_tags + * - galaxy_cluster_relation_tags + * + * Ignored: As this is defined by users, let them do the migration themselves + * - tag_collection_tags + * - template_tags + * - favorite_tags + * + * @param int $source_id + * @param int $target_id + * @return array + * @throws Exception + */ + private function __updateTagToNormalized($source_id, $target_id): array + { + return $this->Tag->mergeTag($source_id, $target_id); + } } diff --git a/app/Model/User.php b/app/Model/User.php index a313ba05f..0dab0b0cb 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -269,6 +269,27 @@ class User extends AppModel $passwordHasher = new BlowfishConstantPasswordHasher(); $this->data[$this->alias]['password'] = $passwordHasher->hash($this->data[$this->alias]['password']); } + $user = $this->data; + $action = empty($this->id) ? 'add' : 'edit'; + $user_id = $action == 'add' ? 0 : $user['User']['id']; + $trigger_id = 'user-before-save'; + $workflowErrors = []; + $logging = [ + 'model' => 'User', + 'action' => $action, + 'id' => $user_id, + 'message' => __('The workflow `%s` prevented the saving of user %s', $trigger_id, $user_id), + ]; + if ( + empty($user['User']['action']) || + ( + $user['User']['action'] != 'logout' && + $user['User']['action'] != 'login' + ) + ) { + $success = $this->executeTrigger($trigger_id, $user['User'], $workflowErrors, $logging); + return !empty($success); + } return true; } @@ -276,6 +297,23 @@ class User extends AppModel { $pubToZmq = $this->pubToZmq('user'); $kafkaTopic = $this->kafkaTopic('user'); + $action = empty($created) ? 'edit' : 'add'; + $user = $this->data; + if ( + empty($user['User']['action']) || + ( + $user['User']['action'] != 'logout' && + $user['User']['action'] != 'login' + ) + ) { + $workflowErrors = []; + $logging = [ + 'model' => 'User', + 'action' => $action, + 'id' => $user['User']['id'], + ]; + $this->executeTrigger('user-after-save', $user['User'], $workflowErrors, $logging); + } if ($pubToZmq || $kafkaTopic) { if (!empty($this->data)) { $user = $this->data; @@ -1581,4 +1619,14 @@ class User extends AppModel 'conditions' => array('EventDelegation.org_id' => $user['org_id']) )); } + + /** + * Generate code that is used in event alert unsubscribe link. + * @return string + */ + public function unsubscribeCode(array $user) + { + $salt = Configure::read('Security.salt'); + return substr(hash('sha256', "{$user['id']}|$salt"), 0, 8); + } } diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php new file mode 100644 index 000000000..b4c03eece --- /dev/null +++ b/app/Model/Workflow.php @@ -0,0 +1,1352 @@ + [ + 'roleModel' => 'Role', + 'roleKey' => 'role_id', + 'change' => 'full' + ], + ]; + + public $belongsTo = [ + ]; + + public $validate = [ + 'value' => [ + 'stringNotEmpty' => [ + 'rule' => ['stringNotEmpty'] + ] + ], + 'uuid' => [ + 'uuid' => [ + 'rule' => 'uuid', + 'message' => 'Please provide a valid RFC 4122 UUID' + ], + 'unique' => [ + 'rule' => 'isUnique', + 'message' => 'The UUID provided is not unique', + 'required' => 'create' + ] + ], + 'data' => [ + 'hasAcyclicGraph' => [ + 'rule' => ['hasAcyclicGraph'], + 'message' => 'Cannot save a workflow containing a cycle', + ], + 'hasOneTrigger' => [ + 'rule' => ['hasOneTrigger'], + 'message' => 'Cannot save a workflow containing no or more than one trigger', + ], + 'satisfiesMultipleConnectionCondition' => [ + 'rule' => ['satisfiesMultipleConnectionCondition'], + 'message' => 'Cannot save a workflow having more than one connection per output', + ] + ] + ]; + + /** @var WorkflowGraphTool */ + public $workflowGraphTool; + public $defaultContain = [ + ]; + + private $loaded_modules = []; + private $loaded_classes = []; + private $error_while_loading = []; + + private $module_initialized = false; + private $modules_enabled_by_default = ['generic-if', 'distribution-if', 'published-if', 'organisation-if', 'tag-if', 'concurrent-task', 'stop-execution', 'webhook', 'push-zmq']; + + const CAPTURE_FIELDS_EDIT = ['name', 'description', 'timestamp', 'data', 'debug_enabled']; + const CAPTURE_FIELDS_ADD = ['uuid', 'name', 'description', 'timestamp', 'data', 'trigger_id', 'debug_enabled']; + + const MODULE_ROOT_PATH = APP . 'Model/WorkflowModules/'; + const CUSTOM_MODULE_ROOT_PATH = APP . 'Lib/WorkflowModules/'; + const REDIS_KEY_WORKFLOW_NAMESPACE = 'workflow'; + const REDIS_KEY_WORKFLOW_PER_TRIGGER = 'workflow:workflow_list:%s'; + const REDIS_KEY_TRIGGER_PER_WORKFLOW = 'workflow:trigger_list:%s'; + const REDIS_KEY_MODULES_ENABLED = 'workflow:modules_enabled'; + + public function __construct($id = false, $table = null, $ds = null) + { + parent::__construct($id, $table, $ds); + $this->workflowGraphTool = new WorkflowGraphTool(); + } + + public function beforeValidate($options = array()) + { + parent::beforeValidate(); + if (empty($this->data['Workflow']['uuid'])) { + $this->data['Workflow']['uuid'] = CakeText::uuid(); + } else { + $this->data['Workflow']['uuid'] = strtolower($this->data['Workflow']['uuid']); + } + if (empty($this->data['Workflow']['data'])) { + $this->data['Workflow']['data'] = []; + } + if (empty($this->data['Workflow']['timestamp'])) { + $this->data['Workflow']['timestamp'] = time(); + } + if (empty($this->data['Workflow']['description'])) { + $this->data['Workflow']['description'] = ''; + } + return true; + } + + public function afterFind($results, $primary = false) + { + foreach ($results as $k => $result) { + if (empty($result['Workflow']['data'])) { + $result['Workflow']['data'] = '{}'; + } + $results[$k]['Workflow']['data'] = JsonTool::decode($result['Workflow']['data']); + if (!empty($result['Workflow']['id'])) { + $trigger_ids = $this->__getTriggersIDPerWorkflow((int) $result['Workflow']['id']); + $results[$k]['Workflow']['listening_triggers'] = $this->getModuleByID($trigger_ids); + } + } + return $results; + } + + public function beforeSave($options = []) + { + if (is_array($this->data['Workflow']['data'])) { + $this->data['Workflow']['data'] = JsonTool::encode($this->data['Workflow']['data']); + } + return true; + } + + public function afterSave($created, $options = []) + { + $this->updateListeningTriggers($this->data); + } + + public function beforeDelete($cascade = true) + { + parent::beforeDelete($cascade); + $workflow = $this->find('first', [ // $this->data is empty in afterDelete?! + 'recursive' => -1, + 'conditions' => ['Workflow.id' => $this->id] + ]); + $workflow['Workflow']['data'] = []; // Make sure not trigger are listening + $this->workflowToDelete = $workflow; + } + + public function afterDelete() + { + // $this->data is empty?! + parent::afterDelete(); + $this->updateListeningTriggers($this->workflowToDelete); + } + + public function enableDefaultModules() + { + $this->toggleModules($this->modules_enabled_by_default, true, false); + } + + protected function checkTriggerEnabled($trigger_id) + { + $settingName = sprintf('Plugin.Workflow_triggers_%s', $trigger_id); + $module_disabled = empty(Configure::read($settingName)); + if ($module_disabled) { + return false; + } + + $filename = sprintf('Module_%s.php', preg_replace('/[^a-zA-Z0-9_]/', '_', Inflector::underscore($trigger_id))); + $module_config = $this->__getClassFromModuleFiles('trigger', [$filename], false)['classConfigs']; + return empty($module_config['disabled']); + } + + protected function getEnabledModules(): array + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + $list = $redis->sMembers(Workflow::REDIS_KEY_MODULES_ENABLED); + return !empty($list) ? $list : []; + } + + public function toggleModule($module_id, $enable, $is_trigger=false): bool + { + if (!empty($is_trigger)) { + $settingName = sprintf('Plugin.Workflow_triggers_%s', $module_id); + $server = ClassRegistry::init('Server'); + return $server->serverSettingsSaveValue($settingName, !empty($enable), false); + } else { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + if ($enable) { + $redis->sAdd(Workflow::REDIS_KEY_MODULES_ENABLED, $module_id); + } else { + $redis->sRem(Workflow::REDIS_KEY_MODULES_ENABLED, $module_id); + } + } + return true; + } + + public function toggleDebug($workflow_id, $enable): bool + { + $workflow = $this->fetchWorkflow($workflow_id); + $workflow['Workflow']['debug_enabled'] = !empty($enable); + $result = $this->editWorkflow($workflow); + return empty($result['errrors']); + } + + public function toggleModules($module_ids, $enable, $is_trigger=false): int + { + $enabled_count = 0; + foreach ($module_ids as $module_id) { + $enabled_count += $this->toggleModule($module_id, $enable, $is_trigger) ? 1 : 0; + } + return $enabled_count; + } + + protected function checkTriggerListenedTo($trigger_id) + { + return !empty($this->__getWorkflowsIDPerTrigger($trigger_id)); + } + + public function rebuildRedis() + { + $redis = $this->setupRedisWithException(); + $workflows = $this->fetchWorkflows(); + $keys = $redis->keys(Workflow::REDIS_KEY_WORKFLOW_NAMESPACE . ':*'); + $redis->delete($keys); + foreach ($workflows as $wokflow) { + $this->updateListeningTriggers($wokflow); + } + } + + /** + * updateListeningTriggers + * - Update the list of triggers that will be run this workflow + * - Update the list of workflows that are run by their triggers + * - Update the ordered list of workflows that are run by their triggers + * + * @param array $workflow + */ + public function updateListeningTriggers($workflow) + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + $this->logException('Failed to setup redis ', $e); + return false; + } + if (!is_array($workflow['Workflow']['data'])) { + $workflow['Workflow']['data'] = JsonTool::decode($workflow['Workflow']['data']); + } + $original_trigger_list_id = $this->__getTriggersIDPerWorkflow((int)$workflow['Workflow']['id']); + $new_node_trigger_list = $this->workflowGraphTool->extractTriggersFromWorkflow($workflow['Workflow']['data'], true); + $new_node_trigger_list_per_id = Hash::combine($new_node_trigger_list, '{n}.data.id', '{n}'); + $new_trigger_list_id = array_keys($new_node_trigger_list_per_id); + $trigger_to_remove = array_diff($original_trigger_list_id, $new_trigger_list_id); + $trigger_to_add = array_diff($new_trigger_list_id, $original_trigger_list_id); + if (!empty($trigger_to_remove)) { + $pipeline = $redis->multi(); + foreach ($trigger_to_remove as $trigger_id) { + $pipeline->sRem(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_id), $workflow['Workflow']['id']); + $pipeline->sRem(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow['Workflow']['id']), $trigger_id); + } + $pipeline->exec(); + } + if (!empty($trigger_to_add)) { + $pipeline = $redis->multi(); + foreach ($trigger_to_add as $trigger_id) { + $pipeline->sAdd(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_id), $workflow['Workflow']['id']); + $pipeline->sAdd(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow['Workflow']['id']), $trigger_id); + } + $pipeline->exec(); + } + } + + /** + * __getWorkflowsIDPerTrigger Get list of workflow IDs listening to the specified trigger + * + * @param string $trigger_id + * @return bool|array + */ + private function __getWorkflowsIDPerTrigger($trigger_id): array + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + $list = $redis->sMembers(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_id)); + return !empty($list) ? $list : []; + } + + /** + * __getTriggersIDPerWorkflow Get list of trigger name running to the specified workflow + * + * @param int $workflow_id + * @return bool|array + */ + private function __getTriggersIDPerWorkflow(int $workflow_id) + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + return $redis->sMembers(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow_id)); + } + + public function getListeningWorkflowForTrigger(array $trigger): array + { + return array_map(function($id) { + return intval($id); + }, $this->__getWorkflowsIDPerTrigger($trigger['id'])); + } + + /** + * attachWorkflowToTriggers Collect the workflows listening to this trigger + * + * @param array $triggers + * @return array + */ + public function attachWorkflowToTriggers(array $triggers): array + { + $workflows = $this->fetchWorkflows([ + 'conditions' => [ + 'Workflow.trigger_id' => Hash::extract($triggers, '{n}.id'), + ], + 'fields' => ['*'], + ]); + $workflows_per_trigger = Hash::combine($workflows, '{n}.Workflow.trigger_id', '{n}'); + foreach ($triggers as $i => $trigger) { + if (!empty($workflows_per_trigger[$trigger['id']])) { + $triggers[$i]['Workflow'] = $workflows_per_trigger[$trigger['id']]['Workflow']; + } + } + return $triggers; + } + + /** + * hasAcyclicGraph Return if the graph is acyclic or not + * + * @param array $graphData + * @return boolean + */ + public function hasAcyclicGraph(array $workflow): bool + { + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $isAcyclic = $this->workflowGraphTool->isAcyclic($graphData); + return $isAcyclic; + } + + /** + * hasOneTrigger Return if the graph contain more than one instance of the same trigger + * + * @param array $graphData + * @return boolean + */ + public function hasOneTrigger(array $workflow): bool + { + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $triggers = $this->workflowGraphTool->extractTriggersFromWorkflow($graphData, true); + return count($triggers) == 1; + } + + /** + * satisfiesMultipleConnectionCondition Return if the graph contain more than one instance of the same trigger + * + * @param array $graphData + * @return boolean + */ + public function satisfiesMultipleConnectionCondition(array $workflow): bool + { + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $hasMultipleOutputConnection = $this->workflowGraphTool->hasMultipleOutputConnection($graphData); + return !$hasMultipleOutputConnection; + } + + /** + * executeWorkflow + * + * @param int $workflow_id + * @param array $data + * @param array $blockingErrors + * @return array + */ + public function executeWorkflow($workflow_id, array $data, array &$blockingErrors=[]): array + { + $this->loadAllWorkflowModules(); + + $workflow = $this->fetchWorkflow($workflow_id, true); + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $startNode = $this->workflowGraphTool->extractTriggerFromWorkflow($graphData, true); + $startNodeID = $startNode['id']; + $trigger_id = $startNode['data']['id']; + if ($startNode == -1) { + $blockingErrors[] = __('Invalid start node `%s`', $startNodeID); + return false; + } + + $triggerModule = $this->getModuleClassByType('trigger', $trigger_id, true); + if (!empty($triggerModule->disabled)) { + return true; + } + $result = $this->__runWorkflow($workflow, $triggerModule, $data, $startNodeID, $blockingErrors); + return $result; + } + + /** + * executeWorkflowForTrigger + * + * @param string $trigger_id + * @param array $data + * @throws TriggerNotFoundException + */ + public function executeWorkflowForTriggerRouter($trigger_id, array $data, array &$blockingErrors=[], array $logging=[]): bool + { + $this->loadAllWorkflowModules(); + + if (empty($this->loaded_modules['trigger'][$trigger_id])) { + throw new TriggerNotFoundException(__('Unknown trigger `%s`', $trigger_id)); + } + $trigger = $this->loaded_modules['trigger'][$trigger_id]; + if (!empty($trigger['disabled'])) { + return true; + } + + if (empty($trigger['blocking'])) { + $this->Job = ClassRegistry::init('Job'); + $jobId = $this->Job->createJob( + 'SYSTEM', + Job::WORKER_PRIO, + 'executeWorkflowForTrigger', + sprintf('Workflow for trigger `%s`', $trigger_id), + __('Executing non-blocking workflow for trigger `%s`', $trigger_id) + ); + $this->Job->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_WORKFLOW, + [ + 'executeWorkflowForTrigger', + $trigger_id, + JsonTool::encode($data), + JsonTool::encode($logging), + $jobId + ], + true, + $jobId + ); + return true; + } else { + $blockingPathExecutionSuccess = $this->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors); + return $blockingPathExecutionSuccess; + } + } + + /** + * executeWorkflowForTrigger + * + * @param string $trigger_id + * @param array $data + * @param array $errors + * @return boolean True if the execution for the blocking path was a success + * @throws TriggerNotFoundException + */ + public function executeWorkflowForTrigger($trigger_id, array $data, array &$blockingErrors=[]): bool + { + $this->loadAllWorkflowModules(); + + $triggerModule = $this->getModuleClassByType('trigger', $trigger_id, true); + if (!empty($triggerModule->disabled)) { + return true; + } + + $workflow = $this->fetchWorkflowByTrigger($trigger_id, true); + if (empty($workflow)) { + throw new WorkflowNotFoundException(__('Could not get workflow for trigger `%s`', $trigger_id)); + } + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $startNodeID = $this->workflowGraphTool->getNodeIdForTrigger($graphData, $trigger_id); + if ($startNodeID == -1) { + $blockingErrors[] = __('Invalid start node `%s`', $startNodeID); + return false; + } + $result = $this->__runWorkflow($workflow, $triggerModule, $data, $startNodeID, $blockingErrors); + return $result['success']; + } + + /** + * runWorkflow + * + * @param array $workflow + * @param $triggerModule + * @param array $data + * @param int $startNodeID + * @return array + */ + private function __runWorkflow(array $workflow, $triggerModule, array $data, $startNodeID, &$blockingErrors=[]): array + { + $this->Log = ClassRegistry::init('Log'); + $message = __('Started executing workflow for trigger `%s` (%s)', $triggerModule->id, $workflow['Workflow']['id']); + $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message); + $this->__logToFile($workflow, $message); + $workflow = $this->__incrementWorkflowExecutionCount($workflow); + $walkResult = []; + $data = $this->__normalizeDataForTrigger($triggerModule, $data); + $for_path = !empty($triggerModule->blocking) ? GraphWalker::PATH_TYPE_BLOCKING : GraphWalker::PATH_TYPE_NON_BLOCKING; + $this->sendRequestToDebugEndpoint($workflow, [], '/init?type=' . $for_path, $data); + + $blockingPathExecutionSuccess = $this->walkGraph($workflow, $startNodeID, $for_path, $data, $blockingErrors, $walkResult); + $executionStoppedByStopModule = in_array('stop-execution', Hash::extract($walkResult, 'blocking_nodes.{n}.data.id')); + if (empty($blockingPathExecutionSuccess)) { + $message = __('Execution stopped. %s', PHP_EOL . implode(', ', $blockingErrors)); + $this->logExecutionError($workflow, $message); + } + $outcomeText = 'failure'; + if (!empty($blockingPathExecutionSuccess)) { + $outcomeText = 'success'; + } else if ($executionStoppedByStopModule) { + $outcomeText = 'blocked'; + } + $message = __('Finished executing workflow for trigger `%s` (%s). Outcome: %s', $triggerModule->id, $workflow['Workflow']['id'], $outcomeText); + + $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message); + $this->__logToFile($workflow, $message); + $this->sendRequestToDebugEndpoint($workflow, [], '/end?outcome=' . $outcomeText, $walkResult); + return [ + 'outcomeText' => $outcomeText, + 'walkResult' => $walkResult, + 'success' => $blockingPathExecutionSuccess, + ]; + } + + /** + * walkGraph Walk the graph for the provided trigger and execute each nodes + * + * @param array $workflow The worflow to walk + * @param int $startNode The ID of the trigger to start from + * @param string|null $for_path If provided, execute the workflow for the provided path. If not provided, execute the worflow regardless of the path + * @param array $data + * @param array $errors + * @return boolean If all module returned a successful response + */ + public function walkGraph(array $workflow, int $startNode, $for_path=null, array $data=[], array &$errors=[], array &$walkResult=[]): bool + { + $walkResult = [ + 'blocking_nodes' => [], + 'executed_nodes' => [], + 'blocked_paths' => [], + ]; + $this->Organisation = ClassRegistry::init('Organisation'); + $hostOrg = $this->Organisation->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'id' => Configure::read('MISP.host_org_id') + ], + ]); + if (!empty($hostOrg)) { + $userForWorkflow = [ + 'email' => 'SYSTEM', + 'id' => 0, + 'org_id' => $hostOrg['Organisation']['id'], + 'Role' => ['perm_site_admin' => 1], + 'Organisation' => $hostOrg['Organisation'] + ]; + } else { + $this->User = ClassRegistry::init('User'); + $userForWorkflow = $this->User->find('first', [ + 'recursive' => -1, + 'conditions' => [ + 'Role.perm_site_admin' => 1, + 'User.disabled' => 0 + ], + 'contain' => [ + 'Organisation' => ['fields' => ['name']], + 'Role' => ['fields' => ['*']], + ], + 'fields' => ['User.org_id', 'User.id', 'User.email'], + ]); + $userForWorkflow['Server'] = []; + $userForWorkflow = $this->User->rearrangeToAuthForm($userForWorkflow); + } + if (empty($userForWorkflow)) { + $errors[] = __('Could not find a valid user to run the workflow. Please set setting `MISP.host_org_id` or make sure a valid site_admin user exists.'); + return false; + } + $roamingData = $this->workflowGraphTool->getRoamingData($userForWorkflow, $data, $workflow, $startNode); + $graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data']; + $graphWalker = $this->workflowGraphTool->getWalkerIterator($graphData, $this, $startNode, $for_path, $roamingData); + $preventExecutionForPaths = []; + foreach ($graphWalker as $graphNode) { + $node = $graphNode['node']; + $moduleClass = $this->getModuleClass($node); + foreach ($preventExecutionForPaths as $path_to_block) { + if ($path_to_block == array_slice($graphNode['path_list'], 0, count($path_to_block))) { + $walkResult['blocked_paths'][] = $graphNode['path_list']; + continue 2; + } + } + $nodeError = []; + $success = $this->executeNode($node, $roamingData, $nodeError); + $walkResult['executed_nodes'][] = $node; + if (empty($success)) { + $walkResult['blocking_nodes'][] = $node; + if (!empty($nodeError)) { + $errors[] = __( + 'Node `%s` (%s) from Workflow `%s` (%s) returned the following error: %s', + $node['data']['id'], + $node['id'], + $workflow['Workflow']['name'], + $workflow['Workflow']['id'], + implode(', ', $nodeError) + ); + } + if (!empty($moduleClass->blocking)) { + return false; // Node stopped execution for any path. If a module is blocking and it failed, stop the walk + } else if ($graphNode['path_type'] == GraphWalker::PATH_TYPE_NON_BLOCKING) { + $preventExecutionForPaths[] = $graphNode['path_list']; // Paths down the chain should not be executed + } + } + } + return true; + } + + public function executeNode(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + $roamingData->setCurrentNode($node['id']); + $moduleClass = $this->getModuleClass($node); + if (!empty($moduleClass->disabled)) { + $message = __('Could not execute disabled module `%s`.', $node['data']['id']); + $this->logExecutionError($roamingData->getWorkflow(), $message); + $errors[] = $message; + $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'disabled_module'), $roamingData->getData()); + return false; + } + if (!is_null($moduleClass)) { + try { + $success = $moduleClass->exec($node, $roamingData, $errors); + } catch (Exception $e) { + $message = __('Error while executing module %s. Error: %s', $node['data']['id'], $e->getMessage()); + $this->logExecutionError($roamingData->getWorkflow(), $message); + $errors[] = $message; + $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s&message=%s', $moduleClass->id, 'error', $e->getMessage()), $roamingData->getData()); + return false; + } + } else { + $message = sprintf(__('Could not load class for module: %s'), $node['data']['id']); + $this->logExecutionError($roamingData->getWorkflow(), $message); + $errors[] = $message; + $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData()); + return false; + } + $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData()); + return $success; + } + + private function __normalizeDataForTrigger($triggerClass, array $data): array + { + if (method_exists($triggerClass, 'normalizeData')) { + return $triggerClass->normalizeData($data); + } + return $data; + } + + private function digestExecutionResult(array $walkResult) + { + if (empty($walkResult['Nodes that stopped execution'])) { + return __('All nodes executed.'); + } + $str = []; + foreach ($walkResult['Nodes that stopped execution'] as $node) { + $str[] = __('Node `%s` (%s) stopped execution.', $node['data']['id'], $node['id']); + } + return implode(', ', $str); + } + + public function getModuleClass($node) + { + $this->loadAllWorkflowModules(); + $moduleClass = $this->loaded_classes[$node['data']['module_type']][$node['data']['id']] ?? null; + return $moduleClass; + } + + /** + * getModuleClassByType + * + * @param string $module_type + * @param string $id + * @param boolean $throwException + * @return + * @throws ModuleNotFoundException + */ + public function getModuleClassByType($module_type, $id, $throwException=false) + { + $this->loadAllWorkflowModules(); + $moduleClass = $this->loaded_classes[$module_type][$id] ?? null; + if (is_null($moduleClass) && !empty($throwException)) { + if ($module_type == 'trigger') { + throw new TriggerNotFoundException(__('Unknown module `%s` for module type `%s`', $id, $module_type)); + } else { + throw new ModuleNotFoundException(__('Unknown module `%s` for module type `%s`', $id, $module_type)); + } + } + return $moduleClass; + } + + /** + * getModuleConfigByType + * + * @param string $module_type + * @param string $id + * @param boolean $throwException + * @return array + * @throws ModuleNotFoundException + */ + public function getModuleConfigByType($module_type, $id, $throwException=false): array + { + $this->loadAllWorkflowModules(); + $moduleConfig = $this->loaded_modules[$module_type][$id] ?? null; + if (is_null($moduleConfig) && !empty($throwException)) { + throw new ModuleNotFoundException(__('Unknown module `%s` for module type `%s`', $id, $module_type)); + } + return $moduleConfig; + } + + public function attachNotificationToModules(array $modules, array $workflow): array + { + $trigger_is_misp_core_format = false; + $trigger_is_blocking = false; + $trigger_id = $this->workflowGraphTool->extractTriggerFromWorkflow($workflow['Workflow']['data'], false); + if (!empty($trigger_id)) { + $triggerClass = $this->getModuleClassByType('trigger', $trigger_id, true); + $trigger_is_misp_core_format = !empty($triggerClass->misp_core_format); + $trigger_is_blocking = !empty($triggerClass->blocking); + } + foreach ($modules as $moduleType => $modulesByType) { + foreach ($modulesByType as $i => $module) { + $modules[$moduleType][$i]['notifications'] = !empty($module['notifications']) ? $module['notifications'] : [ + 'error' => [], + 'warning' => [], + 'info' => [], + ]; + if ($module['disabled']) { + $modules[$moduleType][$i]['notifications']['error'][] = [ + 'text' => __('Module disabled'), + 'description' => __('This module is disabled and thus will not be executed.'), + 'details' => [ + __('Disabled modules that are blocking will also stop the execution') + ], + '__show_in_sidebar' => false, + '__show_in_node' => true, + ]; + } + if (!$trigger_is_blocking && !empty($module['blocking'])) { + $modules[$moduleType][$i]['notifications']['warning'][] = [ + 'text' => __('Blocking module might not work as intended'), + 'description' => __('This module is a blocking module for a non-blocking trigger.'), + 'details' => [ + __('The Blocking modules will be executed. However, it will not block the remaining of the execution') + ], + '__show_in_sidebar' => true, + '__show_in_node' => true, + ]; + } + if ($moduleType != 'modules_trigger') { + if (!$trigger_is_misp_core_format && !empty($module['expect_misp_core_format'])) { + $modules[$moduleType][$i]['notifications']['warning'][] = [ + 'text' => __('Potential data format issue'), + 'description' => __('This module might not work properly as it expect data compliant with the MISP core format.'), + 'details' => [ + __('This module expect data to be compliant with the MISP core format. However, the data passed by the trigger might not be under this format.') + ], + '__show_in_sidebar' => true, + '__show_in_node' => true, + ]; + } + } + } + } + return $modules; + } + + public function loadAllWorkflowModules() + { + if ($this->module_initialized) { + return; + } + $phpModuleFiles = Workflow::__listPHPModuleFiles(); + foreach ($phpModuleFiles as $type => $files) { + if ($type == 'custom') { + continue; + } + $classModuleFromFiles = $this->__getClassFromModuleFiles($type, $files, false); + foreach ($classModuleFromFiles['classConfigs'] as $i => $config) { + $classModuleFromFiles['classConfigs'][$i]['module_type'] = $type; + } + $this->loaded_modules[$type] = $classModuleFromFiles['classConfigs']; + $this->loaded_classes[$type] = $classModuleFromFiles['instancedClasses']; + } + // Load custom PHP modules from Lib + foreach ($phpModuleFiles['custom'] as $type => $files) { + $classModuleFromFiles = $this->__getClassFromModuleFiles($type, $files, true); + foreach ($classModuleFromFiles['classConfigs'] as $i => $config) { + $classModuleFromFiles['classConfigs'][$i]['module_type'] = $type; + } + $this->loaded_modules[$type] = array_merge($this->loaded_modules[$type], $classModuleFromFiles['classConfigs']); + $this->loaded_classes[$type] = array_merge($this->loaded_classes[$type], $classModuleFromFiles['instancedClasses']); + } + // Load module from misp-module service + $modules_from_service = $this->__getModulesFromModuleService() ?? []; + $misp_module_class = $this->__getClassForMispModule($modules_from_service); + $misp_module_configs = []; + foreach ($misp_module_class as $i => $module_class) { + $misp_module_configs[$i] = $module_class->getConfig(); + $misp_module_configs[$i]['module_type'] = 'action'; + } + $this->loaded_modules['action'] = array_merge($this->loaded_modules['action'], $misp_module_configs); + $this->loaded_classes['action'] = array_merge($this->loaded_classes['action'], $misp_module_class); + $this->__mergeGlobalConfigIntoLoadedModules(); + $this->module_initialized = true; + } + + private function __mergeGlobalConfigIntoLoadedModules() + { + foreach ($this->loaded_modules['trigger'] as &$trigger) { + $module_disabled = empty(Configure::read(sprintf('Plugin.Workflow_triggers_%s', $trigger['id']))); + $trigger['html_template'] = !empty($trigger['html_template']) ? $trigger['html_template'] : 'trigger'; + $trigger['disabled'] = $module_disabled; + $this->loaded_classes['trigger'][$trigger['id']]->disabled = $module_disabled; + $this->loaded_classes['trigger'][$trigger['id']]->html_template = !empty($trigger['html_template']) ? $trigger['html_template'] : 'trigger'; + } + $enabledModules = $this->getEnabledModules(); + array_walk($this->loaded_modules['logic'], function (&$logic) use ($enabledModules) { + $module_disabled = !in_array($logic['id'], $enabledModules); + $logic['disabled'] = $module_disabled; + $this->loaded_classes['logic'][$logic['id']]->disabled = $module_disabled; + }); + array_walk($this->loaded_modules['action'], function (&$action) use ($enabledModules) { + $module_disabled = !in_array($action['id'], $enabledModules); + $action['disabled'] = $module_disabled; + $this->loaded_classes['action'][$action['id']]->disabled = $module_disabled; + }); + + } + + private function __getEnabledModulesFromModuleService() + { + if (empty($this->Module)) { + $this->Module = ClassRegistry::init('Module'); + } + $enabledModules = $this->Module->getEnabledModules(null, 'Action'); + $misp_module_config = empty($enabledModules) ? false : $enabledModules; + return $misp_module_config; + } + + private function __getModulesFromModuleService() + { + if (empty($this->Module)) { + $this->Module = ClassRegistry::init('Module'); + } + $modules = $this->Module->getModules('Action'); + if (is_array($modules)) { + foreach ($modules as $i => $temp) { + if (!isset($temp['meta']['module-type']) || !in_array('action', $temp['meta']['module-type'])) { + unset($modules[$i]); + } + } + } + return $modules; + } + + private function __getClassForMispModule($misp_module_configs) + { + $filepathMispModule = sprintf('%s/%s', Workflow::MODULE_ROOT_PATH, 'Module_misp_module.php'); + $className = 'Module_misp_module'; + $reflection = null; + try { + require_once($filepathMispModule); + try { + $reflection = new \ReflectionClass($className); + } catch (\ReflectionException $e) { + return $e->getMessage(); + } + } catch (Exception $e) { + return $e->getMessage(); + } + $moduleClasses = []; + if (is_array($misp_module_configs)) { + foreach ($misp_module_configs as $moduleConfig) { + $mainClass = $reflection->newInstance($moduleConfig); + if ($mainClass->checkLoading() === 'The Factory Must Grow') { + $moduleClasses[$mainClass->id] = $mainClass; + } + } + } + return $moduleClasses; + } + + /** + * __listPHPModuleFiles List all PHP modules files + * + * @param boolean|array $targetDir If provided, will only collect files from that directory + * @return array + */ + private static function __listPHPModuleFiles($targetDir=false): array + { + $dirs = ['trigger', 'logic', 'action']; + if (!empty($targetDir)) { + $dirs = $targetDir; + } + $files = []; + foreach ($dirs as $dir) { + $folder = new Folder(Workflow::MODULE_ROOT_PATH . $dir); + $filesInFolder = $folder->find('.*\.php', true); + $files[$dir] = array_diff($filesInFolder, ['..', '.']); + if ($dir == 'action' || $dir == 'logic') { // No custom module for the triggers + $customFolder = new Folder(Workflow::CUSTOM_MODULE_ROOT_PATH . $dir); + $filesInCustomFolder = $customFolder->find('.*\.php', true); + $files['custom'][$dir] = array_diff($filesInCustomFolder, ['..', '.']); + } + } + return $files; + } + + private function __getClassFromModuleFiles($type, $files, $isCustom=false) + { + $instancedClasses = []; + $classConfigs = []; + foreach ($files as $filename) { + $filepath = sprintf('%s%s/%s', (!empty($isCustom) ? Workflow::CUSTOM_MODULE_ROOT_PATH : Workflow::MODULE_ROOT_PATH), $type, $filename); + $instancedClass = $this->__getClassFromModuleFile($filepath); + if (is_string($instancedClass)) { + $this->__logLoadingError($filename, $instancedClass); + $this->error_while_loading[$filename] = $instancedClass; + continue; + } + if (!empty($classConfigs[$instancedClass->id])) { + throw new WorkflowDuplicatedModuleIDException(__('Module %s has already been defined', $instancedClass->id)); + } + $classConfigs[$instancedClass->id] = $instancedClass->getConfig(); + $instancedClasses[$instancedClass->id] = $instancedClass; + if (!empty($isCustom)) { + $classConfigs[$instancedClass->id]['is_custom'] = true; + $instancedClasses[$instancedClass->id]->is_custom = true; + } + } + return [ + 'classConfigs' => $classConfigs, + 'instancedClasses' => $instancedClasses, + ]; + } + + public function logExecutionError($workflow, $message) + { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message); + $this->__logToFile($workflow, $message); + } + + /** + * __logToFile Log to file + * + * @param array $workflow + * @param string $message + * @return void + */ + private function __logToFile($workflow, $message) + { + $logEntry = sprintf('[%s] Workflow(%s:%s). %s' . PHP_EOL, date('Y-m-d H:i:s'), $workflow['Workflow']['trigger_id'], $workflow['Workflow']['id'], $message); + // file_put_contents(APP . 'tmp/logs/workflow-execution.log', $logEntry, FILE_APPEND | LOCK_EX); + FileAccessTool::writeToFile(APP . 'tmp/logs/workflow-execution.log', $logEntry, false, true); + } + + private function __logLoadingError($filename, $error) + { + $this->Log = ClassRegistry::init('Log'); + $message = __('Could not load module for file `%s`.', $filename); + $this->Log->createLogEntry('SYSTEM', 'load_module', 'Workflow', 0, $message, $error); + } + + /** + * getProcessorClass + * + * @param string $filePath + * @param string $processorMainClassName + * @return object|string Object loading success, string containing the error if failure + */ + private function __getClassFromModuleFile($filepath) + { + $className = explode('/', $filepath); + $className = str_replace('.php', '', $className[count($className)-1]); + try { + if (!@include_once($filepath)) { + $message = __('Could not load module for path %s. File does not exists.', $filepath); + $this->log($message, LOG_ERR); + return $message; + } + try { + $reflection = new \ReflectionClass($className); + } catch (\ReflectionException $e) { + $message = __('Could not load module for path %s. Could not instanciate class', $filepath); + $this->logException($message, $e); + return $message; + } + $mainClass = $reflection->newInstance(); + if ($mainClass->checkLoading() === 'The Factory Must Grow') { + return $mainClass; + } + } catch (Exception $e) { + $message = __('Could not load module for path %s', $filepath); + $this->logException($message, $e); + return $message; + } + } + + public function getModuleLoadingError(): array + { + return $this->error_while_loading; + } + + public function getModulesByType($module_type=false): array + { + $this->loadAllWorkflowModules(); + + $modules_trigger = $this->loaded_modules['trigger']; + $modules_logic = $this->loaded_modules['logic']; + $modules_action = $this->loaded_modules['action']; + + $this->__sortModulesByName($modules_trigger); + $this->__sortModulesByName($modules_logic); + $this->__sortModulesByName($modules_action); + $modules_trigger = array_values($modules_trigger); + $modules_logic = array_values($modules_logic); + $modules_action = array_values($modules_action); + $modules = [ + 'modules_trigger' => $modules_trigger, + 'modules_logic' => $modules_logic, + 'modules_action' => $modules_action, + ]; + if (!empty($module_type)) { + if (!empty($modules['modules_' . $module_type])) { + return $modules['modules_' . $module_type]; + } else { + return []; + } + } + return $modules; + } + + private function __sortModulesByName(&$modules) + { + uasort($modules, function ($module1, $module2) { + if ($module1['name'] == $module2['name']) { + return 0; + } + return ($module1['name'] < $module2['name']) ? -1 : 1; + }); + } + + public function getModules(): array + { + $modulesByType = $this->getModulesByType(); + return array_merge($modulesByType['modules_trigger'], $modulesByType['modules_logic'], $modulesByType['modules_action']); + } + + /** + * getModules Return the module from the provided ID + * + * @param string|array $module_ids + * @return array + */ + public function getModuleByID($module_ids): array + { + $returnAString = false; + if (!is_array($module_ids)) { + $returnAString = true; + $module_ids = [$module_ids]; + } + $matchingModules = []; + $modules = $this->getModules(); + foreach ($modules as $module) { + if (in_array($module['id'], $module_ids)) { + $matchingModules[] = $module; + } + } + if (empty($matchingModules)) { + return []; + } + return $returnAString ? $matchingModules[0] : $matchingModules; + } + + private function __incrementWorkflowExecutionCount(array $workflow): array + { + $workflow['Workflow']['counter'] = intval($workflow['Workflow']['counter']) + 1; + $this->save($workflow, ['fieldList' => ['counter']]); + return $this->fetchWorkflow($workflow['Workflow']['id']); + } + + /** + * fetchWorkflows + * + * @param array $options + * @param bool $full + * @return array + */ + public function fetchWorkflows(array $options = array(), $full = false) + { + $params = array( + 'contain' => $this->defaultContain, + 'recursive' => -1 + ); + if ($full) { + $params['recursive'] = 1; + } + if (isset($options['fields'])) { + $params['fields'] = $options['fields']; + } + if (isset($options['conditions'])) { + $params['conditions']['AND'][] = $options['conditions']; + } + if (isset($options['group'])) { + $params['group'] = !empty($options['group']) ? $options['group'] : false; + } + if (isset($options['contain'])) { + $params['contain'] = !empty($options['contain']) ? $options['contain'] : []; + } + if (isset($options['order'])) { + $params['order'] = !empty($options['order']) ? $options['order'] : []; + } + $workflows = $this->find('all', $params); + return $workflows; + } + + /** + * fetchWorkflow + * + * @param int|string $id + * @param bool $throwErrors + * @throws NotFoundException + * @return array + */ + public function fetchWorkflow($id, bool $throwErrors = true): array + { + $options = []; + if (is_numeric($id)) { + $options = ['conditions' => ['Workflow.id' => $id]]; + } elseif (Validation::uuid($id)) { + $options = ['conditions' => ['Workflow.uuid' => $id]]; + } else { + if ($throwErrors) { + throw new NotFoundException(__('Invalid workflow')); + } + return []; + } + $workflow = $this->fetchWorkflows($options); + if (empty($workflow)) { + if ($throwErrors) { + throw new NotFoundException(__('Invalid workflow')); + } + return []; + } + return $workflow[0]; + } + + /** + * fetchWorkflowByTrigger + * + * @param int|string $trigger_id + * @param bool $throwErrors + * @throws NotFoundException + * @return array + */ + public function fetchWorkflowByTrigger($trigger_id, bool $throwErrors = true): array + { + $options = [ + 'conditions' => [ + 'Workflow.trigger_id' => $trigger_id, + ] + ]; + $workflow = $this->fetchWorkflows($options); + if (empty($workflow)) { + if ($throwErrors) { + throw new NotFoundException(__('Invalid workflow')); + } + return []; + } + return $workflow[0]; + } + + /** + * addWorkflow Add a worflow + * + * @param array $trigger + * @return array Any errors preventing the edition + */ + public function addWorkflow(array $workflow): array + { + $errors = []; + $this->create(); + $saved = $this->__saveAndReturnErrors($workflow, ['fieldList' => self::CAPTURE_FIELDS_ADD], $errors); + return [ + 'saved' => $saved, + 'errors' => $errors, + ]; + } + + /** + * editWorkflow Edit a worflow + * + * @param array $workflow + * @return array Any errors preventing the edition + */ + public function editWorkflow(array $workflow): array + { + $errors = []; + if (!isset($workflow['Workflow']['uuid'])) { + $errors[] = __('Workflow doesn\'t have an UUID'); + return $errors; + } + $existingWorkflow = $this->fetchWorkflow($workflow['Workflow']['id']); + $workflow['Workflow']['id'] = $existingWorkflow['Workflow']['id']; + unset($workflow['Workflow']['timestamp']); + $saved = $this->__saveAndReturnErrors($workflow, ['fieldList' => self::CAPTURE_FIELDS_EDIT], $errors); + return [ + 'saved' => $saved, + 'errors' => $errors, + ]; + } + + /** + * genGraphDataForTrigger Generate fake graph data under the drawflow format + * + * @param string $trigger_id + * @return array + */ + public function genGraphDataForTrigger($trigger_id): array + { + if (empty($this->loaded_modules['trigger'][$trigger_id])) { + throw new TriggerNotFoundException(__('Unknown trigger `%s`', $trigger_id)); + } + $module_config = $this->loaded_modules['trigger'][$trigger_id]; + $data = [ + 1 => [ + 'class' => 'block-type-trigger', + 'data' => $module_config, + 'id' => 1, + 'inputs' => [], + 'outputs' => [ + 'output_1' => [ + 'connections' => [] + ], + ], + 'pos_x' => 0, + 'pos_y' => 0, + 'typenode' => false, + ] + ]; + return $data; + } + + /** + * hasPathWarnings + * + * @param array $graphData + * @param array $edges + * @return boolean + */ + public function hasPathWarnings(array $graphData, array &$edges=[]) + { + $startNodes = $this->workflowGraphTool->extractConcurrentTasksFromWorkflow($graphData, true); + $concurrentNodeIDs = Hash::extract($startNodes, '{n}.id'); + $roamingData = $this->workflowGraphTool->getRoamingData(); + foreach ($concurrentNodeIDs as $concurrentNodeID) { + $graphWalker = $this->workflowGraphTool->getWalkerIterator($graphData, $this, $concurrentNodeID, GraphWalker::PATH_TYPE_INCLUDE_LOGIC, $roamingData); + foreach ($graphWalker as $graphNode) { + $moduleClass = $this->getModuleClass($graphNode['node']); + if (!empty($moduleClass->blocking)) { + $parsedPathList = GraphWalker::parsePathList($graphNode['path_list']); + foreach ($parsedPathList as $pathEntry) { + $edges[] = [ + $pathEntry['source_id'], + $pathEntry['next_node_id'], + __('This path leads to a blocking node from a non-blocking context'), + $moduleClass->blocking, + $moduleClass->id, + $graphNode['node']['id'], + ]; + } + } + } + } + return !empty($edges); + } + + private function __saveAndReturnErrors($data, $saveOptions = [], &$errors = []) + { + $saveSuccess = $this->save($data, $saveOptions); + if (!$saveSuccess) { + foreach ($this->validationErrors as $validationError) { + $errors[] = $validationError[0]; + } + } else { + if (!empty($saveSuccess['Workflow']['data'])) { + $saveSuccess['Workflow']['data'] = JsonTool::decode($saveSuccess['Workflow']['data']); + } + } + return $saveSuccess; + } + + public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='/', array $data=[]) + { + $debug_url = Configure::read('Plugin.Workflow_debug_url'); + if (empty($workflow['Workflow']['debug_enabled'])) { + return; + } + App::uses('HttpSocket', 'Network/Http'); + $socket = new HttpSocket([ + 'timeout' => 5 + ]); + $uri = sprintf('%s%s', $debug_url, $path); + $dataToPost = [ + 'source' => [ + 'node_id' => $node['id'] ?? '', + 'module_id' => $node['data']['id'] ?? '', + 'filters' => $node['data']['saved_filters'] ?? '', + 'parameters' => $node['data']['indexed_params'] ?? '', + ], + 'timestamp' => date("c"), + 'data' => $data, + ]; + $socket->post($uri, JsonTool::encode($dataToPost)); + } + + public function getDotNotation($id) + { + App::uses('GraphvizDOTTool', 'Tools'); + $workflow = $this->fetchWorkflow($id); + $dot = GraphvizDOTTool::dot($workflow['Workflow']['data']); + return $dot; + } + + public function getMermaid($id) + { + App::uses('MermaidFlowchartTool', 'Tools'); + $workflow = $this->fetchWorkflow($id); + $mermaid = MermaidFlowchartTool::mermaid($workflow['Workflow']['data']); + return $mermaid; + } +} \ No newline at end of file diff --git a/app/Model/WorkflowBlueprint.php b/app/Model/WorkflowBlueprint.php new file mode 100644 index 000000000..283f8c222 --- /dev/null +++ b/app/Model/WorkflowBlueprint.php @@ -0,0 +1,173 @@ + [ + 'roleModel' => 'Role', + 'roleKey' => 'role_id', + 'change' => 'full' + ], + ]; + + public $belongsTo = [ + ]; + + public $validate = [ + 'value' => [ + 'stringNotEmpty' => [ + 'rule' => ['stringNotEmpty'] + ] + ], + 'uuid' => [ + 'uuid' => [ + 'rule' => 'uuid', + 'message' => 'Please provide a valid RFC 4122 UUID' + ], + 'unique' => [ + 'rule' => 'isUnique', + 'message' => 'The UUID provided is not unique', + 'required' => 'create' + ] + ], + ]; + + const CAPTURE_FIELDS = ['name', 'description', 'timestamp', 'data']; + + public function beforeValidate($options = array()) + { + parent::beforeValidate(); + if (empty($this->data['WorkflowBlueprint']['uuid'])) { + $this->data['WorkflowBlueprint']['uuid'] = CakeText::uuid(); + } else { + $this->data['WorkflowBlueprint']['uuid'] = strtolower($this->data['WorkflowBlueprint']['uuid']); + } + if (empty($this->data['WorkflowBlueprint']['data'])) { + $this->data['WorkflowBlueprint']['data'] = []; + } + if (empty($this->data['WorkflowBlueprint']['timestamp'])) { + $this->data['WorkflowBlueprint']['timestamp'] = time(); + } + return true; + } + + public function afterFind($results, $primary = false) + { + foreach ($results as $k => $result) { + if (empty($result['WorkflowBlueprint']['data'])) { + $result['WorkflowBlueprint']['data'] = '{}'; + } + $results[$k]['WorkflowBlueprint']['data'] = JsonTool::decode($result['WorkflowBlueprint']['data']); + $results[$k] = $this->attachModuleDataToBlueprint($results[$k]); + if (!empty($results[$k]['WorkflowBlueprint']['data'])) { + $results[$k]['WorkflowBlueprint']['mermaid'] = $this->getMermaidForData($results[$k]['WorkflowBlueprint']['data']); + } + } + return $results; + } + + public function beforeSave($options = []) + { + if (is_array($this->data['WorkflowBlueprint']['data'])) { + $this->data['WorkflowBlueprint']['data'] = JsonTool::encode($this->data['WorkflowBlueprint']['data']); + } + return true; + } + + public function attachModuleDataToBlueprint(array $blueprint) + { + $this->Workflow = ClassRegistry::init('Workflow'); + foreach ($blueprint['WorkflowBlueprint']['data'] as $i => $node) { + $module = $this->Workflow->getModuleConfigByType($node['data']['module_type'], $node['data']['id']); + $blueprint['WorkflowBlueprint']['data'][$i]['data']['module_data'] = $module; + } + return $blueprint; + } + + + /** + * __readBlueprintsFromRepo Reads blueprints from the misp-workflow-blueprints repository + * + * @return array + */ + private function __readBlueprintsFromRepo(): array + { + $dir = new Folder(self::REPOSITORY_PATH); + $files = $dir->find('.*\.json'); + $blueprints = []; + foreach ($files as $file) { + $blueprints[] = FileAccessTool::readJsonFromFile($dir->pwd() . DS . $file); + } + return $blueprints; + } + + /** + * update Update the blueprint using the default repository + * + * @param boolean $force + * @return void + */ + public function update($force=false) + { + $blueprints_from_repo = $this->__readBlueprintsFromRepo(); + if (empty($blueprints_from_repo)) { + throw new NotFoundException(__('Default blueprints could not be loaded or `%s` folder is empty', self::REPOSITORY_PATH)); + } + $existing_blueprints = $this->find('all', [ + 'recursive' => -1 + ]); + $existing_blueprints_by_uuid = Hash::combine($existing_blueprints, '{n}.WorkflowBlueprint.uuid', '{n}.WorkflowBlueprint'); + foreach ($blueprints_from_repo as $blueprint_from_repo) { + $blueprint_from_repo = $blueprint_from_repo['WorkflowBlueprint']; + $blueprint_from_repo['default'] = true; + if (!empty($existing_blueprints_by_uuid[$blueprint_from_repo['uuid']])) { + $existing_blueprint = $existing_blueprints_by_uuid[$blueprint_from_repo['uuid']]; + if ($force || $blueprint_from_repo['timestamp'] > $existing_blueprint['timestamp']) { + $blueprint_from_repo['id'] = $existing_blueprint['id']; + $this->save($blueprint_from_repo); + continue; + } + } else { + $this->create(); + $this->save($blueprint_from_repo); + } + } + } + + public function getMermaidForData($workflowBlueprintData) + { + App::uses('MermaidFlowchartTool', 'Tools'); + $mermaid = MermaidFlowchartTool::mermaid($workflowBlueprintData); + return $mermaid; + } + + public function getDotNotation($id) + { + App::uses('GraphvizDOTTool', 'Tools'); + $blueprint = $this->find('first', [ + 'conditions' => ['id' => $id], + 'recursive' => -1, + ]); + $dot = GraphvizDOTTool::dot($blueprint['WorkflowBlueprint']['data']); + return $dot; + } + + public function getMermaid($id) + { + App::uses('MermaidFlowchartTool', 'Tools'); + $blueprint = $this->find('first', [ + 'conditions' => ['id' => $id], + 'recursive' => -1, + ]); + $mermaid = MermaidFlowchartTool::mermaid($blueprint['WorkflowBlueprint']['data']); + return $mermaid; + } +} diff --git a/app/Model/WorkflowModules/Module_misp_module.php b/app/Model/WorkflowModules/Module_misp_module.php new file mode 100644 index 000000000..8d37b0c4c --- /dev/null +++ b/app/Model/WorkflowModules/Module_misp_module.php @@ -0,0 +1,112 @@ +id = Inflector::underscore($misp_module_config['name']); + $this->name = $misp_module_config['name']; + $this->description = $misp_module_config['meta']['description']; + if (!empty($misp_module_config['meta']['icon'])) { + $this->icon = $misp_module_config['meta']['icon']; + } + if (!empty($misp_module_config['meta']['icon_class'])) { + $this->icon_class = $misp_module_config['meta']['icon_class']; + } + if (!empty($misp_module_config['meta']['inputs'])) { + $this->inputs = (int)$misp_module_config['meta']['inputs']; + } + if (!empty($misp_module_config['meta']['outputs'])) { + $this->inputs = (int)$misp_module_config['meta']['outputs']; + } + if (!empty($misp_module_config['meta']['config']['blocking'])) { + $this->blocking = !empty($misp_module_config['meta']['config']['blocking']); + } + if (!empty($misp_module_config['meta']['config']['expect_misp_core_format'])) { + $this->expect_misp_core_format = !empty($misp_module_config['meta']['config']['expect_misp_core_format']); + } + if (!empty($misp_module_config['meta']['config']['support_filters'])) { + $this->support_filters = !empty($misp_module_config['meta']['config']['support_filters']); + } + if (!empty($misp_module_config['meta']['config'])) { + foreach ($misp_module_config['meta']['config']['params'] as $paramName => $moduleParam) { + $this->params[] = $this->translateParams($paramName, $moduleParam); + } + } + $this->Module = ClassRegistry::init('Module'); + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData); + $postData = ['module' => $this->name]; + $rData = $roamingData->getData(); + $postData['data'] = $rData; + if ($this->support_filters) { + $filters = $this->getFilters($node); + $extracted = $this->extractData($rData, $filters['selector']); + if ($extracted === false) { + return false; + } + $filteredItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']); + $postData['filteredItems'] = !empty($filteredItems) ? $filteredItems : $rData; + } + + $indexedParams = $this->getParamsWithValues($node); + $postData['params'] = Hash::combine($indexedParams, '{s}.id', '{s}.value'); + $params = $this->getParamsWithValues($node); + $matchingData = []; + foreach ($params as $param) { + if (!empty($param['_isHashPath'])) { + $matchingData[$param['label']] = !empty($param['value']) ? $this->extractData($rData, $param['value']) : $rData; + } + } + if (!empty($matchingData)) { + $postData['matchingData'] = $matchingData; + } + + $query = $this->Module->queryModuleServer($postData, false, 'Action', false, $postData['data']); + if (!empty($query['error'])) { + $errors[] = $query['error']; + return false; + } + return true; + } + + // FIXME: We might want to align the module config with what's currently supported + protected function translateParams($paramName, $moduleParam): array + { + $param = [ + 'id' => Inflector::slug(Inflector::underscore($paramName)), + 'label' => Inflector::humanize($paramName), + 'placeholder' => $moduleParam['value'] ?? '', + ]; + if ($moduleParam['type'] == 'hash_path') { + $param['type'] = 'input'; + $param['_isHashPath'] = true; + } elseif ($moduleParam['type'] == 'large_string') { + $param['type'] = 'textarea'; + } else { + $param['type'] = 'input'; + } + return $param; + } +} diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php new file mode 100644 index 000000000..eddc1c24e --- /dev/null +++ b/app/Model/WorkflowModules/WorkflowBaseModule.php @@ -0,0 +1,250 @@ + 'selector', 'value' => ''], + ['text' => 'value', 'value' => ''], + ['text' => 'operator', 'value' => ''], + ['text' => 'path', 'value' => ''], + ]; + public $params = []; + + private $Event; + + /** @var PubSubTool */ + private static $loadedPubSubTool; + + public function __construct() + { + } + + protected function mergeNodeConfigIntoParameters($node): array + { + $fullIndexedParams = []; + foreach ($this->params as $param) { + $param['value'] = $nodeParamByID[$param['id']]['value'] ?? null; + $param['value'] = $node['data']['indexed_params'][$param['id']] ?? null; + $fullIndexedParams[$param['id']] = $param; + } + return $fullIndexedParams; + } + + protected function getParamsWithValues($node): array + { + $indexedParams = $this->mergeNodeConfigIntoParameters($node); + foreach ($indexedParams as $id => $param) { + $indexedParams[$id]['value'] = $param['value'] ?? ($param['default'] ?? ''); + } + return $indexedParams; + } + + protected function filtersEnabled($node): bool + { + $indexedFilters = $this->getFilters($node); + foreach ($indexedFilters as $k => $v) { + if ($v != '') { + return true; + } + } + return false; + } + + protected function getFilters($node): array + { + $indexedFilters = []; + $nodeParam = []; + foreach ($node['data']['saved_filters'] as $name => $value) { + $nodeParam[$name] = $value; + } + foreach ($this->saved_filters as $filter) { + $filter['value'] = $nodeParam[$filter['text']] ?? $filter['value']; + $indexedFilters[$filter['text']] = $filter['value']; + } + return $indexedFilters; + } + + public function getConfig(): array + { + $reflection = new ReflectionObject($this); + $properties = []; + foreach ($reflection->getProperties() as $property) { + if ($property->isPublic()) { + $properties[$property->getName()] = $property->getValue($this); + } + } + return $properties; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + return true; + } + + protected function push_zmq($message) + { + if (!self::$loadedPubSubTool) { + App::uses('PubSubTool', 'Tools'); + $pubSubTool = new PubSubTool(); + $pubSubTool->initTool(); + self::$loadedPubSubTool = $pubSubTool; + } + $pubSubTool = self::$loadedPubSubTool; + $pubSubTool->workflow_push($message); + } + + protected function logError($message) + { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry('SYSTEM', 'exec_module', 'Workflow', $this->id, $message); + } + + public function checkLoading() + { + return 'The Factory Must Grow'; + } + + protected function extractData($data, $path) + { + $extracted = $data; + if (!empty($path)) { + try { + $extracted = Hash::extract($data, $path); + } catch (Exception $e) { + return false; + } + } + return $extracted; + } + + protected function extractDataForFilters(array $node, WorkflowRoamingData $roamingData) + { + $rData = $roamingData->getData(); + if (empty($this->support_filters)) { + return $rData; + } + $filters = $this->getFilters($node); + if (in_array(null, array_values($filters))) { + return $rData; + } + $extracted = $this->extractData($rData, $filters['selector']); + if ($extracted === false) { + return $rData; + } + $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']); + return $matchingItems; + } + + protected function evaluateCondition($data, $operator, $value): bool + { + if ($operator == 'in') { + return is_array($data) && in_array($value, $data); + } elseif ($operator == 'not_in') { + return is_array($data) && !in_array($value, $data); + } elseif ($operator == 'equals') { + return !is_array($data) && $data == $value; + } elseif ($operator == 'not_equals') { + return !is_array($data) && $data != $value; + } elseif ($operator == 'in_or' || $operator == 'in_and' || $operator == 'not_in_or' || $operator == 'not_in_and') { + if (!is_array($data) || !is_array($value)) { + return false; + } + $matching = array_filter($data, function($item) use ($value) { + return in_array($item, $value); + }); + if ($operator == 'in_or') { + return !empty($matching); + } elseif ($operator == 'in_and') { + return array_values($matching) == array_values($value); + } elseif ($operator == 'not_in_or') { + return empty($matching); + } elseif ($operator == 'not_in_and') { + return array_values($matching) != array_values($value); + } + } + return false; + } + + protected function getItemsMatchingCondition($items, $value, $operator, $path) + { + foreach ($items as $i => $item) { + $subItem = $this->extractData($item, $path, $operator); + if (in_array($operator, ['equals', 'not_equals'])) { + $subItem = !empty($subItem) ? $subItem[0] : $subItem; + } + if (!$this->evaluateCondition($subItem, $operator, $value)) { + unset($items[$i]); + } + } + return $items; + } +} + +class WorkflowBaseTriggerModule extends WorkflowBaseModule +{ + const OVERHEAD_LOW = 1; + const OVERHEAD_MEDIUM = 2; + const OVERHEAD_HIGH = 3; + + public $scope = 'others'; + public $blocking = false; + public $misp_core_format = false; + public $trigger_overhead = self::OVERHEAD_LOW; + public $trigger_overhead_message = ''; + public $inputs = 0; + public $outputs = 1; + + /** + * normalizeData Massage the data before entering the workflow + * + * @param array $data + * @return array|false + */ + public function normalizeData(array $data) + { + if (!empty($this->misp_core_format)) { + $converted = $this->convertData($data); + if (empty($converted)) { + return false; + } + return $converted; + } + return $data; + } + + /** + * convertData function + * + * @param array $data + * @return array + */ + protected function convertData(array $data): array + { + App::uses('WorkflowFormatConverterTool', 'Tools'); + return WorkflowFormatConverterTool::convert($data); + } +} + +class WorkflowBaseLogicModule extends WorkflowBaseModule +{ + public $blocking = false; + public $inputs = 1; + public $outputs = 2; +} + +class WorkflowBaseActionModule extends WorkflowBaseModule +{ +} diff --git a/app/Model/WorkflowModules/action/Module_enrich_event.php b/app/Model/WorkflowModules/action/Module_enrich_event.php new file mode 100644 index 000000000..1bc6db996 --- /dev/null +++ b/app/Model/WorkflowModules/action/Module_enrich_event.php @@ -0,0 +1,89 @@ +Module = ClassRegistry::init('Module'); + $modules = $this->Module->getModules('Enrichment'); + $moduleOptions = []; + if (is_array($modules)) { + $moduleOptions = array_merge([''], Hash::combine($modules, '{n}.name', '{n}.name')); + } else { + $moduleOptions[] = $modules; + } + $this->params = [ + [ + 'id' => 'modules', + 'label' => 'Modules', + 'type' => 'select', + 'options' => $moduleOptions, + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + if (empty($params['modules']['value'])) { + $errors[] = __('No enrichmnent module selected'); + return false; + } + $rData = $roamingData->getData(); + $event_id = $rData['Event']['id']; + $options = [ + 'user' => $roamingData->getUser(), + 'event_id' => $event_id, + 'modules' => [$params['modules']['value']] + ]; + $filters = $this->getFilters($node); + $extracted = $this->extractData($rData, $filters['selector']); + if ($extracted === false) { + return false; + } + $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']); + if (!empty($matchingItems)) { + $extractedUUIDs = $this->extractData($matchingItems, '{n}.uuid'); + if ($extractedUUIDs === false) { + return false; + } + $options['attribute_uuids'] = $extractedUUIDs; + } + + $this->Event = ClassRegistry::init('Event'); + $result = $this->Event->enrichment($options); + if ($result === true) { + $this->push_zmq([ + 'Warning' => __('Error while trying to reach enrichment service or no module available'), + 'Attribute added' => 0 + ]); + } else { + $this->push_zmq([ + 'Enriching event' => $event_id, + 'Attribute added' => $result + ]); + $fullEvent = $this->Event->fetchEvent($roamingData->getUser(), [ + 'eventid' => $event_id, + 'includeAttachments' => 1 + ]); + $roamingData->setData($fullEvent[0]); + } + return true; + } +} diff --git a/app/Model/WorkflowModules/action/Module_push_zmq.php b/app/Model/WorkflowModules/action/Module_push_zmq.php new file mode 100644 index 000000000..e6ea39ba6 --- /dev/null +++ b/app/Model/WorkflowModules/action/Module_push_zmq.php @@ -0,0 +1,43 @@ +params = [ + [ + 'id' => 'data_extraction_path', + 'label' => 'Data extraction path', + 'type' => 'input', + 'default' => '', + 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name', + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + $path = $params['match_condition']['value']; + $data = $roamingData->getData(); + $extracted = $this->extractData($data, $path); + if ($extracted === false) { + $errors[] = __('Error while trying to extract data with path `%s`', $path); + return false; + } + $this->push_zmq(JsonTool::encode($extracted)); + return true; + } +} diff --git a/app/Model/WorkflowModules/action/Module_stop_execution.php b/app/Model/WorkflowModules/action/Module_stop_execution.php new file mode 100644 index 000000000..5266a1557 --- /dev/null +++ b/app/Model/WorkflowModules/action/Module_stop_execution.php @@ -0,0 +1,26 @@ + 0, + ]; + $this->Tag = ClassRegistry::init('Tag'); + $this->Event = ClassRegistry::init('Event'); + $this->Attribute = ClassRegistry::init('Attribute'); + $tags = $this->Tag->find('all', [ + 'conditions' => $conditions, + 'recursive' => -1, + 'order' => ['name asc'], + 'fields' => ['Tag.id', 'Tag.name'] + ]); + $tags = array_column(array_column($tags, 'Tag'), 'name', 'id'); + $this->params = [ + [ + 'id' => 'scope', + 'label' => 'Scope', + 'type' => 'select', + 'options' => [ + 'event' => __('Event'), + 'attribute' => __('Attributes'), + ], + 'default' => 'event', + ], + [ + 'id' => 'action', + 'label' => 'Action', + 'type' => 'select', + 'options' => [ + 'add' => __('Add Tags'), + 'remove' => __('Remove Tags'), + ], + 'default' => 'add', + ], + [ + 'id' => 'tags', + 'label' => 'Tags', + 'type' => 'picker', + 'multiple' => true, + 'options' => $tags, + 'placeholder' => __('Pick a tag'), + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + + $rData = $roamingData->getData(); + + if ($this->filtersEnabled($node)) { + $filters = $this->getFilters($node); + $extracted = $this->extractData($rData, $filters['selector']); + if ($extracted === false) { + return false; + } + $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']); + } else { + $matchingItems = $rData; + if ($params['scope']['value'] == 'attribute') { + $matchingItems = Hash::extract($matchingItems, '_AttributeFlattened.{n}'); + } + } + $result = false; + if ($params['scope']['value'] == 'event') { + if ($params['action']['value'] == 'remove') { + $result = $this->__removeTagsFromEvent($matchingItems, $params['tags']['value']); + } else { + $result = $this->__addTagsToEvent($matchingItems, $params['tags']['value']); + } + } else { + if ($params['action']['value'] == 'remove') { + $result = $this->__removeTagsFromAttributes($matchingItems, $params['tags']['value']); + } else { + $result = $this->__addTagsToAttributes($matchingItems, $params['tags']['value']); + } + } + return $result; + } + + private function __addTagsToAttributes(array $attributes, array $tags): bool + { + $success = false; + foreach ($attributes as $attribute) { + $saveSuccess = $this->Attribute->attachTagsFromAttributeAndTouch($attribute['id'], $attribute['event_id'], $tags); + $success = $success || !empty($saveSuccess); + } + return $success; + } + + private function __removeTagsFromAttributes(array $attributes,array $tags): bool + { + $success = false; + foreach ($attributes as $attribute) { + $saveSuccess = $this->Attribute->detachTagsFromAttributeAndTouch($attribute['id'], $attribute['event_id'], $tags); + $success = $success || !empty($saveSuccess); + } + return $success; + } + + private function __addTagsToEvent(array $event, array $tags): bool + { + return !empty($this->Event->attachTagsToEventAndTouch($event['Event']['id'], $tags)); + } + + private function __removeTagsFromEvent(array $event, array $tags): bool + { + return !empty($this->Event->detachTagsFromEventAndTouch($event['Event']['id'], $tags)); + } +} diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php new file mode 100644 index 000000000..c0c3c4393 --- /dev/null +++ b/app/Model/WorkflowModules/action/Module_webhook.php @@ -0,0 +1,108 @@ +params = [ + [ + 'id' => 'url', + 'label' => 'Payload URL', + 'type' => 'input', + 'placeholder' => 'https://example.com/test', + ], + [ + 'id' => 'content_type', + 'label' => 'Content type', + 'type' => 'select', + 'default' => 'json', + 'options' => [ + 'json' => 'application/json', + 'form' => 'application/x-www-form-urlencoded', + ], + ], + [ + 'id' => 'data_extraction_path', + 'label' => 'Data extraction path', + 'type' => 'input', + 'default' => '', + 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name', + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + if (empty($params['url']['value'])) { + $errors[] = __('URL not provided.'); + return false; + } + + $rData = $roamingData->getData(); + $path = $params['data_extraction_path']['value']; + $extracted = !empty($params['data_extraction_path']['value']) ? $this->extractData($rData, $path) : $rData; + try { + $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $extracted); + if ($response->isOk()) { + return true; + } + if ($response->code === 403 || $response->code === 401) { + $errors[] = __('Authentication failed.'); + return false; + } + $errors[] = __('Something went wrong with the request or the remote side is having issues. Body returned: %s', $response->body); + return false; + } catch (SocketException $e) { + $errors[] = __('Something went wrong while sending the request. Error returned: %s', $e->getMessage()); + return false; + } catch (Exception $e) { + $errors[] = __('Something went wrong. Error returned: %s', $e->getMessage()); + return false; + } + $errors[] = __('Something went wrong with the request or the remote side is having issues.'); + return false; + } + + private function doRequest($url, $contentType, array $data) + { + $this->Event = ClassRegistry::init('Event'); // We just need a model to use AppModel functions + $version = implode('.', $this->Event->checkMISPVersion()); + $commit = $this->Event->checkMIPSCommit(); + + $request = [ + 'header' => [ + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + 'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit), + ] + ]; + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocket(null, $this->timeout); + if ($contentType == 'form') { + $request['header']['Content-Type'] = 'application/x-www-form-urlencoded'; + $response = $HttpSocket->post($url, $data, $request); + } else { + $response = $HttpSocket->post($url, JsonTool::encode($data), $request); + } + return $response; + } +} diff --git a/app/Model/WorkflowModules/logic/Module_concurrent_task.php b/app/Model/WorkflowModules/logic/Module_concurrent_task.php new file mode 100644 index 000000000..bccfa49e6 --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_concurrent_task.php @@ -0,0 +1,59 @@ +Workflow = ClassRegistry::init('Workflow'); + $this->Job = ClassRegistry::init('Job'); + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool + { + parent::exec($node, $roamingData, $errors); + + $data = $roamingData->getData(); + $node_id_to_exec = (int)$data['__node_id_to_exec']; + unset($data['__node_id_to_exec']); + $roamingData->setData($data); + + $jobId = $this->Job->createJob( + $roamingData->getUser(), + Job::WORKER_PRIO, + 'workflowParallelTask', + sprintf('Workflow ID: %s', $roamingData->getWorkflow()['Workflow']['id']), + __('Running workflow parallel tasks.') + ); + $this->Job->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_WORKFLOW, + [ + 'walkGraph', + $roamingData->getWorkflow()['Workflow']['id'], + $node_id_to_exec, + JsonTool::encode($roamingData->getData()), + GraphWalker::PATH_TYPE_NON_BLOCKING, + $jobId + ], + true, + $jobId + ); + return true; + } +} diff --git a/app/Model/WorkflowModules/logic/Module_distribution_if.php b/app/Model/WorkflowModules/logic/Module_distribution_if.php new file mode 100644 index 000000000..5110a221c --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_distribution_if.php @@ -0,0 +1,128 @@ + 'Is', + 'not_equals' => 'Is not', + 'more_restrictive_or_equal_than' => 'More restrictive or equal than', + 'more_permisive_or_equal_than' => 'More permisive or equal than', + ]; + + public function __construct() + { + parent::__construct(); + $this->Attribute = ClassRegistry::init('Attribute'); + $distributionLevels = $this->Attribute->shortDist; + unset($distributionLevels[4]); + unset($distributionLevels[5]); + $distribution_param = []; + foreach ($distributionLevels as $i => $text) { + $distribution_param[] = ['name' => $text, 'value' => $i]; + } + $this->params = [ + [ + 'id' => 'scope', + 'label' => 'Scope', + 'type' => 'select', + 'options' => [ + 'attribute' => __('Final distribution of the Attribute'), + 'event' => __('Distribution of the Event'), + ], + 'default' => 'attribute', + ], + [ + 'id' => 'condition', + 'label' => 'Condition', + 'type' => 'select', + 'default' => 'equals', + 'options' => $this->operators, + ], + [ + 'id' => 'distribution', + 'label' => 'Distribution', + 'type' => 'select', + 'default' => '0', + 'options' => $distribution_param, + 'placeholder' => __('Pick a distribution'), + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + + $scope = $params['scope']['value']; + $operator = $params['condition']['value']; + $value = $params['distribution']['value']; + $data = $roamingData->getData(); + $final_distribution = $this->__getPropagatedDistribution($data['Event']); + if ($scope == 'attribute') { + $final_distribution = $this->__getPropagatedDistribution( + $data['Event'], + $data['Event']['Attribute'][0]['Object'] ?? [], + $data['Event']['Attribute'][0] + ); + } + if ($final_distribution == -1) { + return false; // distribution not supported + } + if ($operator == 'more_restrictive_or_equal_than') { + $operator = 'in'; + $distribution_range = range(0, $value); + } else if ($operator == 'more_permisive_or_equal_than') { + $operator = 'in'; + $distribution_range = range($value, 3); + } else { + $distribution_range = intval($value); + } + if ($operator == 'more_restrictive_or_equal_than' || $operator == 'more_permisive_or_equal_than') { + $distribution_range = array_diff($value, [4]); // ignore sharing_group for now + } + $eval = $this->evaluateCondition($distribution_range, $operator, $final_distribution); + return !empty($eval); + } + + /** + * __getPropagatedDistribution Get the final distribution of the attribute where distribution of its parent (events/objects) is applied + * + * @param array $event + * @param array $object + * @param array $attribute + * @return integer + */ + private function __getPropagatedDistribution(array $event, array $object=[], array $attribute=[]): int + { + $finalDistribution = 5; + if (!empty($attribute)) { + $finalDistribution = intval($attribute['distribution']); + } + if (!empty($object)) { // downgrade based on the object distribution + $finalDistribution = min($finalDistribution, intval($object['distribution'])); + } + $finalDistribution = min($finalDistribution, intval($event['distribution'])); // downgrade based on the event distribution + if (!empty($attribute)) { + if ($attribute['distribution'] == 5) { // mirror distribution for the one of the event + $attribute['distribution'] = intval($event['distribution']); + } + } + if ($finalDistribution == 4) { // ignore sharing group for now + $finalDistribution = -1; + } + return $finalDistribution; + } +} diff --git a/app/Model/WorkflowModules/logic/Module_generic_if.php b/app/Model/WorkflowModules/logic/Module_generic_if.php new file mode 100644 index 000000000..244efef53 --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_generic_if.php @@ -0,0 +1,65 @@ + 'In', + 'not_in' => 'Not in', + 'equals' => 'Equals', + 'not_equals' => 'Not equals', + ]; + + public function __construct() + { + parent::__construct(); + $this->params = [ + [ + 'id' => 'value', + 'label' => 'Value', + 'type' => 'input', + 'placeholder' => 'tlp:red', + ], + [ + 'id' => 'operator', + 'label' => 'Operator', + 'type' => 'select', + 'default' => 'in', + 'options' => $this->operators, + ], + [ + 'id' => 'hash_path', + 'label' => 'Hash path', + 'type' => 'input', + 'placeholder' => 'Attribute.{n}.Tag', + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + $path = $params['hash_path']['value']; + $operator = $params['operator']['value']; + $value = $params['value']['value']; + $data = $roamingData->getData(); + $extracted = []; + if ($operator == 'equals' || $operator == 'not_equals') { + $extracted = Hash::get($data, $path, []); + } else { + $extracted = Hash::extract($data, $path); + } + $eval = $this->evaluateCondition($extracted, $operator, $value); + return !empty($eval); + } +} diff --git a/app/Model/WorkflowModules/logic/Module_organisation_if.php b/app/Model/WorkflowModules/logic/Module_organisation_if.php new file mode 100644 index 000000000..7455b825e --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_organisation_if.php @@ -0,0 +1,76 @@ + 'Is', + 'not_equals' => 'Is not', + ]; + + public function __construct() + { + parent::__construct(); + $this->Organisation = ClassRegistry::init('Organisation'); + $orgs = $this->Organisation->find('list', [ + 'fields' => ['id', 'name'], + 'order' => 'LOWER(name)' + ]); + $this->params = [ + [ + 'id' => 'org_type', + 'label' => 'Organisation Type', + 'type' => 'select', + 'options' => [ + 'org' => __('Owner Organisation'), + 'orgc' => __('Creator Organisation'), + ], + 'default' => 'orgc', + ], + [ + 'id' => 'condition', + 'label' => 'Condition', + 'type' => 'select', + 'default' => 'equals', + 'options' => $this->operators, + ], + [ + 'id' => 'org_id', + 'type' => 'picker', + 'multiple' => false, + 'options' => $orgs, + 'default' => 1, + 'placeholder' => __('Pick an organisation'), + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + + $org_type = $params['org_type']['value']; + $operator = $params['condition']['value']; + $org_id = $params['org_id']['value']; + $data = $roamingData->getData(); + $path = 'Event.org_id'; + if ($org_type == 'orgc') { + $path = 'Event.orgc_id'; + } + $extracted_org = intval(Hash::get($data, $path)) ?? -1; + $eval = $this->evaluateCondition($extracted_org, $operator, $org_id); + return !empty($eval); + } +} diff --git a/app/Model/WorkflowModules/logic/Module_published_if.php b/app/Model/WorkflowModules/logic/Module_published_if.php new file mode 100644 index 000000000..14ce23d37 --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_published_if.php @@ -0,0 +1,47 @@ + 'Event is published', + 'not_equals' => 'Event is not published', + ]; + + public function __construct() + { + parent::__construct(); + $this->params = [ + [ + 'id' => 'condition', + 'label' => 'Condition', + 'type' => 'select', + 'default' => 'equals', + 'options' => $this->operators, + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + + $operator = $params['condition']['value']; + $data = $roamingData->getData(); + $path = 'Event.published'; + $is_published = !empty(Hash::get($data, $path)); + $eval = $this->evaluateCondition($is_published, $operator, true); + return $eval; + } +} diff --git a/app/Model/WorkflowModules/logic/Module_tag_if.php b/app/Model/WorkflowModules/logic/Module_tag_if.php new file mode 100644 index 000000000..93a7c14db --- /dev/null +++ b/app/Model/WorkflowModules/logic/Module_tag_if.php @@ -0,0 +1,94 @@ + 'Is tagged with any (OR)', + 'in_and' => 'Is tagged with all (AND)', + 'not_in_or' => 'Is not tagged with any (OR)', + 'not_in_and' => 'Is not tagged with all (AND)', + ]; + + public function __construct() + { + parent::__construct(); + $conditions = [ + 'Tag.is_galaxy' => 0, + ]; + $this->Tag = ClassRegistry::init('Tag'); + $tags = $this->Tag->find('all', [ + 'conditions' => $conditions, + 'recursive' => -1, + 'order' => ['name asc'], + 'fields' => ['Tag.id', 'Tag.name'] + ]); + $tags = array_column(array_column($tags, 'Tag'), 'name', 'id'); + $this->params = [ + [ + 'id' => 'scope', + 'label' => 'Scope', + 'type' => 'select', + 'options' => [ + 'event' => __('Event'), + 'attribute' => __('Attribute'), + 'event_attribute' => __('Inherited Attribute'), + ], + 'default' => 'event', + ], + [ + 'id' => 'condition', + 'label' => 'Condition', + 'type' => 'select', + 'default' => 'in_or', + 'options' => $this->operators, + ], + [ + 'id' => 'tags', + 'label' => 'Tags', + 'type' => 'picker', + 'multiple' => true, + 'options' => $tags, + 'placeholder' => __('Pick a tag'), + ], + ]; + } + + public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool + { + parent::exec($node, $roamingData, $errors); + $params = $this->getParamsWithValues($node); + + $value = $params['tags']['value']; + $operator = $params['condition']['value']; + $scope = $params['scope']['value']; + $data = $roamingData->getData(); + $extracted = $this->__getTagFromScope($scope, $data); + $eval = $this->evaluateCondition($extracted, $operator, $value); + return !empty($eval); + } + + private function __getTagFromScope($scope, array $data): array + { + $path = ''; + if ($scope == 'attribute') { + $path = 'Event._AttributeFlattened.{n}.Tag.{n}.id'; + } elseif ($scope == 'event_attribute') { + $path = 'Event._AttributeFlattened.{n}._allTags.{n}.id'; + } else { + $path = 'Event.Tag.{n}.id'; + } + return Hash::extract($data, $path) ?? []; + } +} diff --git a/app/Model/WorkflowModules/trigger/Module_attribute_after_save.php b/app/Model/WorkflowModules/trigger/Module_attribute_after_save.php new file mode 100644 index 000000000..c11e4c7bb --- /dev/null +++ b/app/Model/WorkflowModules/trigger/Module_attribute_after_save.php @@ -0,0 +1,22 @@ +trigger_overhead_message = __('This trigger is called each time an Attribute has been saved. This means that when a large quantity of Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Attributes.'); + } +} diff --git a/app/Model/WorkflowModules/trigger/Module_enrichment_before_query.php b/app/Model/WorkflowModules/trigger/Module_enrichment_before_query.php new file mode 100644 index 000000000..9be19d2c5 --- /dev/null +++ b/app/Model/WorkflowModules/trigger/Module_enrichment_before_query.php @@ -0,0 +1,21 @@ +trigger_overhead_message = __('This trigger is called each time an Event or Attribute have been saved. This means that when a large quantity of Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Attributes.'); + } +} diff --git a/app/Model/WorkflowModules/trigger/Module_event_publish.php b/app/Model/WorkflowModules/trigger/Module_event_publish.php new file mode 100644 index 000000000..5e131c42a --- /dev/null +++ b/app/Model/WorkflowModules/trigger/Module_event_publish.php @@ -0,0 +1,21 @@ +trigger_overhead_message = __('This trigger is called each time an Object has been saved. This means that when a large quantity of Objects are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Objects.'); + } +} diff --git a/app/Model/WorkflowModules/trigger/Module_post_after_save.php b/app/Model/WorkflowModules/trigger/Module_post_after_save.php new file mode 100644 index 000000000..ea00835ce --- /dev/null +++ b/app/Model/WorkflowModules/trigger/Module_post_after_save.php @@ -0,0 +1,43 @@ +Thread = ClassRegistry::init('Thread'); + $thread = $this->Thread->find('first', [ + 'recursive' => -1, + 'conditions' => ['id' => $data['Post']['thread_id']], + ]); + $data['Thread'] = !empty($thread) ? $thread['Thread'] : []; + if (!empty($thread) && !empty($thread['Thread']['event_id'])) { + $this->Event = ClassRegistry::init('Event'); + $event = $this->Event->find('first', [ + 'recursive' => -1, + 'conditions' => ['id' => $thread['Thread']['event_id']], + ]); + $event = $this->convertData($event); + $data['Event'] = $event['Event']; + } + return $data; + } +} diff --git a/app/Model/WorkflowModules/trigger/Module_user_after_save.php b/app/Model/WorkflowModules/trigger/Module_user_after_save.php new file mode 100644 index 000000000..17789a7e8 --- /dev/null +++ b/app/Model/WorkflowModules/trigger/Module_user_after_save.php @@ -0,0 +1,20 @@ +Log = ClassRegistry::init('Log'); $this->Log->create(); - $this->settings['fields'] = array('username' => 'email'); + $this->settings['fields'] = ['username' => 'email']; } /** @@ -108,37 +109,51 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { * @param string $level Log level * @param string $logmessage Message to log * @return bool result of the log action - */ - private function _log($level, $logmessage) + */ + private function _log($level, $logmessage) { - $log = array( + $log = [ 'org' => 'SYSTEM', 'model' => 'User', 'model_id' => 0, 'email' => false, 'action' => 'auth', 'title' => $logmessage - ); - $this->Log->save($log); + ]; + $this->Log->save($log); CakeLog::write($level, $logmessage); return true; } + /** + * Log non 200-ish HTTP responses + * + * @param string $level Log level + * @param string $url Requested URL + * @param HttpSocketResponse $response HTTP response + * @return bool result of the log action + */ + private function _logHttpError(string $level, string $url, HttpSocketResponse $response) + { + $this->_log($level, "POST request to url: {$url} returned HTTP code: {$response->code} with response body: {$response->body}"); + + return true; + } + /** * Find the user to authenticate with * * @param CakeRequest $request The request that contains login information. * @return mixed False on login failure. An array of User data on success. - */ - public function getUser(CakeRequest $request) + */ + public function getUser(CakeRequest $request) { // we only proceed if called with a request to authenticate via AAD if (array_key_exists('AzureAD', $request->query) and $request->query['AzureAD'] == 'enable') { $user = $this->_getUserAad($request); return $user; - } - elseif (array_key_exists('code', $request->query)) // in the Azure flow + } elseif (array_key_exists('code', $request->query)) // in the Azure flow { $user = $this->_getUserAad($request); return $user; @@ -152,8 +167,8 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { * @param CakeRequest $request The request that contains login information. * @param CakeResponse $response Unused response object. * @return mixed False on login failure. An array of User data on success. - */ - public function authenticate(CakeRequest $request, CakeResponse $response) + */ + public function authenticate(CakeRequest $request, CakeResponse $response) { return self::getUser($request); } @@ -163,10 +178,10 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { * * @param CakeRequest $request The request that contains login information. * @return mixed False on login failure. An array of User data on success. - */ + */ private function _getUserAad(CakeRequest $request) { - if (!headers_sent()) { + if (!headers_sent()) { if (!isset($_GET["code"]) and !isset($_GET["error"])) { $url = self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/authorize?"; $url .= "state=" . session_id(); @@ -179,68 +194,68 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { $this->_log("info", "Redirect to Azure for authentication."); exit; // we need to exit once the header to redirect to Azure is sent - } - elseif (isset($_GET["error"])) { //Second load of this page begins, but hopefully we end up to the next elseif section... - $this->_log("warning", "Return from Aure redirect. Error received at the beginning of second stage. _GET: " . http_build_query($_GET,'',' - ')); + } elseif (isset($_GET["error"])) { //Second load of this page begins, but hopefully we end up to the next elseif section... + $this->_log("warning", "Return from Aure redirect. Error received at the beginning of second stage. _GET: " . http_build_query($_GET, '', ' - ')); return false; - } - elseif (strcmp(session_id(), $_GET["state"]) == 0) { + } elseif (strcmp(session_id(), $_GET["state"]) == 0) { //Verifying the received tokens with Azure and finalizing the authentication part - $content = "grant_type=authorization_code"; - $content .= "&client_id=" . self::$client_id; - $content .= "&redirect_uri=" . urlencode(self::$redirect_uri); - $content .= "&code=" . $_GET["code"]; - $content .= "&client_secret=" . urlencode(self::$client_secret); - $options = array( - "http" => array( //Use "http" even if you send the request with https - "method" => "POST", - "header" => "Content-Type: application/x-www-form-urlencoded\r\n" . - "Content-Length: " . strlen($content) . "\r\n", - "content" => $content - ) - ); - - $context = stream_context_create($options); - $json = file_get_contents(self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/token", false, $context); - if ($json === false) { - $this->_log("warning", "Error received during Bearer token fetch (context)."); - // For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), ""); - return false; - } + $params = [ + 'grant_type' => 'authorization_code', + 'client_id' => self::$client_id, + 'redirect_uri' => self::$redirect_uri, + 'code' => $_GET["code"], + 'client_secret' => self::$client_secret + ]; - $authdata = json_decode($json, true); + $options = [ + 'header' => [ + 'Content-Type' => 'application/x-www-form-urlencoded' + ] + ]; + $url = self::$auth_provider . self::$ad_tenant . "/oauth2/v2.0/token"; + + $response = (new HttpSocket())->post($url, $params, $options); + + if (!$response->isOk()) { + $this->_log("warning", "Error received during Bearer token fetch (context)."); + $this->_logHttpError("debug", $url, $response); + return false; + } + + $authdata = json_decode($response->body, true); if (isset($authdata["error"])) { $this->_log("warning", "Error received during Bearer token fetch (authdata)."); - // For debug : "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email); + $this->_log("debug", "Response: " . json_encode($authdata["error"])); return false; } - $options = array( - "http" => array( //Use "http" even if you send the request with https - "method" => "GET", - "header" => "Accept: application/json\r\n" . - "Authorization: Bearer " . $authdata["access_token"] . "\r\n" - ) - ); - - $context = stream_context_create($options); $this->_log("info", "Fetching user data from Azure."); - $json = file_get_contents(self::$auth_provider_user . "/v1.0/me", false, $context); - if ($json === false) { + + $options = [ + 'header' => [ + 'Accept' => 'application/json', + 'Authorization' => 'Bearer ' . $authdata["access_token"] + ] + ]; + $url = self::$auth_provider_user . "/v1.0/me"; + + $response = (new HttpSocket())->get($url, null, $options); + + if (!$response->isOk()) { $this->_log("warning", "Error received during user data fetch."); - // For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email); + $this->_logHttpError("debug", $url, $response); return false; } - $userdata = json_decode($json, true); //This should now contain your logged on user information - if (isset($userdata["error"])){ + $userdata = json_decode($response->body, true); //This should now contain your logged on user information + if (isset($userdata["error"])) { $this->_log("warning", "User data fetch contained an error."); - // For debug : "\$userdata[]" => $userdata, "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email); + $this->_log("debug", "Response: " . json_encode($userdata["error"])); return false; - } + } $mispUsername = false; - if (isset($userdata["userPrincipalName"])){ + if (isset($userdata["userPrincipalName"])) { $userPrincipalName = $userdata["userPrincipalName"]; /* @@ -251,15 +266,19 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { if (self::$check_ad_groups) { if ($this->_checkAdGroup($authdata)) { $mispUsername = $userPrincipalName; + $this->_log("info", "Successful AAD group check for for ${mispUsername}"); } - } - else { + } else { $mispUsername = $userPrincipalName; } if ($mispUsername) { - $this->_log("info", "Attempt authentication for ${mispUsername}"); - return $this->_findUser($mispUsername); + $this->_log("info", "Attempt AAD authentication for ${mispUsername}"); + $user = $this->_findUser($mispUsername); + if ($user) { + $this->_log("info", "AAD authentication successful for ${mispUsername}"); + } + return $user; } } } @@ -274,39 +293,39 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { * * @param array $authdata The authdata array received from Azure * @return mixed False if no MISP groups have been found; String if a group was found - */ - private function _checkAdGroup($authdata) + */ + private function _checkAdGroup($authdata) { - $options = array( - "http" => array( //Use "http" even if you send the request with https - "method" => "GET", - "header" => "Accept: application/json\r\n" . - "Authorization: Bearer " . $authdata["access_token"] . "\r\n" - ) - ); - - $context = stream_context_create($options); $this->_log("info", "Fetching user group data from Azure."); - $json = file_get_contents(self::$auth_provider_user . "/v1.0/me/memberOf", false, $context); - if ($json === false) { - $this->_log("warning", "Error received during user group data fetch."); - // For debug : "PHP_Error" => error_get_last(), "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email); - return false; - } + $options = [ + 'header' => [ + 'Accept' => 'application/json', + 'Authorization' => 'Bearer ' . $authdata["access_token"] + ] + ]; + $url = self::$auth_provider_user . "/v1.0/me/memberOf"; - $groupdata = json_decode($json, true); //This should now contain your logged on user memberOf (groups) information + $response = (new HttpSocket())->get($url, null, $options); + + if (!$response->isOk()) { + $this->_log("warning", "Error received during user group data fetch."); + $this->_logHttpError("debug", $url, $response); + return false; + } + + $groupdata = json_decode($response->body, true); //This should now contain your logged on user memberOf (groups) information if (isset($groupdata["error"])) { $this->_log("warning", "Group data fetch contained an error."); - // For debug : "\$groupdata[]" => $groupdata, "\$authdata[]" => $authdata, "\$_GET[]" => $_GET, "HTTP_msg" => $options), $error_email); + $this->_log("debug", "Response: " . json_encode($groupdata["error"])); return false; - } + } // Now check if the user has any of the MISP AAD groups enabled foreach ($groupdata["value"] as $group) { $groupdisplayName = $group["displayName"]; if ($groupdisplayName == self::$misp_siteadmin) { return self::$misp_siteadmin; - } + } if ($groupdisplayName == self::$misp_orgadmin) { return self::$misp_orgadmin; } @@ -315,7 +334,8 @@ class AadAuthenticateAuthenticate extends BaseAuthenticate { } } + $this->_log("warning", "The user is not a member of any of the MISP AAD groups."); + return false; } - } diff --git a/app/Plugin/AadAuth/README.md b/app/Plugin/AadAuth/README.md index 0a4fdfb06..d5e9447eb 100755 --- a/app/Plugin/AadAuth/README.md +++ b/app/Plugin/AadAuth/README.md @@ -128,4 +128,15 @@ Add the information we made a note of earlier when creating the `App Registation All fields need to match explicitly without any leading or trailing whitespace, if you added the groups these are case sensitive. +### Disable users password change +By default MISP will still create a password for the user, when enrolling a new user on MISP, uncheck the _"Send credentials automatically"_ checkbox. +![Send credentials automatically](.images/Picture39.png) + +Additionally, it is recommended to set the following settings in the MISP config: + +* `MISP.disableUserSelfManagement => true`: Removes the ability of users to change their user settings and reset their authentication keys. +* `MISP.disable_user_login_change => true`: Removes the ability of users to change their username (email), except for site admins. +* `MISP.disable_user_password_change => true`: Removes the ability of users to change their own password. + +This way users will not be able to change their passwords and by-pass the AAD login flow. \ No newline at end of file diff --git a/app/View/Attributes/ajax/recorrelationConfirmation.ctp b/app/View/Attributes/ajax/recorrelationConfirmation.ctp new file mode 100644 index 000000000..aaa7d5c92 --- /dev/null +++ b/app/View/Attributes/ajax/recorrelationConfirmation.ctp @@ -0,0 +1,26 @@ +
+ Form->create('Attribute', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/attributes/generateCorrelation']); + $message = __('Recorrelate instance'); + $buttonTitle = __('Recorrelate'); + ?> + +
+ ' . __('Are you sure you wish to start a recorrelation for the currently active correlation engine?') . '

'; + echo '

' . __('Depending on the system and the amount of attributes, this might take a long time.') . '

'; + ?> + + + + + + +
+ + + +
+
+ Form->end(); ?> +
diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index 5dd7695c8..a4cbf350a 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -2,7 +2,6 @@ $modules = isset($modules) ? $modules : null; $cortex_modules = isset($cortex_modules) ? $cortex_modules : null; - echo '
'; echo $this->element('/genericElements/IndexTable/index_table', [ 'data' => [ diff --git a/app/View/AuditLogs/admin_index.ctp b/app/View/AuditLogs/admin_index.ctp index 35e1a2749..81c556d48 100644 --- a/app/View/AuditLogs/admin_index.ctp +++ b/app/View/AuditLogs/admin_index.ctp @@ -228,20 +228,20 @@ - - - - - + + + + + @@ -287,10 +287,6 @@
Paginator->sort('created') ?>Paginator->sort('user_id', __('User')) ?>Paginator->sort('ip', __('IP')) ?>Paginator->sort('org_id', __('Org')) ?>Paginator->sort('action') ?>LightPaginator->sort('created') ?>LightPaginator->sort('user_id', __('User')) ?>LightPaginator->sort('ip', __('IP')) ?>LightPaginator->sort('org_id', __('Org')) ?>LightPaginator->sort('action') ?>

- Paginator->counter(array( - 'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}') - )); - ?>

'; + if (empty($ajax)) { + echo $this->element('/genericElements/SideMenu/side_menu', $menuData); + } + + diff --git a/app/View/Correlations/top.ctp b/app/View/Correlations/top.ctp index 29f2bc3e4..1176f32d0 100644 --- a/app/View/Correlations/top.ctp +++ b/app/View/Correlations/top.ctp @@ -2,7 +2,7 @@ echo sprintf('', empty($ajax) ? ' class="index"' : ''); echo $this->element('genericElements/IndexTable/index_table', [ 'data' => [ - 'stupid_pagination' => 1, + 'light_paginator' => 1, 'data' => $data, 'top_bar' => [ 'children' => [ diff --git a/app/View/DecayingModel/decaying_tool.ctp b/app/View/DecayingModel/decaying_tool.ctp index 8098f55e7..80e1887da 100644 --- a/app/View/DecayingModel/decaying_tool.ctp +++ b/app/View/DecayingModel/decaying_tool.ctp @@ -167,7 +167,7 @@ element('genericElements/assetLoader', array( 'css' => array('decayingTool'), - 'js' => array('d3', 'Chart.min', 'decayingTool') + 'js' => array('d3', 'Chart.min', 'decayingTool', 'jquery-ui.min') )); ?> diff --git a/app/View/Elements/Events/View/attribute_correlations.ctp b/app/View/Elements/Events/View/attribute_correlations.ctp index 262c863b9..5cb188daf 100644 --- a/app/View/Elements/Events/View/attribute_correlations.ctp +++ b/app/View/Elements/Events/View/attribute_correlations.ctp @@ -1,57 +1,81 @@ $relatedAttribute) { - if (isset($relatedEvents[$relatedAttribute['id']])) { - unset($event['Related' . $scope][$object['id']][$k]); - } else { - $relatedEvents[$relatedAttribute['id']] = true; - } -} -$count = count($event['Related' . $scope][$object['id']]); -foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) { - if ($i == 4 && $count > 5) { - $expandButton = __('Show %s more...', $count - 4); - echo sprintf( - '
  • %s
    • ', - $linkColour, - $expandButton - ); - } - $relatedData = array( - 'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A', - 'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A', - 'Event' => $relatedAttribute['info'], - 'Correlating Value' => $relatedAttribute['value'] - ); - $popover = ''; - foreach ($relatedData as $k => $v) { - $popover .= '' . h($k) . ': ' . h($v) . '
    '; - } - $link = $this->Html->link( - $relatedAttribute['id'], - $withPivot ? - ['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] : - ['controller' => 'events', 'action' => 'view', $relatedAttribute['id']], - ['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue'] - ); - echo sprintf( - '
  • %s 
    • ', - ($i > 4 || $i == 4 && $count > 5) ? 'correlation-expanded-area' : '', - ($i > 4 || $i == 4 && $count > 5) ? 'style="display:none;"' : '', - h($popover), - $link - ); + if (!empty($event['Related' . $scope][$object['id']])) { + $i = 0; + $linkColour = ($scope == 'Attribute') ? 'red' : 'white'; + $withPivot = isset($withPivot) ? $withPivot : false; + // remove duplicates + $relatedEvents = array(); + foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) { + if (isset($relatedEvents[$relatedAttribute['id']])) { + unset($event['Related' . $scope][$object['id']][$k]); + } else { + $relatedEvents[$relatedAttribute['id']] = true; + } + } + $count = count($event['Related' . $scope][$object['id']]); + foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) { + if ($i == 4 && $count > 5) { + $expandButton = __('Show %s more...', $count - 4); + echo sprintf( + '
  • %s
    • ', + $linkColour, + $expandButton + ); + } + $relatedData = array( + 'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A', + 'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A', + 'Event' => $relatedAttribute['info'], + 'Correlating Value' => $relatedAttribute['value'] + ); + $popover = ''; + foreach ($relatedData as $k => $v) { + $popover .= '' . h($k) . ': ' . h($v) . '
    '; + } + $link = $this->Html->link( + $relatedAttribute['id'], + $withPivot ? + ['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] : + ['controller' => 'events', 'action' => 'view', $relatedAttribute['id']], + ['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue'] + ); + echo sprintf( + '
  • %s 
    • ', + ($i > 4 || $i == 4 && $count > 5) ? 'correlation-expanded-area' : '', + ($i > 4 || $i == 4 && $count > 5) ? 'style="display:none;"' : '', + h($popover), + $link + ); - $i++; -} -if ($i > 5) { - echo sprintf( - '', - $linkColour, - __('Collapse…') - ); -} + $i++; + } + if ($i > 5) { + echo sprintf( + '', + $linkColour, + __('Collapse…') + ); + } + } + if (!empty($object['correlation_exclusion'])) { + echo sprintf( + '%s ', + __('The attribute value matches a correlation exclusion rule defined by a site-administrator for this instance. Click the magnifying glass to search for all occurrences of the value.'), + __('Excluded.') + ); + } else if (!empty($object['over_correlation'])) { + echo sprintf( + '%s ', + __('The instance threshold for the maximum number of correlations for the given attribute value has been exceeded. Click the magnifying glass to search for all occurrences of the value.'), + __('Too many correlations.') + ); + } + echo $this->Html->link( + '', + ['controller' => 'attributes', 'action' => 'search', 'value' => $object['value']], + [ + 'class' => 'fa fa-search black', + 'title' => __('Search for value'), + 'aria-label' => __('Search for value') + ] + ); diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp index 177678146..1ba949f61 100644 --- a/app/View/Elements/Events/View/row_attribute.ctp +++ b/app/View/Elements/Events/View/row_attribute.ctp @@ -1,396 +1,396 @@ - - - - - + ?> + + + + + + + + + + + element('/Events/View/seen_field', array('object' => $object)); ?> - - - - - element('/Events/View/seen_field', array('object' => $object)); ?> - - Time->date($object['timestamp']) ?> - - - - ' . h($object['event_id']) . ''; ?> - - - + Time->date($object['timestamp']) ?> OrgImg->getOrgLogo($extensionOrg, 24); - else: - echo $this->OrgImg->getOrgLogo($event['Orgc'], 24); - endif; - endif; ?> - - > -
    - -
    - - > - -
    :
    - -
    - -
    - - > -
    - element('/Events/View/value_field', array('object' => $object)); - if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) { - $commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId); - $spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields); - $popupButton = sprintf('', __('Show hover enrichment'), $commonDataFields); - echo sprintf( - '%s %s', - $spanExtra, - $value, - $popupButton - ); - } else { - echo $value; - } - ?> -
    - - -
    - element('ajaxTags', array( - 'attributeId' => $objectId, - 'tags' => $object['AttributeTag'], - 'tagAccess' => ($isSiteAdmin || $mayModify), - 'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')), - 'context' => $context, - 'scope' => 'attribute', - 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array() - ) - ); ?> -
    - - element('ajaxAttributeTags', array('attributeId' => $objectId, 'attributeTags' => $object['RelatedTags'], 'tagAccess' => false)); - } - echo sprintf( - '
    %s
    ', - 'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"', - $element - ); - } - ?> - - element('galaxyQuickViewNew', array( - 'mayModify' => $mayModify, - 'isAclTagger' => $isAclTagger, - 'data' => (!empty($object['Galaxy']) ? $object['Galaxy'] : array()), - 'event' => $event, - 'target_id' => $objectId, - 'target_type' => 'attribute', - )); - ?> - - > -
    - -
    - - - - > - - - '; - echo $this->element('Events/View/attribute_correlations', array( - 'scope' => 'Attribute', - 'object' => $object, - 'event' => $event, - 'withPivot' => true, - )); - echo ''; - } - ?> - - -
      - h($feed['name']), - __('Provider') => h($feed['provider']), - ); - if (isset($feed['event_uuids'])) { - $relatedData[__('Event UUIDs')] = implode('
      ', array_map('h', $feed['event_uuids'])); - } - $popover = ''; - foreach ($relatedData as $k => $v) { - $popover .= '' . $k . ': ' . $v . '
      '; - } - if ($isSiteAdmin || $hostOrgUser) { - if ($feed['source_format'] === 'misp') { - $liContents = sprintf( - '
      %s%s
      ', - $baseurl, - h($feed['id']), - sprintf( - '', - h(json_encode($feed['event_uuids'])) - ), - sprintf( - '', - h($feed['id']), - h($popover) - ) - ); - } else { - $liContents = sprintf( - '%s', - $baseurl, - h($feed['id']), - h($popover), - h($feed['id']) - ); - } - } else { - $liContents = sprintf( - '%s', - h($feed['id']) - ); - } - echo "
    • $liContents
      • "; - } - } - if (isset($object['Server'])) { - foreach ($object['Server'] as $server) { - $popover = ''; - foreach ($server as $k => $v) { - if ($k == 'id') continue; - if (is_array($v)) { - foreach ($v as $k2 => $v2) { - $v[$k2] = h($v2); - } - $v = implode('
      ', $v); - } else { - $v = h($v); - } - $popover .= '' . Inflector::humanize(h($k)) . ': ' . $v . '
      '; - } - foreach ($server['event_uuids'] as $k => $event_uuid) { - $liContents = ''; - if ($isSiteAdmin) { - $liContents .= sprintf( - '%s ', - $baseurl, - h($server['id']), - h($event_uuid), - h($popover), - 'S' . h($server['id']) . ':' . ($k + 1) - ); - } else { - $liContents .= sprintf( - '%s', - 'S' . h($server['id']) . ':' . ($k + 1) - ); - } - echo "
    • $liContents
      • "; - } - } - } - ?> -
    - - - > - - > -
    - - - -
    - - element('/Events/View/sighting_field', array( - 'object' => $object, - )); - if (!empty($includeSightingdb)) { - echo $this->element('/Events/View/sightingdb_field', array( - 'object' => $object, - )); - } - if (!empty($includeDecayScore)): ?> - -
    - element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?> -
    - - - - - - - -   - - - - - - -   - -   - - - - + - - ' . h($object['event_id']) . ''; ?> + + + + OrgImg->getOrgLogo($extensionOrg, 24); else: - ?> - - OrgImg->getOrgLogo($event['Orgc'], 24); endif; endif; - endif; - ?> - - - $proposal) { - echo $this->element('/Events/View/row_' . $proposal['objectType'], array( - 'object' => $proposal, + ?> + + > +
    + +
    + + > + +
    :
    + +
    + +
    + + > +
    + element('/Events/View/value_field', array('object' => $object)); + if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) { + $commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId); + $spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields); + $popupButton = sprintf('', __('Show hover enrichment'), $commonDataFields); + echo sprintf( + '%s %s', + $spanExtra, + $value, + $popupButton + ); + } else { + echo $value; + } + ?> +
    + + +
    + element('ajaxTags', array( + 'attributeId' => $objectId, + 'tags' => $object['AttributeTag'], + 'tagAccess' => ($isSiteAdmin || $mayModify), + 'localTagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id'] || (int)$me['org_id'] === Configure::read('MISP.host_org_id')), + 'context' => $context, + 'scope' => 'attribute', + 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array() + ) + ); ?> +
    + + element('ajaxAttributeTags', array('attributeId' => $objectId, 'attributeTags' => $object['RelatedTags'], 'tagAccess' => false)); + } + echo sprintf( + '
    %s
    ', + 'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"', + $element + ); + } + ?> + + element('galaxyQuickViewNew', array( 'mayModify' => $mayModify, - 'mayChangeCorrelation' => $mayChangeCorrelation, - 'fieldCount' => $fieldCount, - 'child' => $propKey == $lastElement ? 'last' : true, - 'objectContainer' => $child - )); - } -} + 'isAclTagger' => $isAclTagger, + 'data' => (!empty($object['Galaxy']) ? $object['Galaxy'] : array()), + 'event' => $event, + 'target_id' => $objectId, + 'target_type' => 'attribute', + )); + ?> + + > +
    + +
    + + + + > + + + '; + echo $this->element('Events/View/attribute_correlations', array( + 'scope' => 'Attribute', + 'object' => $object, + 'event' => $event, + 'withPivot' => true, + )); + echo ''; + //} + ?> + + +
      + h($feed['name']), + __('Provider') => h($feed['provider']), + ); + if (isset($feed['event_uuids'])) { + $relatedData[__('Event UUIDs')] = implode('
      ', array_map('h', $feed['event_uuids'])); + } + $popover = ''; + foreach ($relatedData as $k => $v) { + $popover .= '' . $k . ': ' . $v . '
      '; + } + if ($isSiteAdmin || $hostOrgUser) { + if ($feed['source_format'] === 'misp') { + $liContents = sprintf( + '
      %s%s
      ', + $baseurl, + h($feed['id']), + sprintf( + '', + h(json_encode($feed['event_uuids'])) + ), + sprintf( + '', + h($feed['id']), + h($popover) + ) + ); + } else { + $liContents = sprintf( + '%s', + $baseurl, + h($feed['id']), + h($popover), + h($feed['id']) + ); + } + } else { + $liContents = sprintf( + '%s', + h($feed['id']) + ); + } + echo "
    • $liContents
      • "; + } + } + if (isset($object['Server'])) { + foreach ($object['Server'] as $server) { + $popover = ''; + foreach ($server as $k => $v) { + if ($k == 'id') continue; + if (is_array($v)) { + foreach ($v as $k2 => $v2) { + $v[$k2] = h($v2); + } + $v = implode('
      ', $v); + } else { + $v = h($v); + } + $popover .= '' . Inflector::humanize(h($k)) . ': ' . $v . '
      '; + } + foreach ($server['event_uuids'] as $k => $event_uuid) { + $liContents = ''; + if ($isSiteAdmin) { + $liContents .= sprintf( + '%s ', + $baseurl, + h($server['id']), + h($event_uuid), + h($popover), + 'S' . h($server['id']) . ':' . ($k + 1) + ); + } else { + $liContents .= sprintf( + '%s', + 'S' . h($server['id']) . ':' . ($k + 1) + ); + } + echo "
    • $liContents
      • "; + } + } + } + ?> +
    + + + > + + > +
    + + + +
    + + element('/Events/View/sighting_field', array( + 'object' => $object, + )); + if (!empty($includeSightingdb)) { + echo $this->element('/Events/View/sightingdb_field', array( + 'object' => $object, + )); + } + if (!empty($includeDecayScore)): ?> + +
    + element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?> +
    + + + + + + + +   + + + + + + +   + +   + + + + + + + + + + + + $proposal) { + echo $this->element('/Events/View/row_' . $proposal['objectType'], array( + 'object' => $proposal, + 'mayModify' => $mayModify, + 'mayChangeCorrelation' => $mayChangeCorrelation, + 'fieldCount' => $fieldCount, + 'child' => $propKey == $lastElement ? 'last' : true, + 'objectContainer' => $child + )); + } + } diff --git a/app/View/Elements/Workflows/executionPath.ctp b/app/View/Elements/Workflows/executionPath.ctp new file mode 100644 index 000000000..65acf27c1 --- /dev/null +++ b/app/View/Elements/Workflows/executionPath.ctp @@ -0,0 +1,205 @@ +
    +
    +
    +
    +
    +
    +
    + +element('genericElements/assetLoader', [ + 'css' => ['drawflow.min', 'drawflow-default'], + 'js' => ['drawflow.min', 'doT'], +]); +?> + + + + + \ No newline at end of file diff --git a/app/View/Elements/Workflows/infoModal.ctp b/app/View/Elements/Workflows/infoModal.ctp new file mode 100644 index 000000000..a343c14a6 --- /dev/null +++ b/app/View/Elements/Workflows/infoModal.ctp @@ -0,0 +1,183 @@ + \ No newline at end of file diff --git a/app/View/Elements/Workflows/sidebar-block-workflow-blueprint.ctp b/app/View/Elements/Workflows/sidebar-block-workflow-blueprint.ctp new file mode 100644 index 000000000..b2f6e2d7d --- /dev/null +++ b/app/View/Elements/Workflows/sidebar-block-workflow-blueprint.ctp @@ -0,0 +1,52 @@ + [], 'icon_path' => []]; +foreach ($workflowBlueprint['data'] as $node) { + $moduleData = $node['data']['module_data']; + if (!empty($moduleData['icon'])) { + if (!isset($iconCount['icon'][$moduleData['icon']])) { + $iconClasses = sprintf('%s %s', $this->FontAwesome->getClass($moduleData['icon']), $moduleData['icon_class'] ?? ''); + $iconCount['icon'][$iconClasses]['count'] = 0; + $iconCount['icon'][$iconClasses]['id'] = $node['data']['id']; + } + $iconCount['icon'][$iconClasses]['count'] += 1; + } elseif (!empty($moduleData['icon_path'])) { + if (!isset($iconCount['icon_path'][$moduleData['icon_path']])) { + $iconCount['icon_path'][$moduleData['icon_path']]['count'] = 0; + $iconCount['icon_path'][$moduleData['icon_path']]['id'] = $node['data']['id']; + } + $iconCount['icon_path'][$moduleData['icon_path']]['count'] += 1; + } +} +?> + \ No newline at end of file diff --git a/app/View/Elements/Workflows/sidebar-block.ctp b/app/View/Elements/Workflows/sidebar-block.ctp new file mode 100644 index 000000000..f3b691c3e --- /dev/null +++ b/app/View/Elements/Workflows/sidebar-block.ctp @@ -0,0 +1,57 @@ + 'info', + 'warning' => 'warning', + 'error' => 'danger', +]; +?> + \ No newline at end of file diff --git a/app/View/Elements/genericElements/GlobalMenu/global_menu_single.ctp b/app/View/Elements/genericElements/GlobalMenu/global_menu_single.ctp index 8fb0e018c..cf378889e 100644 --- a/app/View/Elements/genericElements/GlobalMenu/global_menu_single.ctp +++ b/app/View/Elements/genericElements/GlobalMenu/global_menu_single.ctp @@ -21,7 +21,7 @@ echo sprintf( '
  • %s%s
    • ', $data['url'], - (empty($data['html']) ? '' : h($data['html'])), + (empty($data['html']) ? '' : $data['html']), (empty($data['text']) ? '' : h($data['text'])) ); } diff --git a/app/View/Elements/genericElements/IndexTable/Fields/actions.ctp b/app/View/Elements/genericElements/IndexTable/Fields/actions.ctp index 8d6b96589..2936f18db 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/actions.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/actions.ctp @@ -80,6 +80,9 @@ } $url .= '/' . $url_params_values; } + if (!empty($action['url_suffix'])) { + $url .= $action['url_suffix']; + } if (!empty($action['url_extension'])) { $url .= '.' . $action['url_extension']; } diff --git a/app/View/Elements/genericElements/IndexTable/Fields/boolean.ctp b/app/View/Elements/genericElements/IndexTable/Fields/boolean.ctp index 56e6bdc82..f1835324a 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/boolean.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/boolean.ctp @@ -63,7 +63,7 @@ sprintf( ' (%s)', __('Filter rules'), - $rules_raw, + h($rules_raw), __('Rules') ) ); diff --git a/app/View/Elements/genericElements/IndexTable/Fields/booleanOrNA.ctp b/app/View/Elements/genericElements/IndexTable/Fields/booleanOrNA.ctp index 5653d08e6..b4fd1dd53 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/booleanOrNA.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/booleanOrNA.ctp @@ -16,8 +16,16 @@ if (is_null($flag)) { $aria = __('No'); } +$classes = ['fa', "fa-$icon"]; +if (!empty($field['colors'])) { + $classes[] = $icon == 'check' ? 'green' : 'grey'; +} else { + $classes[] = 'black'; +} + echo sprintf( - '%s', - $icon, $aria, $text + '%s', + implode(' ', $classes), + $aria, $text ); ?> diff --git a/app/View/Elements/genericElements/IndexTable/Fields/checkbox_action.ctp b/app/View/Elements/genericElements/IndexTable/Fields/checkbox_action.ctp index 073fc5b71..e4dbd2e0e 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/checkbox_action.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/checkbox_action.ctp @@ -1,16 +1,22 @@
    ', empty($field['checkbox_container']) ? 'GenericCheckboxContainer' : h($field['checkbox_container']), empty($field['checkbox_name']) ? 'GenericCheckbox' : h($field['checkbox_name']), diff --git a/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp b/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp index f0aa02506..504a8aeda 100644 --- a/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp +++ b/app/View/Elements/genericElements/IndexTable/Fields/relatedEvents.ctp @@ -1,18 +1,20 @@ $object['RelatedAttribute']); -} - -if (!empty($event['RelatedAttribute'][$object['id']])) { - echo '
      '; - echo $this->element('Events/View/attribute_correlations', array( - 'scope' => $field['data']['scope'], - 'object' => $object, - 'event' => $event, - )); - echo '
    '; -} + $object = Hash::extract($row, $field['data']['object']['value_path']); + $event = Hash::extract($row, 'Event'); + if (!empty($object['RelatedAttribute'])) { + $event['RelatedAttribute'] = array($object['id'] => $object['RelatedAttribute']); + } + foreach ($event['RelatedAttribute'] as $k => &$ra) { + if (!empty($ra['Event'])) { + $ra['info'] = $ra['Event']['info']; + $ra['org_id'] = $ra['Event']['org_id']; + } + } + echo sprintf( + '
      %s
    ', + $this->element('Events/View/attribute_correlations', [ + 'scope' => $field['data']['scope'], + 'object' => $object, + 'event' => $event, + ]) + ); diff --git a/app/View/Elements/genericElements/IndexTable/index_table.ctp b/app/View/Elements/genericElements/IndexTable/index_table.ctp index e1b985a5b..004371ebc 100644 --- a/app/View/Elements/genericElements/IndexTable/index_table.ctp +++ b/app/View/Elements/genericElements/IndexTable/index_table.ctp @@ -16,8 +16,12 @@ * )); * */ + $iconHtml = ''; + if (!empty($data['icon'])) { + $iconHtml = sprintf(' ', $this->FontAwesome->getClass($data['icon'])); + } if (!empty($data['title'])) { - echo sprintf('

    %s

    ', h($data['title'])); + echo sprintf('

    %s%s

    ', $iconHtml, h($data['title'])); } if (!empty($data['description'])) { echo sprintf( @@ -35,20 +39,21 @@ } } } + $Paginator = $this->Paginator; + if (!empty($data['light_paginator'])) { + $Paginator = $this->LightPaginator; + } $paginationData = !empty($data['paginatorOptions']) ? $data['paginatorOptions'] : []; if ($ajax && isset($containerId)) { $paginationData['data-paginator'] = "#{$containerId}_content"; } - $this->Paginator->options($paginationData); - $skipPagination = !empty($data['skip_pagination']) || !empty($data['stupid_pagination']); + $Paginator->options($paginationData); + $skipPagination = !empty($data['skip_pagination']); if (!$skipPagination) { - $paginatonLinks = $this->element('/genericElements/IndexTable/pagination_links'); + $paginatonLinks = $this->element('/genericElements/IndexTable/pagination_links', ['options' => ['paginator' => $Paginator]]); echo $paginatonLinks; } - if (!empty($data['stupid_pagination'])) { - echo $this->element('/genericElements/IndexTable/stupid_pagination_links'); - } $hasSearch = false; if (!empty($data['top_bar'])) { foreach ($data['top_bar']['children'] as $child) { @@ -97,7 +102,7 @@ ); echo ''; if (!$skipPagination) { - echo $this->element('/genericElements/IndexTable/pagination_counter', $paginationData); + echo $this->element('/genericElements/IndexTable/pagination_counter', ['options' => ['paginator' => $Paginator]]); echo $paginatonLinks; } $url = $baseurl . '/' . $this->params['controller'] . '/' . $this->params['action']; diff --git a/app/View/Elements/genericElements/IndexTable/pagination_counter.ctp b/app/View/Elements/genericElements/IndexTable/pagination_counter.ctp index 45615167f..fe7e25a17 100644 --- a/app/View/Elements/genericElements/IndexTable/pagination_counter.ctp +++ b/app/View/Elements/genericElements/IndexTable/pagination_counter.ctp @@ -1,3 +1,8 @@ -

    Paginator->counter(array( +

    +Paginator; +echo $Paginator->counter(array( 'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}') -)); ?>

    +)); +?> +

    diff --git a/app/View/Elements/genericElements/IndexTable/pagination_links.ctp b/app/View/Elements/genericElements/IndexTable/pagination_links.ctp index ecf82479c..3b74e6b94 100644 --- a/app/View/Elements/genericElements/IndexTable/pagination_links.ctp +++ b/app/View/Elements/genericElements/IndexTable/pagination_links.ctp @@ -1,19 +1,20 @@ Paginator; echo sprintf( '', - $this->Paginator->first( + $Paginator->first( '« ' . __('first'), ['tag' => 'li', 'escape' => false], null, ['tag' => 'li', 'class' => 'pagination_link first disabled', 'escape' => false, 'disabledTag' => 'span'] ), - $this->Paginator->prev( + $Paginator->prev( '« ' . __('previous'), ['tag' => 'li', 'escape' => false], null, ['tag' => 'li', 'class' => 'pagination_link prev disabled', 'escape' => false, 'disabledTag' => 'span'] ), - $this->Paginator->numbers( + $Paginator->numbers( [ 'modulus' => 6, 'separator' => '', @@ -23,13 +24,13 @@ 'class' => 'pagination_link' ] ), - $this->Paginator->next( + $Paginator->next( __('next') . ' »', ['tag' => 'li', 'escape' => false], null, ['tag' => 'li', 'class' => 'pagination_link next disabled', 'escape' => false, 'disabledTag' => 'span'] ), - $this->Paginator->last( + $Paginator->last( __('last') . ' »', ['tag' => 'li', 'escape' => false], null, diff --git a/app/View/Elements/genericElements/IndexTable/stupid_pagination_links.ctp b/app/View/Elements/genericElements/IndexTable/stupid_pagination_links.ctp deleted file mode 100644 index fd601e801..000000000 --- a/app/View/Elements/genericElements/IndexTable/stupid_pagination_links.ctp +++ /dev/null @@ -1,32 +0,0 @@ - 1) { - $prev = preg_replace('/\/page:[0-9]+/i', '/page:' . ($current_page - 1), $url); - } - $next = preg_replace('/\/page:[0-9]+/i', '/page:' . ($current_page + 1), $url); - if ($prev) { - $prev = sprintf( - '', - h($prev) - ); - } else { - $prev = ''; - } - $next = sprintf( - '', - h($next) - ); - echo sprintf( - '', - $prev, - $next - ); -?> diff --git a/app/View/Elements/genericElements/SideMenu/side_menu.ctp b/app/View/Elements/genericElements/SideMenu/side_menu.ctp index f145df8e3..a4956d7a2 100644 --- a/app/View/Elements/genericElements/SideMenu/side_menu.ctp +++ b/app/View/Elements/genericElements/SideMenu/side_menu.ctp @@ -499,7 +499,13 @@ $divider = $this->element('/genericElements/SideMenu/side_menu_divider'); 'url' => $baseurl . '/correlations/top', 'text' => __('Top Correlations') )); + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'over', + 'url' => $baseurl . '/correlations/overCorrelations', + 'text' => __('Over-correlating Values') + )); break; + case 'warninglist': if ($menuItem === 'view' || $menuItem === 'edit') { echo $this->element('/genericElements/SideMenu/side_menu_link', array( @@ -1597,7 +1603,7 @@ $divider = $this->element('/genericElements/SideMenu/side_menu_divider'); } break; - case 'api': + case 'api': echo $this->element('/genericElements/SideMenu/side_menu_link', array( 'element_id' => 'openapi', 'url' => $baseurl . '/api/openapi', @@ -1611,6 +1617,97 @@ $divider = $this->element('/genericElements/SideMenu/side_menu_divider'); )); } break; + + case 'workflowBlueprints': + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index', + 'url' => '/workflowBlueprints/index', + 'text' => __('List Workflow Blueprints') + )); + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'url' => $baseurl . '/workflowBlueprints/import', + 'text' => __('Import Workflow Blueprints') + )); + if ($isSiteAdmin && ($menuItem === 'view' || $menuItem === 'index')) { + echo $this->element('/genericElements/SideMenu/side_menu_post_link', array( + 'url' => $baseurl . '/workflowBlueprints/update', + 'text' => __('Update Default Blueprints') + )); + echo $this->element('/genericElements/SideMenu/side_menu_post_link', array( + 'url' => $baseurl . '/workflowBlueprints/update/true', + 'text' => __('Force Update Default Blueprints') + )); + } + if ($menuItem === 'view' || $menuItem === 'edit') { + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'view', + 'url' => '/workflowBlueprints/view/' . h($id), + 'text' => __('View Workflow Blueprint') + )); + if ($isSiteAdmin) { + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'edit', + 'url' => '/workflows/edit/' . h($id), + 'text' => __('Edit Workflow Blueprint') + )); + } + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'url' => '/admin/audit_logs/index/model:WorkflowBlueprints/model_id:' . h($id), + 'text' => __('View worflow blueprint history'), + 'requirement' => Configure::read('MISP.log_new_audit') && $canAccess('auditLogs', 'admin_index'), + )); + } + echo $divider; + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index_trigger', + 'url' => '/workflows/triggers', + 'text' => __('List Triggers') + )); + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index_module', + 'url' => '/workflows/moduleIndex', + 'text' => __('List Modules') + )); + break; + + case 'workflows': + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index_trigger', + 'url' => '/workflows/triggers', + 'text' => __('List Triggers') + )); + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index_module', + 'url' => '/workflows/moduleIndex', + 'text' => __('List Modules') + )); + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'index', + 'url' => '/workflowBlueprints/index', + 'text' => __('List Workflow Blueprints') + )); + if ($menuItem === 'view' || $menuItem === 'edit') { + echo $divider; + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'view', + 'url' => '/workflows/view/' . h($id), + 'text' => __('View Workflow') + )); + if ($isSiteAdmin) { + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'element_id' => 'edit', + 'url' => '/workflows/edit/' . h($id), + 'text' => __('Edit Workflow') + )); + } + echo $this->element('/genericElements/SideMenu/side_menu_link', array( + 'url' => '/admin/audit_logs/index/model:Workflow/model_id:' . h($id), + 'text' => __('View worflow history'), + 'requirement' => Configure::read('MISP.log_new_audit') && $canAccess('auditLogs', 'admin_index'), + )); + } + break; + } ?> diff --git a/app/View/Elements/genericElements/SingleViews/Fields/jsonField.ctp b/app/View/Elements/genericElements/SingleViews/Fields/jsonField.ctp index c67f8f928..638a0a3b1 100644 --- a/app/View/Elements/genericElements/SingleViews/Fields/jsonField.ctp +++ b/app/View/Elements/genericElements/SingleViews/Fields/jsonField.ctp @@ -4,9 +4,10 @@ if (is_array($value) && count($value) === 1 && isset($value[0])) { $value = $value[0]; } + $containerClassSuffix = Inflector::variable(h($field['key'])); echo sprintf( '
    ', - h($field['key']) + $containerClassSuffix ); if (is_string($value)) { $value = json_decode($value); @@ -14,6 +15,6 @@ ?> diff --git a/app/View/Elements/genericElements/SingleViews/Fields/markdownField.ctp b/app/View/Elements/genericElements/SingleViews/Fields/markdownField.ctp new file mode 100644 index 000000000..b9bbf59e1 --- /dev/null +++ b/app/View/Elements/genericElements/SingleViews/Fields/markdownField.ctp @@ -0,0 +1,61 @@ +element('genericElements/assetLoader', [ + 'js' => [ + 'markdown-it', + 'mermaid', + ], +]); +$invalidMarkdown = substr_count($markdown, PHP_EOL) <= 2; +?> + +
    + +
    + +
    + + + + diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp index de9b49f14..9f90233b2 100755 --- a/app/View/Elements/global_menu.ctp +++ b/app/View/Elements/global_menu.ctp @@ -422,6 +422,19 @@ 'type' => 'separator', 'requirement' => Configure::read('MISP.enableEventBlocklisting') !== false && $isSiteAdmin ), + array( + 'html' => sprintf( + '%s%s', + __('Workflows'), + __('new') + ), + 'url' => $baseurl . '/workflows/triggers', + 'requirement' => $isSiteAdmin + ), + array( + 'type' => 'separator', + 'requirement' => Configure::read('MISP.enableEventBlocklisting') !== false && $isSiteAdmin + ), array( 'text' => __('Blocklist Event'), 'url' => $baseurl . '/eventBlocklists/add', @@ -454,6 +467,15 @@ 'text' => __('Top Correlations'), 'url' => $baseurl . '/correlations/top', 'requirement' => $isSiteAdmin + ], + [ + 'html' => sprintf( + '%s%s', + __('Over-correlating values'), + __('new') + ), + 'url' => $baseurl . '/correlations/overCorrelations', + 'requirement' => $isSiteAdmin ] ) ), @@ -539,7 +561,7 @@ ); } ?> -").css({overflow:"hidden"}),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++e.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize;return{left:this.originalPosition.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize;return{top:this.originalPosition.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(t,e,i){return _.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},sw:function(t,e,i){return _.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[t,e,i]))},ne:function(t,e,i){return _.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},nw:function(t,e,i){return _.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[t,e,i]))}},_propagate:function(t,e){_.ui.plugin.call(this,t,[e,this.ui()]),"resize"!==t&&this._trigger(t,e,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),_.ui.plugin.add("resizable","animate",{stop:function(e){var i=_(this).resizable("instance"),t=i.options,s=i._proportionallyResizeElements,n=s.length&&/textarea/i.test(s[0].nodeName),o=n&&i._hasScroll(s[0],"left")?0:i.sizeDiff.height,r=n?0:i.sizeDiff.width,n={width:i.size.width-r,height:i.size.height-o},r=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,o=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(_.extend(n,o&&r?{top:o,left:r}:{}),{duration:t.animateDuration,easing:t.animateEasing,step:function(){var t={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};s&&s.length&&_(s[0]).css({width:t.width,height:t.height}),i._updateCache(t),i._propagate("resize",e)}})}}),_.ui.plugin.add("resizable","containment",{start:function(){var i,s,n=_(this).resizable("instance"),t=n.options,e=n.element,o=t.containment,r=o instanceof _?o.get(0):/parent/.test(o)?e.parent().get(0):o;r&&(n.containerElement=_(r),/document/.test(o)||o===document?(n.containerOffset={left:0,top:0},n.containerPosition={left:0,top:0},n.parentData={element:_(document),left:0,top:0,width:_(document).width(),height:_(document).height()||document.body.parentNode.scrollHeight}):(i=_(r),s=[],_(["Top","Right","Left","Bottom"]).each(function(t,e){s[t]=n._num(i.css("padding"+e))}),n.containerOffset=i.offset(),n.containerPosition=i.position(),n.containerSize={height:i.innerHeight()-s[3],width:i.innerWidth()-s[1]},t=n.containerOffset,e=n.containerSize.height,o=n.containerSize.width,o=n._hasScroll(r,"left")?r.scrollWidth:o,e=n._hasScroll(r)?r.scrollHeight:e,n.parentData={element:r,left:t.left,top:t.top,width:o,height:e}))},resize:function(t){var e=_(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.position,o=e._aspectRatio||t.shiftKey,r={top:0,left:0},h=e.containerElement,t=!0;h[0]!==document&&/static/.test(h.css("position"))&&(r=s),n.left<(e._helper?s.left:0)&&(e.size.width=e.size.width+(e._helper?e.position.left-s.left:e.position.left-r.left),o&&(e.size.height=e.size.width/e.aspectRatio,t=!1),e.position.left=i.helper?s.left:0),n.top<(e._helper?s.top:0)&&(e.size.height=e.size.height+(e._helper?e.position.top-s.top:e.position.top),o&&(e.size.width=e.size.height*e.aspectRatio,t=!1),e.position.top=e._helper?s.top:0),i=e.containerElement.get(0)===e.element.parent().get(0),n=/relative|absolute/.test(e.containerElement.css("position")),i&&n?(e.offset.left=e.parentData.left+e.position.left,e.offset.top=e.parentData.top+e.position.top):(e.offset.left=e.element.offset().left,e.offset.top=e.element.offset().top),n=Math.abs(e.sizeDiff.width+(e._helper?e.offset.left-r.left:e.offset.left-s.left)),s=Math.abs(e.sizeDiff.height+(e._helper?e.offset.top-r.top:e.offset.top-s.top)),n+e.size.width>=e.parentData.width&&(e.size.width=e.parentData.width-n,o&&(e.size.height=e.size.width/e.aspectRatio,t=!1)),s+e.size.height>=e.parentData.height&&(e.size.height=e.parentData.height-s,o&&(e.size.width=e.size.height*e.aspectRatio,t=!1)),t||(e.position.left=e.prevPosition.left,e.position.top=e.prevPosition.top,e.size.width=e.prevSize.width,e.size.height=e.prevSize.height)},stop:function(){var t=_(this).resizable("instance"),e=t.options,i=t.containerOffset,s=t.containerPosition,n=t.containerElement,o=_(t.helper),r=o.offset(),h=o.outerWidth()-t.sizeDiff.width,o=o.outerHeight()-t.sizeDiff.height;t._helper&&!e.animate&&/relative/.test(n.css("position"))&&_(this).css({left:r.left-s.left-i.left,width:h,height:o}),t._helper&&!e.animate&&/static/.test(n.css("position"))&&_(this).css({left:r.left-s.left-i.left,width:h,height:o})}}),_.ui.plugin.add("resizable","alsoResize",{start:function(){var t=_(this).resizable("instance").options;_(t.alsoResize).each(function(){var t=_(this);t.data("ui-resizable-alsoresize",{width:parseFloat(t.width()),height:parseFloat(t.height()),left:parseFloat(t.css("left")),top:parseFloat(t.css("top"))})})},resize:function(t,i){var e=_(this).resizable("instance"),s=e.options,n=e.originalSize,o=e.originalPosition,r={height:e.size.height-n.height||0,width:e.size.width-n.width||0,top:e.position.top-o.top||0,left:e.position.left-o.left||0};_(s.alsoResize).each(function(){var t=_(this),s=_(this).data("ui-resizable-alsoresize"),n={},e=t.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];_.each(e,function(t,e){var i=(s[e]||0)+(r[e]||0);i&&0<=i&&(n[e]=i||null)}),t.css(n)})},stop:function(){_(this).removeData("ui-resizable-alsoresize")}}),_.ui.plugin.add("resizable","ghost",{start:function(){var t=_(this).resizable("instance"),e=t.size;t.ghost=t.originalElement.clone(),t.ghost.css({opacity:.25,display:"block",position:"relative",height:e.height,width:e.width,margin:0,left:0,top:0}),t._addClass(t.ghost,"ui-resizable-ghost"),!1!==_.uiBackCompat&&"string"==typeof t.options.ghost&&t.ghost.addClass(this.options.ghost),t.ghost.appendTo(t.helper)},resize:function(){var t=_(this).resizable("instance");t.ghost&&t.ghost.css({position:"relative",height:t.size.height,width:t.size.width})},stop:function(){var t=_(this).resizable("instance");t.ghost&&t.helper&&t.helper.get(0).removeChild(t.ghost.get(0))}}),_.ui.plugin.add("resizable","grid",{resize:function(){var t,e=_(this).resizable("instance"),i=e.options,s=e.size,n=e.originalSize,o=e.originalPosition,r=e.axis,h="number"==typeof i.grid?[i.grid,i.grid]:i.grid,a=h[0]||1,l=h[1]||1,c=Math.round((s.width-n.width)/a)*a,p=Math.round((s.height-n.height)/l)*l,u=n.width+c,f=n.height+p,d=i.maxWidth&&i.maxWidthu,s=i.minHeight&&i.minHeight>f;i.grid=h,m&&(u+=a),s&&(f+=l),d&&(u-=a),g&&(f-=l),/^(se|s|e)$/.test(r)?(e.size.width=u,e.size.height=f):/^(ne)$/.test(r)?(e.size.width=u,e.size.height=f,e.position.top=o.top-p):/^(sw)$/.test(r)?(e.size.width=u,e.size.height=f,e.position.left=o.left-c):((f-l<=0||u-a<=0)&&(t=e._getPaddingPlusBorderDimensions(this)),0 *",opacity:!1,placeholder:!1,revert:!1,scroll:!0,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1e3,activate:null,beforeStop:null,change:null,deactivate:null,out:null,over:null,receive:null,remove:null,sort:null,start:null,stop:null,update:null},_isOverAxis:function(t,e,i){return e<=t&&t*{ cursor: "+o.cursor+" !important; }").appendTo(n)),o.zIndex&&(this.helper.css("zIndex")&&(this._storedZIndex=this.helper.css("zIndex")),this.helper.css("zIndex",o.zIndex)),o.opacity&&(this.helper.css("opacity")&&(this._storedOpacity=this.helper.css("opacity")),this.helper.css("opacity",o.opacity)),this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName&&(this.overflowOffset=this.scrollParent.offset()),this._trigger("start",t,this._uiHash()),this._preserveHelperProportions||this._cacheHelperProportions(),!i)for(s=this.containers.length-1;0<=s;s--)this.containers[s]._trigger("activate",t,this._uiHash(this));return _.ui.ddmanager&&(_.ui.ddmanager.current=this),_.ui.ddmanager&&!o.dropBehaviour&&_.ui.ddmanager.prepareOffsets(this,t),this.dragging=!0,this._addClass(this.helper,"ui-sortable-helper"),this.helper.parent().is(this.appendTo)||(this.helper.detach().appendTo(this.appendTo),this.offset.parent=this._getParentOffset()),this.position=this.originalPosition=this._generatePosition(t),this.originalPageX=t.pageX,this.originalPageY=t.pageY,this.lastPositionAbs=this.positionAbs=this._convertPositionTo("absolute"),this._mouseDrag(t),!0},_scroll:function(t){var e=this.options,i=!1;return this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName?(this.overflowOffset.top+this.scrollParent[0].offsetHeight-t.pageYt[this.floating?"width":"height"]?l&&c:o",i.document[0]);return i._addClass(t,"ui-sortable-placeholder",s||i.currentItem[0].className)._removeClass(t,"ui-sortable-helper"),"tbody"===n?i._createTrPlaceholder(i.currentItem.find("tr").eq(0),_("",i.document[0]).appendTo(t)):"tr"===n?i._createTrPlaceholder(i.currentItem,t):"img"===n&&t.attr("src",i.currentItem.attr("src")),s||t.css("visibility","hidden"),t},update:function(t,e){s&&!o.forcePlaceholderSize||(e.height()&&(!o.forcePlaceholderSize||"tbody"!==n&&"tr"!==n)||e.height(i.currentItem.innerHeight()-parseInt(i.currentItem.css("paddingTop")||0,10)-parseInt(i.currentItem.css("paddingBottom")||0,10)),e.width()||e.width(i.currentItem.innerWidth()-parseInt(i.currentItem.css("paddingLeft")||0,10)-parseInt(i.currentItem.css("paddingRight")||0,10)))}}),i.placeholder=_(o.placeholder.element.call(i.element,i.currentItem)),i.currentItem.after(i.placeholder),o.placeholder.update(i,i.placeholder)},_createTrPlaceholder:function(t,e){var i=this;t.children().each(function(){_(" ",i.document[0]).attr("colspan",_(this).attr("colspan")||1).appendTo(e)})},_contactContainers:function(t){for(var e,i,s,n,o,r,h,a,l,c=null,p=null,u=this.containers.length-1;0<=u;u--)_.contains(this.currentItem[0],this.containers[u].element[0])||(this._intersectsWith(this.containers[u].containerCache)?c&&_.contains(this.containers[u].element[0],c.element[0])||(c=this.containers[u],p=u):this.containers[u].containerCache.over&&(this.containers[u]._trigger("out",t,this._uiHash(this)),this.containers[u].containerCache.over=0));if(c)if(1===this.containers.length)this.containers[p].containerCache.over||(this.containers[p]._trigger("over",t,this._uiHash(this)),this.containers[p].containerCache.over=1);else{for(i=1e4,s=null,n=(a=c.floating||this._isFloating(this.currentItem))?"left":"top",o=a?"width":"height",l=a?"pageX":"pageY",e=this.items.length-1;0<=e;e--)_.contains(this.containers[p].element[0],this.items[e].item[0])&&this.items[e].item[0]!==this.currentItem[0]&&(r=this.items[e].item.offset()[n],h=!1,t[l]-r>this.items[e][o]/2&&(h=!0),Math.abs(t[l]-r)this.containment[2]&&(i=this.containment[2]+this.offset.click.left),t.pageY-this.offset.click.top>this.containment[3]&&(s=this.containment[3]+this.offset.click.top)),e.grid&&(t=this.originalPageY+Math.round((s-this.originalPageY)/e.grid[1])*e.grid[1],s=!this.containment||t-this.offset.click.top>=this.containment[1]&&t-this.offset.click.top<=this.containment[3]?t:t-this.offset.click.top>=this.containment[1]?t-e.grid[1]:t+e.grid[1],t=this.originalPageX+Math.round((i-this.originalPageX)/e.grid[0])*e.grid[0],i=!this.containment||t-this.offset.click.left>=this.containment[0]&&t-this.offset.click.left<=this.containment[2]?t:t-this.offset.click.left>=this.containment[0]?t-e.grid[0]:t+e.grid[0])),{top:s-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+("fixed"===this.cssPosition?-this.scrollParent.scrollTop():o?0:n.scrollTop()),left:i-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+("fixed"===this.cssPosition?-this.scrollParent.scrollLeft():o?0:n.scrollLeft())}},_rearrange:function(t,e,i,s){i?i[0].appendChild(this.placeholder[0]):e.item[0].parentNode.insertBefore(this.placeholder[0],"down"===this.direction?e.item[0]:e.item[0].nextSibling),this.counter=this.counter?++this.counter:1;var n=this.counter;this._delay(function(){n===this.counter&&this.refreshPositions(!s)})},_clear:function(t,e){this.reverting=!1;var i,s=[];if(!this._noFinalSort&&this.currentItem.parent().length&&this.placeholder.before(this.currentItem),this._noFinalSort=null,this.helper[0]===this.currentItem[0]){for(i in this._storedCSS)"auto"!==this._storedCSS[i]&&"static"!==this._storedCSS[i]||(this._storedCSS[i]="");this.currentItem.css(this._storedCSS),this._removeClass(this.currentItem,"ui-sortable-helper")}else this.currentItem.show();function n(e,i,s){return function(t){s._trigger(e,t,i._uiHash(i))}}for(this.fromOutside&&!e&&s.push(function(t){this._trigger("receive",t,this._uiHash(this.fromOutside))}),!this.fromOutside&&this.domPosition.prev===this.currentItem.prev().not(".ui-sortable-helper")[0]&&this.domPosition.parent===this.currentItem.parent()[0]||e||s.push(function(t){this._trigger("update",t,this._uiHash())}),this!==this.currentContainer&&(e||(s.push(function(t){this._trigger("remove",t,this._uiHash())}),s.push(function(e){return function(t){e._trigger("receive",t,this._uiHash(this))}}.call(this,this.currentContainer)),s.push(function(e){return function(t){e._trigger("update",t,this._uiHash(this))}}.call(this,this.currentContainer)))),i=this.containers.length-1;0<=i;i--)e||s.push(n("deactivate",this,this.containers[i])),this.containers[i].containerCache.over&&(s.push(n("out",this,this.containers[i])),this.containers[i].containerCache.over=0);if(this.storedCursor&&(this.document.find("body").css("cursor",this.storedCursor),this.storedStylesheet.remove()),this._storedOpacity&&this.helper.css("opacity",this._storedOpacity),this._storedZIndex&&this.helper.css("zIndex","auto"===this._storedZIndex?"":this._storedZIndex),this.dragging=!1,e||this._trigger("beforeStop",t,this._uiHash()),this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.cancelHelperRemoval||(this.helper[0]!==this.currentItem[0]&&this.helper.remove(),this.helper=null),!e){for(i=0;i")[0],m=a.each;function v(t){return null==t?t+"":"object"==typeof t?s[e.call(t)]||"object":typeof t}function w(t,e,i){var s=f[e.type]||{};return null==t?i||!e.def?null:e.def:(t=s.floor?~~t:parseFloat(t),isNaN(t)?e.def:s.mod?(t+s.mod)%s.mod:Math.min(s.max,Math.max(0,t)))}function b(s){var n=p(),o=n._rgba=[];return s=s.toLowerCase(),m(c,function(t,e){var i=e.re.exec(s),i=i&&e.parse(i),e=e.space||"rgba";if(i)return i=n[e](i),n[u[e].cache]=i[u[e].cache],o=n._rgba=i._rgba,!1}),o.length?("0,0,0,0"===o.join()&&a.extend(o,E.transparent),n):E[s]}function y(t,e,i){return 6*(i=(i+1)%1)<1?t+(e-t)*i*6:2*i<1?e:3*i<2?t+(e-t)*(2/3-i)*6:t}g.style.cssText="background-color:rgba(1,1,1,.5)",d.rgba=-1o.mod/2?s+=o.mod:s-n>o.mod/2&&(s-=o.mod)),a[i]=w((n-s)*r+s,e)))}),this[e](a)},blend:function(t){if(1===this._rgba[3])return this;var e=this._rgba.slice(),i=e.pop(),s=p(t)._rgba;return p(a.map(e,function(t,e){return(1-i)*s[e]+i*t}))},toRgbaString:function(){var t="rgba(",e=a.map(this._rgba,function(t,e){return null!=t?t:2").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),e={width:i.width(),height:i.height()},n=document.activeElement;try{n.id}catch(t){n=document.body}return i.wrap(t),i[0]!==n&&!_.contains(i[0],n)||_(n).trigger("focus"),t=i.parent(),"static"===i.css("position")?(t.css({position:"relative"}),i.css({position:"relative"})):(_.extend(s,{position:i.css("position"),zIndex:i.css("z-index")}),_.each(["top","left","bottom","right"],function(t,e){s[e]=i.css(e),isNaN(parseInt(s[e],10))&&(s[e]="auto")}),i.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})),i.css(e),t.css(s).show()},removeWrapper:function(t){var e=document.activeElement;return t.parent().is(".ui-effects-wrapper")&&(t.parent().replaceWith(t),t[0]!==e&&!_.contains(t[0],e)||_(e).trigger("focus")),t}}),_.extend(_.effects,{version:"1.13.1",define:function(t,e,i){return i||(i=e,e="effect"),_.effects.effect[t]=i,_.effects.effect[t].mode=e,i},scaledDimensions:function(t,e,i){if(0===e)return{height:0,width:0,outerHeight:0,outerWidth:0};var s="horizontal"!==i?(e||100)/100:1,e="vertical"!==i?(e||100)/100:1;return{height:t.height()*e,width:t.width()*s,outerHeight:t.outerHeight()*e,outerWidth:t.outerWidth()*s}},clipToBox:function(t){return{width:t.clip.right-t.clip.left,height:t.clip.bottom-t.clip.top,left:t.clip.left,top:t.clip.top}},unshift:function(t,e,i){var s=t.queue();1").insertAfter(t).css({display:/^(inline|ruby)/.test(t.css("display"))?"inline-block":"block",visibility:"hidden",marginTop:t.css("marginTop"),marginBottom:t.css("marginBottom"),marginLeft:t.css("marginLeft"),marginRight:t.css("marginRight"),float:t.css("float")}).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()).addClass("ui-effects-placeholder"),t.data(k+"placeholder",e)),t.css({position:i,left:s.left,top:s.top}),e},removePlaceholder:function(t){var e=k+"placeholder",i=t.data(e);i&&(i.remove(),t.removeData(e))},cleanUp:function(t){_.effects.restoreStyle(t),_.effects.removePlaceholder(t)},setTransition:function(s,t,n,o){return o=o||{},_.each(t,function(t,e){var i=s.cssUnit(e);0");a.appendTo("body").addClass(t.className).css({top:s.top-r,left:s.left-h,height:i.innerHeight(),width:i.innerWidth(),position:n?"fixed":"absolute"}).animate(o,t.duration,t.easing,function(){a.remove(),"function"==typeof e&&e()})}}),_.fx.step.clip=function(t){t.clipInit||(t.start=_(t.elem).cssClip(),"string"==typeof t.end&&(t.end=A(t.end,t.elem)),t.clipInit=!0),_(t.elem).cssClip({top:t.pos*(t.end.top-t.start.top)+t.start.top,right:t.pos*(t.end.right-t.start.right)+t.start.right,bottom:t.pos*(t.end.bottom-t.start.bottom)+t.start.bottom,left:t.pos*(t.end.left-t.start.left)+t.start.left})},N={},_.each(["Quad","Cubic","Quart","Quint","Expo"],function(e,t){N[t]=function(t){return Math.pow(t,e+2)}}),_.extend(N,{Sine:function(t){return 1-Math.cos(t*Math.PI/2)},Circ:function(t){return 1-Math.sqrt(1-t*t)},Elastic:function(t){return 0===t||1===t?t:-Math.pow(2,8*(t-1))*Math.sin((80*(t-1)-7.5)*Math.PI/15)},Back:function(t){return t*t*(3*t-2)},Bounce:function(t){for(var e,i=4;t<((e=Math.pow(2,--i))-1)/11;);return 1/Math.pow(4,3-i)-7.5625*Math.pow((3*e-2)/22-t,2)}}),_.each(N,function(t,e){_.easing["easeIn"+t]=e,_.easing["easeOut"+t]=function(t){return 1-e(1-t)},_.easing["easeInOut"+t]=function(t){return t<.5?e(2*t)/2:1-e(-2*t+2)/2}});_.effects,_.effects.define("blind","hide",function(t,e){var i={up:["bottom","top"],vertical:["bottom","top"],down:["top","bottom"],left:["right","left"],horizontal:["right","left"],right:["left","right"]},s=_(this),n=t.direction||"up",o=s.cssClip(),r={clip:_.extend({},o)},h=_.effects.createPlaceholder(s);r.clip[i[n][0]]=r.clip[i[n][1]],"show"===t.mode&&(s.cssClip(r.clip),h&&h.css(_.effects.clipToBox(r)),r.clip=o),h&&h.animate(_.effects.clipToBox(r),t.duration,t.easing),s.animate(r,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),_.effects.define("slide","show",function(t,e){var i,s,n=_(this),o={up:["bottom","top"],down:["top","bottom"],left:["right","left"],right:["left","right"]},r=t.mode,h=t.direction||"left",a="up"===h||"down"===h?"top":"left",l="up"===h||"left"===h,c=t.distance||n["top"==a?"outerHeight":"outerWidth"](!0),p={};_.effects.createPlaceholder(n),i=n.cssClip(),s=n.position()[a],p[a]=(l?-1:1)*c+s,p.clip=n.cssClip(),p.clip[o[h][1]]=p.clip[o[h][0]],"show"===r&&(n.cssClip(p.clip),n.css(a,p[a]),p.clip=i,p[a]=s),n.animate(p,{queue:!1,duration:t.duration,easing:t.easing,complete:e})})}); +!function (t) { "use strict"; "function" == typeof define && define.amd ? define(["jquery"], t) : t(jQuery) }(function (V) { "use strict"; V.ui = V.ui || {}; V.ui.version = "1.13.2"; var n, i = 0, a = Array.prototype.hasOwnProperty, r = Array.prototype.slice; V.cleanData = (n = V.cleanData, function (t) { for (var e, i, s = 0; null != (i = t[s]); s++)(e = V._data(i, "events")) && e.remove && V(i).triggerHandler("remove"); n(t) }), V.widget = function (t, i, e) { var s, n, o, a = {}, r = t.split(".")[0], l = r + "-" + (t = t.split(".")[1]); return e || (e = i, i = V.Widget), Array.isArray(e) && (e = V.extend.apply(null, [{}].concat(e))), V.expr.pseudos[l.toLowerCase()] = function (t) { return !!V.data(t, l) }, V[r] = V[r] || {}, s = V[r][t], n = V[r][t] = function (t, e) { if (!this || !this._createWidget) return new n(t, e); arguments.length && this._createWidget(t, e) }, V.extend(n, s, { version: e.version, _proto: V.extend({}, e), _childConstructors: [] }), (o = new i).options = V.widget.extend({}, o.options), V.each(e, function (e, s) { function n() { return i.prototype[e].apply(this, arguments) } function o(t) { return i.prototype[e].apply(this, t) } a[e] = "function" == typeof s ? function () { var t, e = this._super, i = this._superApply; return this._super = n, this._superApply = o, t = s.apply(this, arguments), this._super = e, this._superApply = i, t } : s }), n.prototype = V.widget.extend(o, { widgetEventPrefix: s && o.widgetEventPrefix || t }, a, { constructor: n, namespace: r, widgetName: t, widgetFullName: l }), s ? (V.each(s._childConstructors, function (t, e) { var i = e.prototype; V.widget(i.namespace + "." + i.widgetName, n, e._proto) }), delete s._childConstructors) : i._childConstructors.push(n), V.widget.bridge(t, n), n }, V.widget.extend = function (t) { for (var e, i, s = r.call(arguments, 1), n = 0, o = s.length; n < o; n++)for (e in s[n]) i = s[n][e], a.call(s[n], e) && void 0 !== i && (V.isPlainObject(i) ? t[e] = V.isPlainObject(t[e]) ? V.widget.extend({}, t[e], i) : V.widget.extend({}, i) : t[e] = i); return t }, V.widget.bridge = function (o, e) { var a = e.prototype.widgetFullName || o; V.fn[o] = function (i) { var t = "string" == typeof i, s = r.call(arguments, 1), n = this; return t ? this.length || "instance" !== i ? this.each(function () { var t, e = V.data(this, a); return "instance" === i ? (n = e, !1) : e ? "function" != typeof e[i] || "_" === i.charAt(0) ? V.error("no such method '" + i + "' for " + o + " widget instance") : (t = e[i].apply(e, s)) !== e && void 0 !== t ? (n = t && t.jquery ? n.pushStack(t.get()) : t, !1) : void 0 : V.error("cannot call methods on " + o + " prior to initialization; attempted to call method '" + i + "'") }) : n = void 0 : (s.length && (i = V.widget.extend.apply(null, [i].concat(s))), this.each(function () { var t = V.data(this, a); t ? (t.option(i || {}), t._init && t._init()) : V.data(this, a, new e(i, this)) })), n } }, V.Widget = function () { }, V.Widget._childConstructors = [], V.Widget.prototype = { widgetName: "widget", widgetEventPrefix: "", defaultElement: "
    ", options: { classes: {}, disabled: !1, create: null }, _createWidget: function (t, e) { e = V(e || this.defaultElement || this)[0], this.element = V(e), this.uuid = i++, this.eventNamespace = "." + this.widgetName + this.uuid, this.bindings = V(), this.hoverable = V(), this.focusable = V(), this.classesElementLookup = {}, e !== this && (V.data(e, this.widgetFullName, this), this._on(!0, this.element, { remove: function (t) { t.target === e && this.destroy() } }), this.document = V(e.style ? e.ownerDocument : e.document || e), this.window = V(this.document[0].defaultView || this.document[0].parentWindow)), this.options = V.widget.extend({}, this.options, this._getCreateOptions(), t), this._create(), this.options.disabled && this._setOptionDisabled(this.options.disabled), this._trigger("create", null, this._getCreateEventData()), this._init() }, _getCreateOptions: function () { return {} }, _getCreateEventData: V.noop, _create: V.noop, _init: V.noop, destroy: function () { var i = this; this._destroy(), V.each(this.classesElementLookup, function (t, e) { i._removeClass(e, t) }), this.element.off(this.eventNamespace).removeData(this.widgetFullName), this.widget().off(this.eventNamespace).removeAttr("aria-disabled"), this.bindings.off(this.eventNamespace) }, _destroy: V.noop, widget: function () { return this.element }, option: function (t, e) { var i, s, n, o = t; if (0 === arguments.length) return V.widget.extend({}, this.options); if ("string" == typeof t) if (o = {}, t = (i = t.split(".")).shift(), i.length) { for (s = o[t] = V.widget.extend({}, this.options[t]), n = 0; n < i.length - 1; n++)s[i[n]] = s[i[n]] || {}, s = s[i[n]]; if (t = i.pop(), 1 === arguments.length) return void 0 === s[t] ? null : s[t]; s[t] = e } else { if (1 === arguments.length) return void 0 === this.options[t] ? null : this.options[t]; o[t] = e } return this._setOptions(o), this }, _setOptions: function (t) { for (var e in t) this._setOption(e, t[e]); return this }, _setOption: function (t, e) { return "classes" === t && this._setOptionClasses(e), this.options[t] = e, "disabled" === t && this._setOptionDisabled(e), this }, _setOptionClasses: function (t) { var e, i, s; for (e in t) s = this.classesElementLookup[e], t[e] !== this.options.classes[e] && s && s.length && (i = V(s.get()), this._removeClass(s, e), i.addClass(this._classes({ element: i, keys: e, classes: t, add: !0 }))) }, _setOptionDisabled: function (t) { this._toggleClass(this.widget(), this.widgetFullName + "-disabled", null, !!t), t && (this._removeClass(this.hoverable, null, "ui-state-hover"), this._removeClass(this.focusable, null, "ui-state-focus")) }, enable: function () { return this._setOptions({ disabled: !1 }) }, disable: function () { return this._setOptions({ disabled: !0 }) }, _classes: function (n) { var o = [], a = this; function t(t, e) { for (var i, s = 0; s < t.length; s++)i = a.classesElementLookup[t[s]] || V(), i = n.add ? (function () { var i = []; n.element.each(function (t, e) { V.map(a.classesElementLookup, function (t) { return t }).some(function (t) { return t.is(e) }) || i.push(e) }), a._on(V(i), { remove: "_untrackClassesElement" }) }(), V(V.uniqueSort(i.get().concat(n.element.get())))) : V(i.not(n.element).get()), a.classesElementLookup[t[s]] = i, o.push(t[s]), e && n.classes[t[s]] && o.push(n.classes[t[s]]) } return (n = V.extend({ element: this.element, classes: this.options.classes || {} }, n)).keys && t(n.keys.match(/\S+/g) || [], !0), n.extra && t(n.extra.match(/\S+/g) || []), o.join(" ") }, _untrackClassesElement: function (i) { var s = this; V.each(s.classesElementLookup, function (t, e) { -1 !== V.inArray(i.target, e) && (s.classesElementLookup[t] = V(e.not(i.target).get())) }), this._off(V(i.target)) }, _removeClass: function (t, e, i) { return this._toggleClass(t, e, i, !1) }, _addClass: function (t, e, i) { return this._toggleClass(t, e, i, !0) }, _toggleClass: function (t, e, i, s) { var n = "string" == typeof t || null === t, i = { extra: n ? e : i, keys: n ? t : e, element: n ? this.element : t, add: s = "boolean" == typeof s ? s : i }; return i.element.toggleClass(this._classes(i), s), this }, _on: function (n, o, t) { var a, r = this; "boolean" != typeof n && (t = o, o = n, n = !1), t ? (o = a = V(o), this.bindings = this.bindings.add(o)) : (t = o, o = this.element, a = this.widget()), V.each(t, function (t, e) { function i() { if (n || !0 !== r.options.disabled && !V(this).hasClass("ui-state-disabled")) return ("string" == typeof e ? r[e] : e).apply(r, arguments) } "string" != typeof e && (i.guid = e.guid = e.guid || i.guid || V.guid++); var s = t.match(/^([\w:-]*)\s*(.*)$/), t = s[1] + r.eventNamespace, s = s[2]; s ? a.on(t, s, i) : o.on(t, i) }) }, _off: function (t, e) { e = (e || "").split(" ").join(this.eventNamespace + " ") + this.eventNamespace, t.off(e), this.bindings = V(this.bindings.not(t).get()), this.focusable = V(this.focusable.not(t).get()), this.hoverable = V(this.hoverable.not(t).get()) }, _delay: function (t, e) { var i = this; return setTimeout(function () { return ("string" == typeof t ? i[t] : t).apply(i, arguments) }, e || 0) }, _hoverable: function (t) { this.hoverable = this.hoverable.add(t), this._on(t, { mouseenter: function (t) { this._addClass(V(t.currentTarget), null, "ui-state-hover") }, mouseleave: function (t) { this._removeClass(V(t.currentTarget), null, "ui-state-hover") } }) }, _focusable: function (t) { this.focusable = this.focusable.add(t), this._on(t, { focusin: function (t) { this._addClass(V(t.currentTarget), null, "ui-state-focus") }, focusout: function (t) { this._removeClass(V(t.currentTarget), null, "ui-state-focus") } }) }, _trigger: function (t, e, i) { var s, n, o = this.options[t]; if (i = i || {}, (e = V.Event(e)).type = (t === this.widgetEventPrefix ? t : this.widgetEventPrefix + t).toLowerCase(), e.target = this.element[0], n = e.originalEvent) for (s in n) s in e || (e[s] = n[s]); return this.element.trigger(e, i), !("function" == typeof o && !1 === o.apply(this.element[0], [e].concat(i)) || e.isDefaultPrevented()) } }, V.each({ show: "fadeIn", hide: "fadeOut" }, function (o, a) { V.Widget.prototype["_" + o] = function (e, t, i) { var s, n = (t = "string" == typeof t ? { effect: t } : t) ? !0 !== t && "number" != typeof t && t.effect || a : o; "number" == typeof (t = t || {}) ? t = { duration: t } : !0 === t && (t = {}), s = !V.isEmptyObject(t), t.complete = i, t.delay && e.delay(t.delay), s && V.effects && V.effects.effect[n] ? e[o](t) : n !== o && e[n] ? e[n](t.duration, t.easing, i) : e.queue(function (t) { V(this)[o](), i && i.call(e[0]), t() }) } }); var s, x, k, o, l, h, c, u, C; V.widget; function D(t, e, i) { return [parseFloat(t[0]) * (u.test(t[0]) ? e / 100 : 1), parseFloat(t[1]) * (u.test(t[1]) ? i / 100 : 1)] } function I(t, e) { return parseInt(V.css(t, e), 10) || 0 } function T(t) { return null != t && t === t.window } x = Math.max, k = Math.abs, o = /left|center|right/, l = /top|center|bottom/, h = /[\+\-]\d+(\.[\d]+)?%?/, c = /^\w+/, u = /%$/, C = V.fn.position, V.position = { scrollbarWidth: function () { if (void 0 !== s) return s; var t, e = V("
    "), i = e.children()[0]; return V("body").append(e), t = i.offsetWidth, e.css("overflow", "scroll"), t === (i = i.offsetWidth) && (i = e[0].clientWidth), e.remove(), s = t - i }, getScrollInfo: function (t) { var e = t.isWindow || t.isDocument ? "" : t.element.css("overflow-x"), i = t.isWindow || t.isDocument ? "" : t.element.css("overflow-y"), e = "scroll" === e || "auto" === e && t.width < t.element[0].scrollWidth; return { width: "scroll" === i || "auto" === i && t.height < t.element[0].scrollHeight ? V.position.scrollbarWidth() : 0, height: e ? V.position.scrollbarWidth() : 0 } }, getWithinInfo: function (t) { var e = V(t || window), i = T(e[0]), s = !!e[0] && 9 === e[0].nodeType; return { element: e, isWindow: i, isDocument: s, offset: !i && !s ? V(t).offset() : { left: 0, top: 0 }, scrollLeft: e.scrollLeft(), scrollTop: e.scrollTop(), width: e.outerWidth(), height: e.outerHeight() } } }, V.fn.position = function (u) { if (!u || !u.of) return C.apply(this, arguments); var d, p, f, g, m, t, _ = "string" == typeof (u = V.extend({}, u)).of ? V(document).find(u.of) : V(u.of), v = V.position.getWithinInfo(u.within), b = V.position.getScrollInfo(v), y = (u.collision || "flip").split(" "), w = {}, e = 9 === (t = (e = _)[0]).nodeType ? { width: e.width(), height: e.height(), offset: { top: 0, left: 0 } } : T(t) ? { width: e.width(), height: e.height(), offset: { top: e.scrollTop(), left: e.scrollLeft() } } : t.preventDefault ? { width: 0, height: 0, offset: { top: t.pageY, left: t.pageX } } : { width: e.outerWidth(), height: e.outerHeight(), offset: e.offset() }; return _[0].preventDefault && (u.at = "left top"), p = e.width, f = e.height, m = V.extend({}, g = e.offset), V.each(["my", "at"], function () { var t, e, i = (u[this] || "").split(" "); (i = 1 === i.length ? o.test(i[0]) ? i.concat(["center"]) : l.test(i[0]) ? ["center"].concat(i) : ["center", "center"] : i)[0] = o.test(i[0]) ? i[0] : "center", i[1] = l.test(i[1]) ? i[1] : "center", t = h.exec(i[0]), e = h.exec(i[1]), w[this] = [t ? t[0] : 0, e ? e[0] : 0], u[this] = [c.exec(i[0])[0], c.exec(i[1])[0]] }), 1 === y.length && (y[1] = y[0]), "right" === u.at[0] ? m.left += p : "center" === u.at[0] && (m.left += p / 2), "bottom" === u.at[1] ? m.top += f : "center" === u.at[1] && (m.top += f / 2), d = D(w.at, p, f), m.left += d[0], m.top += d[1], this.each(function () { var i, t, a = V(this), r = a.outerWidth(), l = a.outerHeight(), e = I(this, "marginLeft"), s = I(this, "marginTop"), n = r + e + I(this, "marginRight") + b.width, o = l + s + I(this, "marginBottom") + b.height, h = V.extend({}, m), c = D(w.my, a.outerWidth(), a.outerHeight()); "right" === u.my[0] ? h.left -= r : "center" === u.my[0] && (h.left -= r / 2), "bottom" === u.my[1] ? h.top -= l : "center" === u.my[1] && (h.top -= l / 2), h.left += c[0], h.top += c[1], i = { marginLeft: e, marginTop: s }, V.each(["left", "top"], function (t, e) { V.ui.position[y[t]] && V.ui.position[y[t]][e](h, { targetWidth: p, targetHeight: f, elemWidth: r, elemHeight: l, collisionPosition: i, collisionWidth: n, collisionHeight: o, offset: [d[0] + c[0], d[1] + c[1]], my: u.my, at: u.at, within: v, elem: a }) }), u.using && (t = function (t) { var e = g.left - h.left, i = e + p - r, s = g.top - h.top, n = s + f - l, o = { target: { element: _, left: g.left, top: g.top, width: p, height: f }, element: { element: a, left: h.left, top: h.top, width: r, height: l }, horizontal: i < 0 ? "left" : 0 < e ? "right" : "center", vertical: n < 0 ? "top" : 0 < s ? "bottom" : "middle" }; p < r && k(e + i) < p && (o.horizontal = "center"), f < l && k(s + n) < f && (o.vertical = "middle"), x(k(e), k(i)) > x(k(s), k(n)) ? o.important = "horizontal" : o.important = "vertical", u.using.call(this, t, o) }), a.offset(V.extend(h, { using: t })) }) }, V.ui.position = { fit: { left: function (t, e) { var i = e.within, s = i.isWindow ? i.scrollLeft : i.offset.left, n = i.width, o = t.left - e.collisionPosition.marginLeft, a = s - o, r = o + e.collisionWidth - n - s; e.collisionWidth > n ? 0 < a && r <= 0 ? (i = t.left + a + e.collisionWidth - n - s, t.left += a - i) : t.left = !(0 < r && a <= 0) && r < a ? s + n - e.collisionWidth : s : 0 < a ? t.left += a : 0 < r ? t.left -= r : t.left = x(t.left - o, t.left) }, top: function (t, e) { var i = e.within, s = i.isWindow ? i.scrollTop : i.offset.top, n = e.within.height, o = t.top - e.collisionPosition.marginTop, a = s - o, r = o + e.collisionHeight - n - s; e.collisionHeight > n ? 0 < a && r <= 0 ? (i = t.top + a + e.collisionHeight - n - s, t.top += a - i) : t.top = !(0 < r && a <= 0) && r < a ? s + n - e.collisionHeight : s : 0 < a ? t.top += a : 0 < r ? t.top -= r : t.top = x(t.top - o, t.top) } }, flip: { left: function (t, e) { var i = e.within, s = i.offset.left + i.scrollLeft, n = i.width, o = i.isWindow ? i.scrollLeft : i.offset.left, a = t.left - e.collisionPosition.marginLeft, r = a - o, l = a + e.collisionWidth - n - o, h = "left" === e.my[0] ? -e.elemWidth : "right" === e.my[0] ? e.elemWidth : 0, i = "left" === e.at[0] ? e.targetWidth : "right" === e.at[0] ? -e.targetWidth : 0, a = -2 * e.offset[0]; r < 0 ? ((s = t.left + h + i + a + e.collisionWidth - n - s) < 0 || s < k(r)) && (t.left += h + i + a) : 0 < l && (0 < (o = t.left - e.collisionPosition.marginLeft + h + i + a - o) || k(o) < l) && (t.left += h + i + a) }, top: function (t, e) { var i = e.within, s = i.offset.top + i.scrollTop, n = i.height, o = i.isWindow ? i.scrollTop : i.offset.top, a = t.top - e.collisionPosition.marginTop, r = a - o, l = a + e.collisionHeight - n - o, h = "top" === e.my[1] ? -e.elemHeight : "bottom" === e.my[1] ? e.elemHeight : 0, i = "top" === e.at[1] ? e.targetHeight : "bottom" === e.at[1] ? -e.targetHeight : 0, a = -2 * e.offset[1]; r < 0 ? ((s = t.top + h + i + a + e.collisionHeight - n - s) < 0 || s < k(r)) && (t.top += h + i + a) : 0 < l && (0 < (o = t.top - e.collisionPosition.marginTop + h + i + a - o) || k(o) < l) && (t.top += h + i + a) } }, flipfit: { left: function () { V.ui.position.flip.left.apply(this, arguments), V.ui.position.fit.left.apply(this, arguments) }, top: function () { V.ui.position.flip.top.apply(this, arguments), V.ui.position.fit.top.apply(this, arguments) } } }; V.ui.position, V.extend(V.expr.pseudos, { data: V.expr.createPseudo ? V.expr.createPseudo(function (e) { return function (t) { return !!V.data(t, e) } }) : function (t, e, i) { return !!V.data(t, i[3]) } }), V.fn.extend({ disableSelection: (t = "onselectstart" in document.createElement("div") ? "selectstart" : "mousedown", function () { return this.on(t + ".ui-disableSelection", function (t) { t.preventDefault() }) }), enableSelection: function () { return this.off(".ui-disableSelection") } }); var t, d = V, p = {}, e = p.toString, f = /^([\-+])=\s*(\d+\.?\d*)/, g = [{ re: /rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/, parse: function (t) { return [t[1], t[2], t[3], t[4]] } }, { re: /rgba?\(\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/, parse: function (t) { return [2.55 * t[1], 2.55 * t[2], 2.55 * t[3], t[4]] } }, { re: /#([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})?/, parse: function (t) { return [parseInt(t[1], 16), parseInt(t[2], 16), parseInt(t[3], 16), t[4] ? (parseInt(t[4], 16) / 255).toFixed(2) : 1] } }, { re: /#([a-f0-9])([a-f0-9])([a-f0-9])([a-f0-9])?/, parse: function (t) { return [parseInt(t[1] + t[1], 16), parseInt(t[2] + t[2], 16), parseInt(t[3] + t[3], 16), t[4] ? (parseInt(t[4] + t[4], 16) / 255).toFixed(2) : 1] } }, { re: /hsla?\(\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/, space: "hsla", parse: function (t) { return [t[1], t[2] / 100, t[3] / 100, t[4]] } }], m = d.Color = function (t, e, i, s) { return new d.Color.fn.parse(t, e, i, s) }, _ = { rgba: { props: { red: { idx: 0, type: "byte" }, green: { idx: 1, type: "byte" }, blue: { idx: 2, type: "byte" } } }, hsla: { props: { hue: { idx: 0, type: "degrees" }, saturation: { idx: 1, type: "percent" }, lightness: { idx: 2, type: "percent" } } } }, v = { byte: { floor: !0, max: 255 }, percent: { max: 1 }, degrees: { mod: 360, floor: !0 } }, b = m.support = {}, y = d("

    ")[0], w = d.each; function P(t) { return null == t ? t + "" : "object" == typeof t ? p[e.call(t)] || "object" : typeof t } function M(t, e, i) { var s = v[e.type] || {}; return null == t ? i || !e.def ? null : e.def : (t = s.floor ? ~~t : parseFloat(t), isNaN(t) ? e.def : s.mod ? (t + s.mod) % s.mod : Math.min(s.max, Math.max(0, t))) } function S(s) { var n = m(), o = n._rgba = []; return s = s.toLowerCase(), w(g, function (t, e) { var i = e.re.exec(s), i = i && e.parse(i), e = e.space || "rgba"; if (i) return i = n[e](i), n[_[e].cache] = i[_[e].cache], o = n._rgba = i._rgba, !1 }), o.length ? ("0,0,0,0" === o.join() && d.extend(o, B.transparent), n) : B[s] } function H(t, e, i) { return 6 * (i = (i + 1) % 1) < 1 ? t + (e - t) * i * 6 : 2 * i < 1 ? e : 3 * i < 2 ? t + (e - t) * (2 / 3 - i) * 6 : t } y.style.cssText = "background-color:rgba(1,1,1,.5)", b.rgba = -1 < y.style.backgroundColor.indexOf("rgba"), w(_, function (t, e) { e.cache = "_" + t, e.props.alpha = { idx: 3, type: "percent", def: 1 } }), d.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "), function (t, e) { p["[object " + e + "]"] = e.toLowerCase() }), (m.fn = d.extend(m.prototype, { parse: function (n, t, e, i) { if (void 0 === n) return this._rgba = [null, null, null, null], this; (n.jquery || n.nodeType) && (n = d(n).css(t), t = void 0); var o = this, s = P(n), a = this._rgba = []; return void 0 !== t && (n = [n, t, e, i], s = "array"), "string" === s ? this.parse(S(n) || B._default) : "array" === s ? (w(_.rgba.props, function (t, e) { a[e.idx] = M(n[e.idx], e) }), this) : "object" === s ? (w(_, n instanceof m ? function (t, e) { n[e.cache] && (o[e.cache] = n[e.cache].slice()) } : function (t, i) { var s = i.cache; w(i.props, function (t, e) { if (!o[s] && i.to) { if ("alpha" === t || null == n[t]) return; o[s] = i.to(o._rgba) } o[s][e.idx] = M(n[t], e, !0) }), o[s] && d.inArray(null, o[s].slice(0, 3)) < 0 && (null == o[s][3] && (o[s][3] = 1), i.from && (o._rgba = i.from(o[s]))) }), this) : void 0 }, is: function (t) { var n = m(t), o = !0, a = this; return w(_, function (t, e) { var i, s = n[e.cache]; return s && (i = a[e.cache] || e.to && e.to(a._rgba) || [], w(e.props, function (t, e) { if (null != s[e.idx]) return o = s[e.idx] === i[e.idx] })), o }), o }, _space: function () { var i = [], s = this; return w(_, function (t, e) { s[e.cache] && i.push(t) }), i.pop() }, transition: function (t, a) { var e = (h = m(t))._space(), i = _[e], t = 0 === this.alpha() ? m("transparent") : this, r = t[i.cache] || i.to(t._rgba), l = r.slice(), h = h[i.cache]; return w(i.props, function (t, e) { var i = e.idx, s = r[i], n = h[i], o = v[e.type] || {}; null !== n && (null === s ? l[i] = n : (o.mod && (n - s > o.mod / 2 ? s += o.mod : s - n > o.mod / 2 && (s -= o.mod)), l[i] = M((n - s) * a + s, e))) }), this[e](l) }, blend: function (t) { if (1 === this._rgba[3]) return this; var e = this._rgba.slice(), i = e.pop(), s = m(t)._rgba; return m(d.map(e, function (t, e) { return (1 - i) * s[e] + i * t })) }, toRgbaString: function () { var t = "rgba(", e = d.map(this._rgba, function (t, e) { return null != t ? t : 2 < e ? 1 : 0 }); return 1 === e[3] && (e.pop(), t = "rgb("), t + e.join() + ")" }, toHslaString: function () { var t = "hsla(", e = d.map(this.hsla(), function (t, e) { return null == t && (t = 2 < e ? 1 : 0), t = e && e < 3 ? Math.round(100 * t) + "%" : t }); return 1 === e[3] && (e.pop(), t = "hsl("), t + e.join() + ")" }, toHexString: function (t) { var e = this._rgba.slice(), i = e.pop(); return t && e.push(~~(255 * i)), "#" + d.map(e, function (t) { return 1 === (t = (t || 0).toString(16)).length ? "0" + t : t }).join("") }, toString: function () { return 0 === this._rgba[3] ? "transparent" : this.toRgbaString() } })).parse.prototype = m.fn, _.hsla.to = function (t) { if (null == t[0] || null == t[1] || null == t[2]) return [null, null, null, t[3]]; var e = t[0] / 255, i = t[1] / 255, s = t[2] / 255, n = t[3], o = Math.max(e, i, s), a = Math.min(e, i, s), r = o - a, l = o + a, t = .5 * l, i = a === o ? 0 : e === o ? 60 * (i - s) / r + 360 : i === o ? 60 * (s - e) / r + 120 : 60 * (e - i) / r + 240, l = 0 == r ? 0 : t <= .5 ? r / l : r / (2 - l); return [Math.round(i) % 360, l, t, null == n ? 1 : n] }, _.hsla.from = function (t) { if (null == t[0] || null == t[1] || null == t[2]) return [null, null, null, t[3]]; var e = t[0] / 360, i = t[1], s = t[2], t = t[3], i = s <= .5 ? s * (1 + i) : s + i - s * i, s = 2 * s - i; return [Math.round(255 * H(s, i, e + 1 / 3)), Math.round(255 * H(s, i, e)), Math.round(255 * H(s, i, e - 1 / 3)), t] }, w(_, function (l, t) { var e = t.props, o = t.cache, a = t.to, r = t.from; m.fn[l] = function (t) { if (a && !this[o] && (this[o] = a(this._rgba)), void 0 === t) return this[o].slice(); var i = P(t), s = "array" === i || "object" === i ? t : arguments, n = this[o].slice(); return w(e, function (t, e) { t = s["object" === i ? t : e.idx]; null == t && (t = n[e.idx]), n[e.idx] = M(t, e) }), r ? ((t = m(r(n)))[o] = n, t) : m(n) }, w(e, function (a, r) { m.fn[a] || (m.fn[a] = function (t) { var e, i = P(t), s = "alpha" === a ? this._hsla ? "hsla" : "rgba" : l, n = this[s](), o = n[r.idx]; return "undefined" === i ? o : ("function" === i && (i = P(t = t.call(this, o))), null == t && r.empty ? this : ("string" === i && (e = f.exec(t)) && (t = o + parseFloat(e[2]) * ("+" === e[1] ? 1 : -1)), n[r.idx] = t, this[s](n))) }) }) }), (m.hook = function (t) { t = t.split(" "); w(t, function (t, o) { d.cssHooks[o] = { set: function (t, e) { var i, s, n = ""; if ("transparent" !== e && ("string" !== P(e) || (i = S(e)))) { if (e = m(i || e), !b.rgba && 1 !== e._rgba[3]) { for (s = "backgroundColor" === o ? t.parentNode : t; ("" === n || "transparent" === n) && s && s.style;)try { n = d.css(s, "backgroundColor"), s = s.parentNode } catch (t) { } e = e.blend(n && "transparent" !== n ? n : "_default") } e = e.toRgbaString() } try { t.style[o] = e } catch (t) { } } }, d.fx.step[o] = function (t) { t.colorInit || (t.start = m(t.elem, o), t.end = m(t.end), t.colorInit = !0), d.cssHooks[o].set(t.elem, t.start.transition(t.end, t.pos)) } }) })("backgroundColor borderBottomColor borderLeftColor borderRightColor borderTopColor color columnRuleColor outlineColor textDecorationColor textEmphasisColor"), d.cssHooks.borderColor = { expand: function (i) { var s = {}; return w(["Top", "Right", "Bottom", "Left"], function (t, e) { s["border" + e + "Color"] = i }), s } }; var z, A, O, N, E, W, F, L, R, Y, B = d.Color.names = { aqua: "#00ffff", black: "#000000", blue: "#0000ff", fuchsia: "#ff00ff", gray: "#808080", green: "#008000", lime: "#00ff00", maroon: "#800000", navy: "#000080", olive: "#808000", purple: "#800080", red: "#ff0000", silver: "#c0c0c0", teal: "#008080", white: "#ffffff", yellow: "#ffff00", transparent: [null, null, null, 0], _default: "#ffffff" }, j = "ui-effects-", q = "ui-effects-style", K = "ui-effects-animated"; function U(t) { var e, i, s = t.ownerDocument.defaultView ? t.ownerDocument.defaultView.getComputedStyle(t, null) : t.currentStyle, n = {}; if (s && s.length && s[0] && s[s[0]]) for (i = s.length; i--;)"string" == typeof s[e = s[i]] && (n[e.replace(/-([\da-z])/gi, function (t, e) { return e.toUpperCase() })] = s[e]); else for (e in s) "string" == typeof s[e] && (n[e] = s[e]); return n } function X(t, e, i, s) { return t = { effect: t = V.isPlainObject(t) ? (e = t).effect : t }, "function" == typeof (e = null == e ? {} : e) && (s = e, i = null, e = {}), "number" != typeof e && !V.fx.speeds[e] || (s = i, i = e, e = {}), "function" == typeof i && (s = i, i = null), e && V.extend(t, e), i = i || e.duration, t.duration = V.fx.off ? 0 : "number" == typeof i ? i : i in V.fx.speeds ? V.fx.speeds[i] : V.fx.speeds._default, t.complete = s || e.complete, t } function $(t) { return !t || "number" == typeof t || V.fx.speeds[t] || ("string" == typeof t && !V.effects.effect[t] || ("function" == typeof t || "object" == typeof t && !t.effect)) } function G(t, e) { var i = e.outerWidth(), e = e.outerHeight(), t = /^rect\((-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto)\)$/.exec(t) || ["", 0, i, e, 0]; return { top: parseFloat(t[1]) || 0, right: "auto" === t[2] ? i : parseFloat(t[2]), bottom: "auto" === t[3] ? e : parseFloat(t[3]), left: parseFloat(t[4]) || 0 } } V.effects = { effect: {} }, N = ["add", "remove", "toggle"], E = { border: 1, borderBottom: 1, borderColor: 1, borderLeft: 1, borderRight: 1, borderTop: 1, borderWidth: 1, margin: 1, padding: 1 }, V.each(["borderLeftStyle", "borderRightStyle", "borderBottomStyle", "borderTopStyle"], function (t, e) { V.fx.step[e] = function (t) { ("none" !== t.end && !t.setAttr || 1 === t.pos && !t.setAttr) && (d.style(t.elem, e, t.end), t.setAttr = !0) } }), V.fn.addBack || (V.fn.addBack = function (t) { return this.add(null == t ? this.prevObject : this.prevObject.filter(t)) }), V.effects.animateClass = function (n, t, e, i) { var o = V.speed(t, e, i); return this.queue(function () { var i = V(this), t = i.attr("class") || "", e = (e = o.children ? i.find("*").addBack() : i).map(function () { return { el: V(this), start: U(this) } }), s = function () { V.each(N, function (t, e) { n[e] && i[e + "Class"](n[e]) }) }; s(), e = e.map(function () { return this.end = U(this.el[0]), this.diff = function (t, e) { var i, s, n = {}; for (i in e) s = e[i], t[i] !== s && (E[i] || !V.fx.step[i] && isNaN(parseFloat(s)) || (n[i] = s)); return n }(this.start, this.end), this }), i.attr("class", t), e = e.map(function () { var t = this, e = V.Deferred(), i = V.extend({}, o, { queue: !1, complete: function () { e.resolve(t) } }); return this.el.animate(this.diff, i), e.promise() }), V.when.apply(V, e.get()).done(function () { s(), V.each(arguments, function () { var e = this.el; V.each(this.diff, function (t) { e.css(t, "") }) }), o.complete.call(i[0]) }) }) }, V.fn.extend({ addClass: (O = V.fn.addClass, function (t, e, i, s) { return e ? V.effects.animateClass.call(this, { add: t }, e, i, s) : O.apply(this, arguments) }), removeClass: (A = V.fn.removeClass, function (t, e, i, s) { return 1 < arguments.length ? V.effects.animateClass.call(this, { remove: t }, e, i, s) : A.apply(this, arguments) }), toggleClass: (z = V.fn.toggleClass, function (t, e, i, s, n) { return "boolean" == typeof e || void 0 === e ? i ? V.effects.animateClass.call(this, e ? { add: t } : { remove: t }, i, s, n) : z.apply(this, arguments) : V.effects.animateClass.call(this, { toggle: t }, e, i, s) }), switchClass: function (t, e, i, s, n) { return V.effects.animateClass.call(this, { add: e, remove: t }, i, s, n) } }), V.expr && V.expr.pseudos && V.expr.pseudos.animated && (V.expr.pseudos.animated = (W = V.expr.pseudos.animated, function (t) { return !!V(t).data(K) || W(t) })), !1 !== V.uiBackCompat && V.extend(V.effects, { save: function (t, e) { for (var i = 0, s = e.length; i < s; i++)null !== e[i] && t.data(j + e[i], t[0].style[e[i]]) }, restore: function (t, e) { for (var i, s = 0, n = e.length; s < n; s++)null !== e[s] && (i = t.data(j + e[s]), t.css(e[s], i)) }, setMode: function (t, e) { return e = "toggle" === e ? t.is(":hidden") ? "show" : "hide" : e }, createWrapper: function (i) { if (i.parent().is(".ui-effects-wrapper")) return i.parent(); var s = { width: i.outerWidth(!0), height: i.outerHeight(!0), float: i.css("float") }, t = V("

    ").addClass("ui-effects-wrapper").css({ fontSize: "100%", background: "transparent", border: "none", margin: 0, padding: 0 }), e = { width: i.width(), height: i.height() }, n = document.activeElement; try { n.id } catch (t) { n = document.body } return i.wrap(t), i[0] !== n && !V.contains(i[0], n) || V(n).trigger("focus"), t = i.parent(), "static" === i.css("position") ? (t.css({ position: "relative" }), i.css({ position: "relative" })) : (V.extend(s, { position: i.css("position"), zIndex: i.css("z-index") }), V.each(["top", "left", "bottom", "right"], function (t, e) { s[e] = i.css(e), isNaN(parseInt(s[e], 10)) && (s[e] = "auto") }), i.css({ position: "relative", top: 0, left: 0, right: "auto", bottom: "auto" })), i.css(e), t.css(s).show() }, removeWrapper: function (t) { var e = document.activeElement; return t.parent().is(".ui-effects-wrapper") && (t.parent().replaceWith(t), t[0] !== e && !V.contains(t[0], e) || V(e).trigger("focus")), t } }), V.extend(V.effects, { version: "1.13.2", define: function (t, e, i) { return i || (i = e, e = "effect"), V.effects.effect[t] = i, V.effects.effect[t].mode = e, i }, scaledDimensions: function (t, e, i) { if (0 === e) return { height: 0, width: 0, outerHeight: 0, outerWidth: 0 }; var s = "horizontal" !== i ? (e || 100) / 100 : 1, e = "vertical" !== i ? (e || 100) / 100 : 1; return { height: t.height() * e, width: t.width() * s, outerHeight: t.outerHeight() * e, outerWidth: t.outerWidth() * s } }, clipToBox: function (t) { return { width: t.clip.right - t.clip.left, height: t.clip.bottom - t.clip.top, left: t.clip.left, top: t.clip.top } }, unshift: function (t, e, i) { var s = t.queue(); 1 < e && s.splice.apply(s, [1, 0].concat(s.splice(e, i))), t.dequeue() }, saveStyle: function (t) { t.data(q, t[0].style.cssText) }, restoreStyle: function (t) { t[0].style.cssText = t.data(q) || "", t.removeData(q) }, mode: function (t, e) { t = t.is(":hidden"); return "toggle" === e && (e = t ? "show" : "hide"), e = (t ? "hide" === e : "show" === e) ? "none" : e }, getBaseline: function (t, e) { var i, s; switch (t[0]) { case "top": i = 0; break; case "middle": i = .5; break; case "bottom": i = 1; break; default: i = t[0] / e.height }switch (t[1]) { case "left": s = 0; break; case "center": s = .5; break; case "right": s = 1; break; default: s = t[1] / e.width }return { x: s, y: i } }, createPlaceholder: function (t) { var e, i = t.css("position"), s = t.position(); return t.css({ marginTop: t.css("marginTop"), marginBottom: t.css("marginBottom"), marginLeft: t.css("marginLeft"), marginRight: t.css("marginRight") }).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()), /^(static|relative)/.test(i) && (i = "absolute", e = V("<" + t[0].nodeName + ">").insertAfter(t).css({ display: /^(inline|ruby)/.test(t.css("display")) ? "inline-block" : "block", visibility: "hidden", marginTop: t.css("marginTop"), marginBottom: t.css("marginBottom"), marginLeft: t.css("marginLeft"), marginRight: t.css("marginRight"), float: t.css("float") }).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()).addClass("ui-effects-placeholder"), t.data(j + "placeholder", e)), t.css({ position: i, left: s.left, top: s.top }), e }, removePlaceholder: function (t) { var e = j + "placeholder", i = t.data(e); i && (i.remove(), t.removeData(e)) }, cleanUp: function (t) { V.effects.restoreStyle(t), V.effects.removePlaceholder(t) }, setTransition: function (s, t, n, o) { return o = o || {}, V.each(t, function (t, e) { var i = s.cssUnit(e); 0 < i[0] && (o[e] = i[0] * n + i[1]) }), o } }), V.fn.extend({ effect: function () { function t(t) { var e = V(this), i = V.effects.mode(e, r) || o; e.data(K, !0), l.push(i), o && ("show" === i || i === o && "hide" === i) && e.show(), o && "none" === i || V.effects.saveStyle(e), "function" == typeof t && t() } var s = X.apply(this, arguments), n = V.effects.effect[s.effect], o = n.mode, e = s.queue, i = e || "fx", a = s.complete, r = s.mode, l = []; return V.fx.off || !n ? r ? this[r](s.duration, a) : this.each(function () { a && a.call(this) }) : !1 === e ? this.each(t).each(h) : this.queue(i, t).queue(i, h); function h(t) { var e = V(this); function i() { "function" == typeof a && a.call(e[0]), "function" == typeof t && t() } s.mode = l.shift(), !1 === V.uiBackCompat || o ? "none" === s.mode ? (e[r](), i()) : n.call(e[0], s, function () { e.removeData(K), V.effects.cleanUp(e), "hide" === s.mode && e.hide(), i() }) : (e.is(":hidden") ? "hide" === r : "show" === r) ? (e[r](), i()) : n.call(e[0], s, i) } }, show: (R = V.fn.show, function (t) { if ($(t)) return R.apply(this, arguments); t = X.apply(this, arguments); return t.mode = "show", this.effect.call(this, t) }), hide: (L = V.fn.hide, function (t) { if ($(t)) return L.apply(this, arguments); t = X.apply(this, arguments); return t.mode = "hide", this.effect.call(this, t) }), toggle: (F = V.fn.toggle, function (t) { if ($(t) || "boolean" == typeof t) return F.apply(this, arguments); t = X.apply(this, arguments); return t.mode = "toggle", this.effect.call(this, t) }), cssUnit: function (t) { var i = this.css(t), s = []; return V.each(["em", "px", "%", "pt"], function (t, e) { 0 < i.indexOf(e) && (s = [parseFloat(i), e]) }), s }, cssClip: function (t) { return t ? this.css("clip", "rect(" + t.top + "px " + t.right + "px " + t.bottom + "px " + t.left + "px)") : G(this.css("clip"), this) }, transfer: function (t, e) { var i = V(this), s = V(t.to), n = "fixed" === s.css("position"), o = V("body"), a = n ? o.scrollTop() : 0, r = n ? o.scrollLeft() : 0, o = s.offset(), o = { top: o.top - a, left: o.left - r, height: s.innerHeight(), width: s.innerWidth() }, s = i.offset(), l = V("
    "); l.appendTo("body").addClass(t.className).css({ top: s.top - a, left: s.left - r, height: i.innerHeight(), width: i.innerWidth(), position: n ? "fixed" : "absolute" }).animate(o, t.duration, t.easing, function () { l.remove(), "function" == typeof e && e() }) } }), V.fx.step.clip = function (t) { t.clipInit || (t.start = V(t.elem).cssClip(), "string" == typeof t.end && (t.end = G(t.end, t.elem)), t.clipInit = !0), V(t.elem).cssClip({ top: t.pos * (t.end.top - t.start.top) + t.start.top, right: t.pos * (t.end.right - t.start.right) + t.start.right, bottom: t.pos * (t.end.bottom - t.start.bottom) + t.start.bottom, left: t.pos * (t.end.left - t.start.left) + t.start.left }) }, Y = {}, V.each(["Quad", "Cubic", "Quart", "Quint", "Expo"], function (e, t) { Y[t] = function (t) { return Math.pow(t, e + 2) } }), V.extend(Y, { Sine: function (t) { return 1 - Math.cos(t * Math.PI / 2) }, Circ: function (t) { return 1 - Math.sqrt(1 - t * t) }, Elastic: function (t) { return 0 === t || 1 === t ? t : -Math.pow(2, 8 * (t - 1)) * Math.sin((80 * (t - 1) - 7.5) * Math.PI / 15) }, Back: function (t) { return t * t * (3 * t - 2) }, Bounce: function (t) { for (var e, i = 4; t < ((e = Math.pow(2, --i)) - 1) / 11;); return 1 / Math.pow(4, 3 - i) - 7.5625 * Math.pow((3 * e - 2) / 22 - t, 2) } }), V.each(Y, function (t, e) { V.easing["easeIn" + t] = e, V.easing["easeOut" + t] = function (t) { return 1 - e(1 - t) }, V.easing["easeInOut" + t] = function (t) { return t < .5 ? e(2 * t) / 2 : 1 - e(-2 * t + 2) / 2 } }); y = V.effects, V.effects.define("blind", "hide", function (t, e) { var i = { up: ["bottom", "top"], vertical: ["bottom", "top"], down: ["top", "bottom"], left: ["right", "left"], horizontal: ["right", "left"], right: ["left", "right"] }, s = V(this), n = t.direction || "up", o = s.cssClip(), a = { clip: V.extend({}, o) }, r = V.effects.createPlaceholder(s); a.clip[i[n][0]] = a.clip[i[n][1]], "show" === t.mode && (s.cssClip(a.clip), r && r.css(V.effects.clipToBox(a)), a.clip = o), r && r.animate(V.effects.clipToBox(a), t.duration, t.easing), s.animate(a, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), V.effects.define("bounce", function (t, e) { var i, s, n = V(this), o = t.mode, a = "hide" === o, r = "show" === o, l = t.direction || "up", h = t.distance, c = t.times || 5, o = 2 * c + (r || a ? 1 : 0), u = t.duration / o, d = t.easing, p = "up" === l || "down" === l ? "top" : "left", f = "up" === l || "left" === l, g = 0, t = n.queue().length; for (V.effects.createPlaceholder(n), l = n.css(p), h = h || n["top" == p ? "outerHeight" : "outerWidth"]() / 3, r && ((s = { opacity: 1 })[p] = l, n.css("opacity", 0).css(p, f ? 2 * -h : 2 * h).animate(s, u, d)), a && (h /= Math.pow(2, c - 1)), (s = {})[p] = l; g < c; g++)(i = {})[p] = (f ? "-=" : "+=") + h, n.animate(i, u, d).animate(s, u, d), h = a ? 2 * h : h / 2; a && ((i = { opacity: 0 })[p] = (f ? "-=" : "+=") + h, n.animate(i, u, d)), n.queue(e), V.effects.unshift(n, t, 1 + o) }), V.effects.define("clip", "hide", function (t, e) { var i = {}, s = V(this), n = t.direction || "vertical", o = "both" === n, a = o || "horizontal" === n, o = o || "vertical" === n, n = s.cssClip(); i.clip = { top: o ? (n.bottom - n.top) / 2 : n.top, right: a ? (n.right - n.left) / 2 : n.right, bottom: o ? (n.bottom - n.top) / 2 : n.bottom, left: a ? (n.right - n.left) / 2 : n.left }, V.effects.createPlaceholder(s), "show" === t.mode && (s.cssClip(i.clip), i.clip = n), s.animate(i, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), V.effects.define("drop", "hide", function (t, e) { var i = V(this), s = "show" === t.mode, n = t.direction || "left", o = "up" === n || "down" === n ? "top" : "left", a = "up" === n || "left" === n ? "-=" : "+=", r = "+=" == a ? "-=" : "+=", l = { opacity: 0 }; V.effects.createPlaceholder(i), n = t.distance || i["top" == o ? "outerHeight" : "outerWidth"](!0) / 2, l[o] = a + n, s && (i.css(l), l[o] = r + n, l.opacity = 1), i.animate(l, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), V.effects.define("explode", "hide", function (t, e) { var i, s, n, o, a, r, l = t.pieces ? Math.round(Math.sqrt(t.pieces)) : 3, h = l, c = V(this), u = "show" === t.mode, d = c.show().css("visibility", "hidden").offset(), p = Math.ceil(c.outerWidth() / h), f = Math.ceil(c.outerHeight() / l), g = []; function m() { g.push(this), g.length === l * h && (c.css({ visibility: "visible" }), V(g).remove(), e()) } for (i = 0; i < l; i++)for (o = d.top + i * f, r = i - (l - 1) / 2, s = 0; s < h; s++)n = d.left + s * p, a = s - (h - 1) / 2, c.clone().appendTo("body").wrap("
    ").css({ position: "absolute", visibility: "visible", left: -s * p, top: -i * f }).parent().addClass("ui-effects-explode").css({ position: "absolute", overflow: "hidden", width: p, height: f, left: n + (u ? a * p : 0), top: o + (u ? r * f : 0), opacity: u ? 0 : 1 }).animate({ left: n + (u ? 0 : a * p), top: o + (u ? 0 : r * f), opacity: u ? 1 : 0 }, t.duration || 500, t.easing, m) }), V.effects.define("fade", "toggle", function (t, e) { var i = "show" === t.mode; V(this).css("opacity", i ? 0 : 1).animate({ opacity: i ? 1 : 0 }, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), V.effects.define("fold", "hide", function (e, t) { var i = V(this), s = e.mode, n = "show" === s, o = "hide" === s, a = e.size || 15, r = /([0-9]+)%/.exec(a), l = !!e.horizFirst ? ["right", "bottom"] : ["bottom", "right"], h = e.duration / 2, c = V.effects.createPlaceholder(i), u = i.cssClip(), d = { clip: V.extend({}, u) }, p = { clip: V.extend({}, u) }, f = [u[l[0]], u[l[1]]], s = i.queue().length; r && (a = parseInt(r[1], 10) / 100 * f[o ? 0 : 1]), d.clip[l[0]] = a, p.clip[l[0]] = a, p.clip[l[1]] = 0, n && (i.cssClip(p.clip), c && c.css(V.effects.clipToBox(p)), p.clip = u), i.queue(function (t) { c && c.animate(V.effects.clipToBox(d), h, e.easing).animate(V.effects.clipToBox(p), h, e.easing), t() }).animate(d, h, e.easing).animate(p, h, e.easing).queue(t), V.effects.unshift(i, s, 4) }), V.effects.define("highlight", "show", function (t, e) { var i = V(this), s = { backgroundColor: i.css("backgroundColor") }; "hide" === t.mode && (s.opacity = 0), V.effects.saveStyle(i), i.css({ backgroundImage: "none", backgroundColor: t.color || "#ffff99" }).animate(s, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), V.effects.define("size", function (s, e) { var n, i = V(this), t = ["fontSize"], o = ["borderTopWidth", "borderBottomWidth", "paddingTop", "paddingBottom"], a = ["borderLeftWidth", "borderRightWidth", "paddingLeft", "paddingRight"], r = s.mode, l = "effect" !== r, h = s.scale || "both", c = s.origin || ["middle", "center"], u = i.css("position"), d = i.position(), p = V.effects.scaledDimensions(i), f = s.from || p, g = s.to || V.effects.scaledDimensions(i, 0); V.effects.createPlaceholder(i), "show" === r && (r = f, f = g, g = r), n = { from: { y: f.height / p.height, x: f.width / p.width }, to: { y: g.height / p.height, x: g.width / p.width } }, "box" !== h && "both" !== h || (n.from.y !== n.to.y && (f = V.effects.setTransition(i, o, n.from.y, f), g = V.effects.setTransition(i, o, n.to.y, g)), n.from.x !== n.to.x && (f = V.effects.setTransition(i, a, n.from.x, f), g = V.effects.setTransition(i, a, n.to.x, g))), "content" !== h && "both" !== h || n.from.y !== n.to.y && (f = V.effects.setTransition(i, t, n.from.y, f), g = V.effects.setTransition(i, t, n.to.y, g)), c && (c = V.effects.getBaseline(c, p), f.top = (p.outerHeight - f.outerHeight) * c.y + d.top, f.left = (p.outerWidth - f.outerWidth) * c.x + d.left, g.top = (p.outerHeight - g.outerHeight) * c.y + d.top, g.left = (p.outerWidth - g.outerWidth) * c.x + d.left), delete f.outerHeight, delete f.outerWidth, i.css(f), "content" !== h && "both" !== h || (o = o.concat(["marginTop", "marginBottom"]).concat(t), a = a.concat(["marginLeft", "marginRight"]), i.find("*[width]").each(function () { var t = V(this), e = V.effects.scaledDimensions(t), i = { height: e.height * n.from.y, width: e.width * n.from.x, outerHeight: e.outerHeight * n.from.y, outerWidth: e.outerWidth * n.from.x }, e = { height: e.height * n.to.y, width: e.width * n.to.x, outerHeight: e.height * n.to.y, outerWidth: e.width * n.to.x }; n.from.y !== n.to.y && (i = V.effects.setTransition(t, o, n.from.y, i), e = V.effects.setTransition(t, o, n.to.y, e)), n.from.x !== n.to.x && (i = V.effects.setTransition(t, a, n.from.x, i), e = V.effects.setTransition(t, a, n.to.x, e)), l && V.effects.saveStyle(t), t.css(i), t.animate(e, s.duration, s.easing, function () { l && V.effects.restoreStyle(t) }) })), i.animate(g, { queue: !1, duration: s.duration, easing: s.easing, complete: function () { var t = i.offset(); 0 === g.opacity && i.css("opacity", f.opacity), l || (i.css("position", "static" === u ? "relative" : u).offset(t), V.effects.saveStyle(i)), e() } }) }), V.effects.define("scale", function (t, e) { var i = V(this), s = t.mode, s = parseInt(t.percent, 10) || (0 === parseInt(t.percent, 10) || "effect" !== s ? 0 : 100), s = V.extend(!0, { from: V.effects.scaledDimensions(i), to: V.effects.scaledDimensions(i, s, t.direction || "both"), origin: t.origin || ["middle", "center"] }, t); t.fade && (s.from.opacity = 1, s.to.opacity = 0), V.effects.effect.size.call(this, s, e) }), V.effects.define("puff", "hide", function (t, e) { t = V.extend(!0, {}, t, { fade: !0, percent: parseInt(t.percent, 10) || 150 }); V.effects.effect.scale.call(this, t, e) }), V.effects.define("pulsate", "show", function (t, e) { var i = V(this), s = t.mode, n = "show" === s, o = 2 * (t.times || 5) + (n || "hide" === s ? 1 : 0), a = t.duration / o, r = 0, l = 1, s = i.queue().length; for (!n && i.is(":visible") || (i.css("opacity", 0).show(), r = 1); l < o; l++)i.animate({ opacity: r }, a, t.easing), r = 1 - r; i.animate({ opacity: r }, a, t.easing), i.queue(e), V.effects.unshift(i, s, 1 + o) }), V.effects.define("shake", function (t, e) { var i = 1, s = V(this), n = t.direction || "left", o = t.distance || 20, a = t.times || 3, r = 2 * a + 1, l = Math.round(t.duration / r), h = "up" === n || "down" === n ? "top" : "left", c = "up" === n || "left" === n, u = {}, d = {}, p = {}, n = s.queue().length; for (V.effects.createPlaceholder(s), u[h] = (c ? "-=" : "+=") + o, d[h] = (c ? "+=" : "-=") + 2 * o, p[h] = (c ? "-=" : "+=") + 2 * o, s.animate(u, l, t.easing); i < a; i++)s.animate(d, l, t.easing).animate(p, l, t.easing); s.animate(d, l, t.easing).animate(u, l / 2, t.easing).queue(e), V.effects.unshift(s, n, 1 + r) }), V.effects.define("slide", "show", function (t, e) { var i, s, n = V(this), o = { up: ["bottom", "top"], down: ["top", "bottom"], left: ["right", "left"], right: ["left", "right"] }, a = t.mode, r = t.direction || "left", l = "up" === r || "down" === r ? "top" : "left", h = "up" === r || "left" === r, c = t.distance || n["top" == l ? "outerHeight" : "outerWidth"](!0), u = {}; V.effects.createPlaceholder(n), i = n.cssClip(), s = n.position()[l], u[l] = (h ? -1 : 1) * c + s, u.clip = n.cssClip(), u.clip[o[r][1]] = u.clip[o[r][0]], "show" === a && (n.cssClip(u.clip), n.css(l, u[l]), u.clip = i, u[l] = s), n.animate(u, { queue: !1, duration: t.duration, easing: t.easing, complete: e }) }), y = !1 !== V.uiBackCompat ? V.effects.define("transfer", function (t, e) { V(this).transfer(t, e) }) : y; V.ui.focusable = function (t, e) { var i, s, n, o, a = t.nodeName.toLowerCase(); return "area" === a ? (s = (i = t.parentNode).name, !(!t.href || !s || "map" !== i.nodeName.toLowerCase()) && (0 < (s = V("img[usemap='#" + s + "']")).length && s.is(":visible"))) : (/^(input|select|textarea|button|object)$/.test(a) ? (n = !t.disabled) && (o = V(t).closest("fieldset")[0]) && (n = !o.disabled) : n = "a" === a && t.href || e, n && V(t).is(":visible") && function (t) { var e = t.css("visibility"); for (; "inherit" === e;)t = t.parent(), e = t.css("visibility"); return "visible" === e }(V(t))) }, V.extend(V.expr.pseudos, { focusable: function (t) { return V.ui.focusable(t, null != V.attr(t, "tabindex")) } }); var Q, J; V.ui.focusable, V.fn._form = function () { return "string" == typeof this[0].form ? this.closest("form") : V(this[0].form) }, V.ui.formResetMixin = { _formResetHandler: function () { var e = V(this); setTimeout(function () { var t = e.data("ui-form-reset-instances"); V.each(t, function () { this.refresh() }) }) }, _bindFormResetHandler: function () { var t; this.form = this.element._form(), this.form.length && ((t = this.form.data("ui-form-reset-instances") || []).length || this.form.on("reset.ui-form-reset", this._formResetHandler), t.push(this), this.form.data("ui-form-reset-instances", t)) }, _unbindFormResetHandler: function () { var t; this.form.length && ((t = this.form.data("ui-form-reset-instances")).splice(V.inArray(this, t), 1), t.length ? this.form.data("ui-form-reset-instances", t) : this.form.removeData("ui-form-reset-instances").off("reset.ui-form-reset")) } }; V.expr.pseudos || (V.expr.pseudos = V.expr[":"]), V.uniqueSort || (V.uniqueSort = V.unique), V.escapeSelector || (Q = /([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g, J = function (t, e) { return e ? "\0" === t ? "�" : t.slice(0, -1) + "\\" + t.charCodeAt(t.length - 1).toString(16) + " " : "\\" + t }, V.escapeSelector = function (t) { return (t + "").replace(Q, J) }), V.fn.even && V.fn.odd || V.fn.extend({ even: function () { return this.filter(function (t) { return t % 2 == 0 }) }, odd: function () { return this.filter(function (t) { return t % 2 == 1 }) } }); var Z; V.ui.keyCode = { BACKSPACE: 8, COMMA: 188, DELETE: 46, DOWN: 40, END: 35, ENTER: 13, ESCAPE: 27, HOME: 36, LEFT: 37, PAGE_DOWN: 34, PAGE_UP: 33, PERIOD: 190, RIGHT: 39, SPACE: 32, TAB: 9, UP: 38 }, V.fn.labels = function () { var t, e, i; return this.length ? this[0].labels && this[0].labels.length ? this.pushStack(this[0].labels) : (e = this.eq(0).parents("label"), (t = this.attr("id")) && (i = (i = this.eq(0).parents().last()).add((i.length ? i : this).siblings()), t = "label[for='" + V.escapeSelector(t) + "']", e = e.add(i.find(t).addBack(t))), this.pushStack(e)) : this.pushStack([]) }, V.fn.scrollParent = function (t) { var e = this.css("position"), i = "absolute" === e, s = t ? /(auto|scroll|hidden)/ : /(auto|scroll)/, t = this.parents().filter(function () { var t = V(this); return (!i || "static" !== t.css("position")) && s.test(t.css("overflow") + t.css("overflow-y") + t.css("overflow-x")) }).eq(0); return "fixed" !== e && t.length ? t : V(this[0].ownerDocument || document) }, V.extend(V.expr.pseudos, { tabbable: function (t) { var e = V.attr(t, "tabindex"), i = null != e; return (!i || 0 <= e) && V.ui.focusable(t, i) } }), V.fn.extend({ uniqueId: (Z = 0, function () { return this.each(function () { this.id || (this.id = "ui-id-" + ++Z) }) }), removeUniqueId: function () { return this.each(function () { /^ui-id-\d+$/.test(this.id) && V(this).removeAttr("id") }) } }), V.widget("ui.accordion", { version: "1.13.2", options: { active: 0, animate: {}, classes: { "ui-accordion-header": "ui-corner-top", "ui-accordion-header-collapsed": "ui-corner-all", "ui-accordion-content": "ui-corner-bottom" }, collapsible: !1, event: "click", header: function (t) { return t.find("> li > :first-child").add(t.find("> :not(li)").even()) }, heightStyle: "auto", icons: { activeHeader: "ui-icon-triangle-1-s", header: "ui-icon-triangle-1-e" }, activate: null, beforeActivate: null }, hideProps: { borderTopWidth: "hide", borderBottomWidth: "hide", paddingTop: "hide", paddingBottom: "hide", height: "hide" }, showProps: { borderTopWidth: "show", borderBottomWidth: "show", paddingTop: "show", paddingBottom: "show", height: "show" }, _create: function () { var t = this.options; this.prevShow = this.prevHide = V(), this._addClass("ui-accordion", "ui-widget ui-helper-reset"), this.element.attr("role", "tablist"), t.collapsible || !1 !== t.active && null != t.active || (t.active = 0), this._processPanels(), t.active < 0 && (t.active += this.headers.length), this._refresh() }, _getCreateEventData: function () { return { header: this.active, panel: this.active.length ? this.active.next() : V() } }, _createIcons: function () { var t, e = this.options.icons; e && (t = V(""), this._addClass(t, "ui-accordion-header-icon", "ui-icon " + e.header), t.prependTo(this.headers), t = this.active.children(".ui-accordion-header-icon"), this._removeClass(t, e.header)._addClass(t, null, e.activeHeader)._addClass(this.headers, "ui-accordion-icons")) }, _destroyIcons: function () { this._removeClass(this.headers, "ui-accordion-icons"), this.headers.children(".ui-accordion-header-icon").remove() }, _destroy: function () { var t; this.element.removeAttr("role"), this.headers.removeAttr("role aria-expanded aria-selected aria-controls tabIndex").removeUniqueId(), this._destroyIcons(), t = this.headers.next().css("display", "").removeAttr("role aria-hidden aria-labelledby").removeUniqueId(), "content" !== this.options.heightStyle && t.css("height", "") }, _setOption: function (t, e) { "active" !== t ? ("event" === t && (this.options.event && this._off(this.headers, this.options.event), this._setupEvents(e)), this._super(t, e), "collapsible" !== t || e || !1 !== this.options.active || this._activate(0), "icons" === t && (this._destroyIcons(), e && this._createIcons())) : this._activate(e) }, _setOptionDisabled: function (t) { this._super(t), this.element.attr("aria-disabled", t), this._toggleClass(null, "ui-state-disabled", !!t), this._toggleClass(this.headers.add(this.headers.next()), null, "ui-state-disabled", !!t) }, _keydown: function (t) { if (!t.altKey && !t.ctrlKey) { var e = V.ui.keyCode, i = this.headers.length, s = this.headers.index(t.target), n = !1; switch (t.keyCode) { case e.RIGHT: case e.DOWN: n = this.headers[(s + 1) % i]; break; case e.LEFT: case e.UP: n = this.headers[(s - 1 + i) % i]; break; case e.SPACE: case e.ENTER: this._eventHandler(t); break; case e.HOME: n = this.headers[0]; break; case e.END: n = this.headers[i - 1] }n && (V(t.target).attr("tabIndex", -1), V(n).attr("tabIndex", 0), V(n).trigger("focus"), t.preventDefault()) } }, _panelKeyDown: function (t) { t.keyCode === V.ui.keyCode.UP && t.ctrlKey && V(t.currentTarget).prev().trigger("focus") }, refresh: function () { var t = this.options; this._processPanels(), !1 === t.active && !0 === t.collapsible || !this.headers.length ? (t.active = !1, this.active = V()) : !1 === t.active ? this._activate(0) : this.active.length && !V.contains(this.element[0], this.active[0]) ? this.headers.length === this.headers.find(".ui-state-disabled").length ? (t.active = !1, this.active = V()) : this._activate(Math.max(0, t.active - 1)) : t.active = this.headers.index(this.active), this._destroyIcons(), this._refresh() }, _processPanels: function () { var t = this.headers, e = this.panels; "function" == typeof this.options.header ? this.headers = this.options.header(this.element) : this.headers = this.element.find(this.options.header), this._addClass(this.headers, "ui-accordion-header ui-accordion-header-collapsed", "ui-state-default"), this.panels = this.headers.next().filter(":not(.ui-accordion-content-active)").hide(), this._addClass(this.panels, "ui-accordion-content", "ui-helper-reset ui-widget-content"), e && (this._off(t.not(this.headers)), this._off(e.not(this.panels))) }, _refresh: function () { var i, t = this.options, e = t.heightStyle, s = this.element.parent(); this.active = this._findActive(t.active), this._addClass(this.active, "ui-accordion-header-active", "ui-state-active")._removeClass(this.active, "ui-accordion-header-collapsed"), this._addClass(this.active.next(), "ui-accordion-content-active"), this.active.next().show(), this.headers.attr("role", "tab").each(function () { var t = V(this), e = t.uniqueId().attr("id"), i = t.next(), s = i.uniqueId().attr("id"); t.attr("aria-controls", s), i.attr("aria-labelledby", e) }).next().attr("role", "tabpanel"), this.headers.not(this.active).attr({ "aria-selected": "false", "aria-expanded": "false", tabIndex: -1 }).next().attr({ "aria-hidden": "true" }).hide(), this.active.length ? this.active.attr({ "aria-selected": "true", "aria-expanded": "true", tabIndex: 0 }).next().attr({ "aria-hidden": "false" }) : this.headers.eq(0).attr("tabIndex", 0), this._createIcons(), this._setupEvents(t.event), "fill" === e ? (i = s.height(), this.element.siblings(":visible").each(function () { var t = V(this), e = t.css("position"); "absolute" !== e && "fixed" !== e && (i -= t.outerHeight(!0)) }), this.headers.each(function () { i -= V(this).outerHeight(!0) }), this.headers.next().each(function () { V(this).height(Math.max(0, i - V(this).innerHeight() + V(this).height())) }).css("overflow", "auto")) : "auto" === e && (i = 0, this.headers.next().each(function () { var t = V(this).is(":visible"); t || V(this).show(), i = Math.max(i, V(this).css("height", "").height()), t || V(this).hide() }).height(i)) }, _activate: function (t) { t = this._findActive(t)[0]; t !== this.active[0] && (t = t || this.active[0], this._eventHandler({ target: t, currentTarget: t, preventDefault: V.noop })) }, _findActive: function (t) { return "number" == typeof t ? this.headers.eq(t) : V() }, _setupEvents: function (t) { var i = { keydown: "_keydown" }; t && V.each(t.split(" "), function (t, e) { i[e] = "_eventHandler" }), this._off(this.headers.add(this.headers.next())), this._on(this.headers, i), this._on(this.headers.next(), { keydown: "_panelKeyDown" }), this._hoverable(this.headers), this._focusable(this.headers) }, _eventHandler: function (t) { var e = this.options, i = this.active, s = V(t.currentTarget), n = s[0] === i[0], o = n && e.collapsible, a = o ? V() : s.next(), r = i.next(), a = { oldHeader: i, oldPanel: r, newHeader: o ? V() : s, newPanel: a }; t.preventDefault(), n && !e.collapsible || !1 === this._trigger("beforeActivate", t, a) || (e.active = !o && this.headers.index(s), this.active = n ? V() : s, this._toggle(a), this._removeClass(i, "ui-accordion-header-active", "ui-state-active"), e.icons && (i = i.children(".ui-accordion-header-icon"), this._removeClass(i, null, e.icons.activeHeader)._addClass(i, null, e.icons.header)), n || (this._removeClass(s, "ui-accordion-header-collapsed")._addClass(s, "ui-accordion-header-active", "ui-state-active"), e.icons && (n = s.children(".ui-accordion-header-icon"), this._removeClass(n, null, e.icons.header)._addClass(n, null, e.icons.activeHeader)), this._addClass(s.next(), "ui-accordion-content-active"))) }, _toggle: function (t) { var e = t.newPanel, i = this.prevShow.length ? this.prevShow : t.oldPanel; this.prevShow.add(this.prevHide).stop(!0, !0), this.prevShow = e, this.prevHide = i, this.options.animate ? this._animate(e, i, t) : (i.hide(), e.show(), this._toggleComplete(t)), i.attr({ "aria-hidden": "true" }), i.prev().attr({ "aria-selected": "false", "aria-expanded": "false" }), e.length && i.length ? i.prev().attr({ tabIndex: -1, "aria-expanded": "false" }) : e.length && this.headers.filter(function () { return 0 === parseInt(V(this).attr("tabIndex"), 10) }).attr("tabIndex", -1), e.attr("aria-hidden", "false").prev().attr({ "aria-selected": "true", "aria-expanded": "true", tabIndex: 0 }) }, _animate: function (t, i, e) { var s, n, o, a = this, r = 0, l = t.css("box-sizing"), h = t.length && (!i.length || t.index() < i.index()), c = this.options.animate || {}, u = h && c.down || c, h = function () { a._toggleComplete(e) }; return n = (n = "string" == typeof u ? u : n) || u.easing || c.easing, o = (o = "number" == typeof u ? u : o) || u.duration || c.duration, i.length ? t.length ? (s = t.show().outerHeight(), i.animate(this.hideProps, { duration: o, easing: n, step: function (t, e) { e.now = Math.round(t) } }), void t.hide().animate(this.showProps, { duration: o, easing: n, complete: h, step: function (t, e) { e.now = Math.round(t), "height" !== e.prop ? "content-box" === l && (r += e.now) : "content" !== a.options.heightStyle && (e.now = Math.round(s - i.outerHeight() - r), r = 0) } })) : i.animate(this.hideProps, o, n, h) : t.animate(this.showProps, o, n, h) }, _toggleComplete: function (t) { var e = t.oldPanel, i = e.prev(); this._removeClass(e, "ui-accordion-content-active"), this._removeClass(i, "ui-accordion-header-active")._addClass(i, "ui-accordion-header-collapsed"), e.length && (e.parent()[0].className = e.parent()[0].className), this._trigger("activate", null, t) } }), V.ui.safeActiveElement = function (e) { var i; try { i = e.activeElement } catch (t) { i = e.body } return i = !(i = i || e.body).nodeName ? e.body : i }, V.widget("ui.menu", { version: "1.13.2", defaultElement: "