diff --git a/composer.json b/composer.json index 1aaac1cc5..7b3fc6fa1 100644 --- a/composer.json +++ b/composer.json @@ -1,9 +1,9 @@ { - "name": "cakephp/app", - "description": "CakePHP skeleton app", + "name": "misp/misp", + "description": "MISP: Open Source Threat Intelligence Platform & Open Standards For Threat Information Sharing", "homepage": "https://cakephp.org", "type": "project", - "license": "MIT", + "license": " AGPL-3.0", "require": { "php": ">=8.0", "admad/cakephp-social-auth": "^1.1", @@ -19,7 +19,7 @@ "pear/crypt_gpg": "^1.6", "php-http/message": "^1.16", "php-http/message-factory": "^1.1", - "supervisorphp/supervisor": "^4.0" + "supervisorphp/supervisor": "^5.0" }, "require-dev": { "cakephp/bake": "^2.0.3", @@ -86,4 +86,4 @@ "php-http/discovery": true } } -} +} \ No newline at end of file diff --git a/phpstan.neon b/phpstan.neon index 4efab46a4..c83323ddf 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -7,4 +7,6 @@ parameters: - message: '#Undefined variable: \$this#' path: %currentWorkingDirectory%/templates - message: '#Function brotli_compress not found.#' + path: %currentWorkingDirectory%/* + - message: '#Call to an undefined method [a-zA-Z0-9\\_]+::changeKey()#' path: %currentWorkingDirectory%/* \ No newline at end of file diff --git a/src/Command/FeedsCommand.php b/src/Command/FeedsCommand.php new file mode 100644 index 000000000..6b9d8fea8 --- /dev/null +++ b/src/Command/FeedsCommand.php @@ -0,0 +1,283 @@ + 'bin/cake servers test `server_id`', + 'fetchIndex' => 'bin/cake servers fetchIndex `server_id`', + 'fetchFeed' => 'bin/cake servers `fetchFeed` `user_id` feed_id|all|csv|text|misp [job_id]', + 'pullAll' => 'bin/cake servers pullAll `user_id` [full|update]', + 'pull' => 'bin/cake servers pull `user_id` `server_id` [full|update]', + 'push' => 'bin/cake servers push `user_id` `server_id` [full|update] [job_id]', + 'pushAll' => 'bin/cake servers pushAll `user_id` [full|update]', + 'listFeeds' => 'bin/cake servers listFeeds [json|table]', + 'viewFeed' => 'bin/cake servers viewFeed `feed_id` [json|table]', + 'toggleFeed' => 'bin/cake servers toggleFeed `feed_id`', + 'toggleFeedCaching' => 'bin/cake servers toggleFeedCaching `feed_id`', + 'cacheServer' => 'bin/cake servers cacheServer `user_id` `server_id|all` [job_id]', + 'cacheServerAll' => 'bin/cake servers cacheServerAll `user_id` [job_id]', + 'cacheFeed' => 'bin/cake servers cacheFeed `user_id` [feed_id|all|csv|text|misp] [job_id]', + ]; + + public function fetchFeed($userId, $feedId, $jobId = null) + { + if (empty($userId) || empty($feedId)) { + $this->showActionUsageAndExit(); + } + + $UsersTable = $this->fetchTable('Users'); + $user = $UsersTable->getAuthUser($userId, true); + + Configure::write('CurrentUserId', $userId); + + $JobsTable = $this->fetchTable('Jobs'); + + if (empty($jobId)) { + $jobId = $JobsTable->createJob($user->toArray(), Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.'); + } + if ($feedId === 'all') { + $feedIds = $this->Feeds->find( + 'column', + [ + 'fields' => ['id'], + 'conditions' => ['enabled' => 1] + ] + )->toArray(); + $successes = 0; + $fails = 0; + foreach ($feedIds as $k => $feedId) { + $JobsTable->saveProgress($jobId, 'Fetching feed: ' . $feedId, 100 * $k / count($feedIds)); + $result = $this->Feeds->downloadFromFeedInitiator($feedId, $user); + if ($result) { + $successes++; + } else { + $fails++; + } + } + $message = 'Job done. ' . $successes . ' feeds pulled successfully, ' . $fails . ' feeds could not be pulled.'; + $JobsTable->saveStatus($jobId, true, $message); + $this->io->out($message); + } else { + $feedEnabled = $this->Feeds->exists( + [ + 'enabled' => 1, + 'id' => $feedId, + ] + ); + if ($feedEnabled) { + $result = $this->Feeds->downloadFromFeedInitiator($feedId, $user, $jobId); + if (!$result) { + $JobsTable->saveStatus($jobId, false, 'Job failed. See error log for more details.'); + $this->io->error('Job failed.'); + } else { + $JobsTable->saveStatus($jobId, true); + $this->io->out('Job done.'); + } + } else { + $message = "Feed with ID $feedId not found or not enabled."; + $JobsTable->saveStatus($jobId, false, $message); + $this->io->error($message); + } + } + } + + public function listFeeds($outputStyle = 'json') + { + $fields = [ + 'id' => 3, + 'source_format' => 10, + 'provider' => 15, + 'url' => 50, + 'enabled' => 8, + 'caching_enabled' => 7 + ]; + + $feeds = $this->Feeds->find( + 'all', + [ + 'recursive' => -1, + 'fields' => array_keys($fields) + ] + ); + if ($outputStyle === 'table') { + $this->io->out(str_repeat('=', 114)); + $this->io->out( + sprintf( + '| %s | %s | %s | %s | %s | %s |', + str_pad('ID', $fields['id'], ' ', STR_PAD_RIGHT), + str_pad('Format', $fields['source_format'], ' ', STR_PAD_RIGHT), + str_pad('Provider', $fields['provider'], ' ', STR_PAD_RIGHT), + str_pad('Url', $fields['url'], ' ', STR_PAD_RIGHT), + str_pad('Fetching', $fields['enabled'], ' ', STR_PAD_RIGHT), + str_pad('Caching', $fields['caching_enabled'], ' ', STR_PAD_RIGHT) + ), + 1, + ConsoleIo::NORMAL + ); + $this->io->out(str_repeat('=', 114)); + foreach ($feeds as $feed) { + $this->io->out( + sprintf( + '| %s | %s | %s | %s | %s | %s |', + str_pad($feed['id'], $fields['id'], ' ', STR_PAD_RIGHT), + str_pad($feed['source_format'], $fields['source_format'], ' ', STR_PAD_RIGHT), + str_pad(mb_substr($feed['provider'], 0, 13), $fields['provider'], ' ', STR_PAD_RIGHT), + str_pad( + mb_substr($feed['url'], 0, 48), + $fields['url'], + ' ', + STR_PAD_RIGHT + ), + $feed['enabled'] ? + '' . str_pad(__('Yes'), $fields['enabled'], ' ', STR_PAD_RIGHT) . '' : + str_pad(__('No'), $fields['enabled'], ' ', STR_PAD_RIGHT), + $feed['caching_enabled'] ? + '' . str_pad(__('Yes'), $fields['caching_enabled'], ' ', STR_PAD_RIGHT) . '' : + str_pad(__('No'), $fields['caching_enabled'], ' ', STR_PAD_RIGHT) + ), + 1, + ConsoleIo::NORMAL + ); + } + $this->io->out(str_repeat('=', 114)); + } else { + $this->outputJson($feeds); + } + } + + public function viewFeed($feedId = null, $outputStyle = 'json') + { + if (empty($feedId)) { + $this->showActionUsageAndExit(); + } + + $feed = $this->Feeds->get($feedId)->toArray(); + + if (empty($feed)) { + throw new Exception(__('Invalid feed.')); + } + if ($outputStyle === 'table') { + $this->io->out(str_repeat('=', 114)); + foreach ($feed as $field => $value) { + if (is_array($value)) { + $value = json_encode($value, JSON_PRETTY_PRINT); + } + $this->io->out( + sprintf( + '| %s | %s |', + str_pad($field, 20, ' ', STR_PAD_RIGHT), + str_pad($value ?? '', 87) + ), + 1, + ConsoleIo::NORMAL + ); + } + $this->io->out(str_repeat('=', 114)); + } else { + $this->outputJson($feed); + } + } + + public function toggleFeed($feedId = null) + { + if (empty($feedId)) { + $this->showActionUsageAndExit(); + } + + $feed = $this->Feeds->get($feedId); + + $feed['enabled'] = ($feed['enabled']) ? 0 : 1; + if ($this->Feeds->save($feed)) { + $this->io->out(__('Feed fetching {0} for feed {1}', ($feed['enabled'] ? __('enabled') : __('disabled')), $feed['id'])); + } else { + $this->io->out(__('Could not toggle fetching for feed {0}', $feed['id'])); + } + } + + public function toggleFeedCaching($feedId = null) + { + if (empty($feedId)) { + $this->showActionUsageAndExit(); + } + + $feed = $this->Feeds->get($feedId); + + $feed['caching_enabled'] = ($feed['caching_enabled']) ? 0 : 1; + if ($this->Feeds->save($feed)) { + $this->io->out(__('Feed caching {0} for feed {1}', ($feed['caching_enabled'] ? __('enabled') : __('disabled')), $feed['id'])); + } else { + $this->io->out(__('Could not toggle caching for feed {0}', $feed['id'])); + } + } + + public function loadDefaultFeeds() + { + $this->Feeds->load_default_feeds(); + $this->io->out(__('Default feed metadata loaded.')); + } + + public function cacheFeed($userId = null, $scope = null, $jobId = null) + { + if (empty($userId) || empty($scope)) { + $this->showActionUsageAndExit(); + } + + $user = $this->getUser($userId); + + $JobsTable = $this->fetchTable('Jobs'); + + if (!empty($jobId)) { + $jobId = $JobsTable->createJob($user, Job::WORKER_DEFAULT, 'cache_feeds', 'Feed: ' . $scope, 'Starting feed caching.'); + } + try { + $result = $this->Feeds->cacheFeedInitiator($user, $jobId, $scope); + } catch (Exception $e) { + $this->logException("Failed caching Feed: $scope", $e); + $result = false; + } + + if ($result === false) { + $message = __('Job failed. See error logs for more details.'); + $JobsTable->saveStatus($jobId, false, $message); + } else { + $total = $result['successes'] + $result['fails']; + $message = __( + '{0} feed from {1} cached. Failed: {2}', + $result['successes'], + $total, + $result['fails'] + ); + if ($result['fails'] > 0) { + $message .= ' ' . __('See error logs for more details.'); + } + $JobsTable->saveStatus($jobId, true, $message); + } + $this->io->out($message); + } +} diff --git a/src/Command/MISPCommand.php b/src/Command/MISPCommand.php new file mode 100644 index 000000000..a75d7d2ab --- /dev/null +++ b/src/Command/MISPCommand.php @@ -0,0 +1,101 @@ +io = $io; + + $this->arguments = $args->getArguments(); + $this->action = array_shift($this->arguments); + + if (empty($this->action)) { + $this->showActionUsageAndExit(); + } + + if (!in_array($this->action, $this->validActions)) { + $this->invalidAction(); + } + + call_user_func([$this, $this->action], ...$this->arguments); + + parent::execute($args, $io); + } + + protected function outputJson($data) + { + $this->io->out(json_encode($data, JSON_PRETTY_PRINT)); + } + + protected function showActionUsageAndExit() + { + $this->io->error('Invalid usage.'); + if (!empty($this->usage) && isset($this->usage[$this->action])) { + $this->io->info('Usage: ' . $this->usage[$this->action]); + } + die(); + } + + protected function invalidAction() + { + $this->io->warning('Invalid action.'); + $this->io->out('Valid actions: ' . implode(', ', $this->validActions)); + die(); + } + + /** + * @return BackgroundJobsTool + */ + public function getBackgroundJobsTool(): BackgroundJobsTool + { + if (!self::$loadedBackgroundJobsTool) { + self::$loadedBackgroundJobsTool = new BackgroundJobsTool(Configure::read('BackgroundJobs')); + ; + } + + return self::$loadedBackgroundJobsTool; + } + + /** + * @param int $userId + * @return array + */ + protected function getUser($userId): array + { + $UsersTable = $this->fetchTable('Users'); + $user = $UsersTable->getAuthUser($userId, true); + + if (empty($user)) { + $this->io->error('User ID do not match an existing user.'); + die(); + } + + return $user->toArray(); + } +} diff --git a/src/Command/ServerCommand.php b/src/Command/ServerCommand.php deleted file mode 100644 index 7f3982f63..000000000 --- a/src/Command/ServerCommand.php +++ /dev/null @@ -1,97 +0,0 @@ -io = $io; - - $arguments = $args->getArguments(); - $action = array_shift($arguments); - - switch ($action) { - case 'fetchFeed': - $this->fetchFeed(...$arguments); - break; - default: - $this->io->err('Invalid action.'); - } - $this->io->out("Bye."); - } - - public function fetchFeed($userId, $feedId, $jobId = null) - { - if (empty($userId) || empty($feedId)) { - $this->io->err('Usage: ' . (new Server())->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL); - die(); - } - - $UsersTable = $this->fetchTable('Users'); - $user = $UsersTable->getAuthUser($userId, true); - - Configure::write('CurrentUserId', $userId); - - $FeedsTable = $this->fetchTable('Feeds'); - $JobsTable = $this->fetchTable('Jobs'); - - if (!empty($jobId)) { - $jobId = $this->args[2]; - } else { - $jobId = $JobsTable->createJob($user->toArray(), Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.'); - } - if ($feedId === 'all') { - $feedIds = $FeedsTable->find('column', array( - 'fields' => array('id'), - 'conditions' => array('enabled' => 1) - ))->toArray(); - $successes = 0; - $fails = 0; - foreach ($feedIds as $k => $feedId) { - $JobsTable->saveProgress($jobId, 'Fetching feed: ' . $feedId, 100 * $k / count($feedIds)); - $result = $FeedsTable->downloadFromFeedInitiator($feedId, $user); - if ($result) { - $successes++; - } else { - $fails++; - } - } - $message = 'Job done. ' . $successes . ' feeds pulled successfully, ' . $fails . ' feeds could not be pulled.'; - $JobsTable->saveStatus($jobId, true, $message); - $this->io->out($message); - } else { - $feedEnabled = $FeedsTable->exists([ - 'enabled' => 1, - 'id' => $feedId, - ]); - if ($feedEnabled) { - $result = $FeedsTable->downloadFromFeedInitiator($feedId, $user, $jobId); - if (!$result) { - $JobsTable->saveStatus($jobId, false, 'Job failed. See error log for more details.'); - $this->io->err('Job failed.'); - } else { - $JobsTable->saveStatus($jobId, true); - $this->io->out('Job done.'); - } - } else { - $message = "Feed with ID $feedId not found or not enabled."; - $JobsTable->saveStatus($jobId, false, $message); - $this->io->err($message); - } - } - } -} diff --git a/src/Command/ServersCommand.php b/src/Command/ServersCommand.php new file mode 100644 index 000000000..1c1ca0b07 --- /dev/null +++ b/src/Command/ServersCommand.php @@ -0,0 +1,371 @@ + 'bin/cake servers test `server_id`', + 'fetchIndex' => 'bin/cake servers fetchIndex `server_id`', + 'fetchFeed' => 'bin/cake servers `fetchFeed` `user_id` feed_id|all|csv|text|misp [job_id]', + 'pullAll' => 'bin/cake servers pullAll `user_id` [full|update]', + 'pull' => 'bin/cake servers pull `user_id` `server_id` [full|update]', + 'push' => 'bin/cake servers push `user_id` `server_id` [full|update] [job_id]', + 'pushAll' => 'bin/cake servers pushAll `user_id` [full|update]', + 'listFeeds' => 'bin/cake servers listFeeds [json|table]', + 'viewFeed' => 'bin/cake servers viewFeed `feed_id` [json|table]', + 'toggleFeed' => 'bin/cake servers toggleFeed `feed_id`', + 'toggleFeedCaching' => 'bin/cake servers toggleFeedCaching `feed_id`', + 'cacheServer' => 'bin/cake servers cacheServer `user_id` `server_id|all` [job_id]', + 'cacheServerAll' => 'bin/cake servers cacheServerAll `user_id` [job_id]', + 'cacheFeed' => 'bin/cake servers cacheFeed `user_id` [feed_id|all|csv|text|misp] [job_id]', + ]; + + public function list() + { + $servers = $this->Servers->find( + 'all', + [ + 'fields' => ['id', 'name', 'url'], + 'recursive' => 0 + ] + ); + foreach ($servers as $server) { + $this->io->out( + sprintf( + '%sServer #%s :: %s :: %s', + PHP_EOL, + $server['id'], + $server['name'], + $server['url'] + ) + ); + } + } + + public function listServers() + { + $servers = $this->Servers->find( + 'all', + [ + 'fields' => ['id', 'name', 'url'], + 'recursive' => 0 + ] + )->toArray(); + $res = ['servers' => $servers]; + $this->outputJson($res); + } + + public function test($serverId = null) + { + if (empty($serverId)) { + $this->showActionUsageAndExit(); + } + + $serverId = intval($serverId); + $server = $this->getServer($serverId); + + $res = $this->Servers->runConnectionTest($server, false); + + $this->outputJson($res); + } + + public function fetchIndex($serverId = null) + { + if (empty($serverId)) { + $this->showActionUsageAndExit(); + } + + $server = $this->getServer($serverId); + + $serverSync = new ServerSyncTool($server, $this->Servers->setupSyncRequest($server)); + $index = $this->Servers->getEventIndexFromServer($serverSync); + + $this->outputJson($index); + } + + public function pullAll($userId = null, $technique = 'full') + { + if (empty($userId)) { + $this->showActionUsageAndExit(); + } + + $user = $this->getUser($userId); + + $servers = $this->Servers->find( + 'list', + [ + 'conditions' => ['pull' => 1], + 'recursive' => -1, + 'order' => 'priority', + 'fields' => ['id', 'name'], + ] + )->toArray(); + + foreach ($servers as $serverId => $serverName) { + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob($user, Job::WORKER_DEFAULT, 'pull', "Server: $serverId", 'Pulling.'); + $backgroundJobId = $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'pull', + $user['id'], + $serverId, + $technique, + $jobId, + ], + true, + $jobId + ); + + $this->io->out("Enqueued pulling from $serverName server as job $backgroundJobId"); + } + } + + public function pull($userId = null, $serverId = null, $technique = 'full', $jobId = null, $force = false) + { + if (empty($userId) || empty($serverId)) { + $this->showActionUsageAndExit(); + } + + $user = $this->getUser($userId); + $server = $this->getServer($serverId); + $JobsTable = $this->fetchTable('Jobs'); + + if (empty($jobId)) { + $jobId = $JobsTable->createJob($user, Job::WORKER_DEFAULT, 'pull', 'Server: ' . $serverId, 'Pulling.'); + } + + try { + $result = $this->Servers->pull($user, $technique, $server, $jobId, $force); + if (is_array($result)) { + $message = __('Pull completed. {0} events pulled, {1} events could not be pulled, {2} proposals pulled, {3} sightings pulled, {4} clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); + $JobsTable->saveStatus($jobId, true, $message); + } else { + $message = __('ERROR: {0}', $result); + $JobsTable->saveStatus($jobId, false, $message); + } + } catch (Exception $e) { + $JobsTable->saveStatus($jobId, false, __('ERROR: {0}', $e->getMessage())); + throw $e; + } + + $this->io->out($message); + } + + public function push($userId = null, $serverId = null, $technique = 'full', $jobId = null) + { + if (empty($userId) || empty($serverId)) { + $this->showActionUsageAndExit(); + } + + $JobsTable = $this->fetchTable('Jobs'); + $user = $this->getUser($userId); + $server = $this->getServer($serverId); + if (empty($jobId)) { + $jobId = $JobsTable->createJob($user, Job::WORKER_DEFAULT, 'push', 'Server: ' . $serverId, 'Pushing.'); + } + + $HttpSocket = new HttpTool(); + $HttpSocket->configFromServer($server); + $result = $this->Servers->push($serverId, $technique, $jobId, $HttpSocket, $user); + + if ($result !== true && !is_array($result)) { + $message = 'Job failed. Reason: ' . $result; + $JobsTable->saveStatus($jobId, false, $message); + } else { + $message = 'Job done.'; + $JobsTable->saveStatus($jobId, true, $message); + } + } + + public function pushAll($userId = null, $technique = 'full') + { + $user = $this->getUser($userId); + + $servers = $this->Servers->find( + 'list', + [ + 'conditions' => ['push' => 1], + 'recursive' => -1, + 'order' => 'priority', + 'fields' => ['id', 'name'], + ] + ); + + foreach ($servers as $serverId => $serverName) { + $jobId = $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'push', + $user['id'], + $serverId, + $technique + ] + ); + + $this->io->out("Enqueued pushing from $serverName server as job $jobId"); + } + } + + + + public function cacheServer($userId = null, $scope = null, $jobId = null) + { + if (empty($userId) || empty($scope)) { + $this->showActionUsageAndExit(); + } + + $JobsTable = $this->fetchTable('Jobs'); + + $user = $this->getUser($userId); + if (empty($jobId)) { + $data = [ + 'worker' => 'default', + 'job_type' => 'cache_servers', + 'job_input' => 'Server: ' . $scope, + 'status' => 0, + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'message' => 'Starting server caching.', + ]; + $job = $JobsTable->newEntity($data); + $JobsTable->save($job); + $jobId = $job->id; + } + $result = $this->Servers->cacheServerInitiator($user, $scope, $jobId); + if ($result !== true) { + $message = 'Job Failed. Reason: ' . $result; + $JobsTable->saveStatus($jobId, false, $message); + } else { + $message = 'Job done.'; + $JobsTable->saveStatus($jobId, true, $message); + } + $this->io->out($message); + } + + public function cacheServerAll($userId = null) + { + if (empty($userId)) { + $this->showActionUsageAndExit(); + } + + $user = $this->getUser($userId); + + $servers = $this->Servers->find( + 'list', + [ + 'conditions' => ['pull' => 1], + 'recursive' => -1, + 'order' => 'priority', + 'fields' => ['id', 'name'], + ] + ); + + foreach ($servers as $serverId => $serverName) { + $jobId = $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheServer', + $user['id'], + $serverId + ] + ); + + $this->io->out("Enqueued cacheServer from $serverName server as job $jobId"); + } + } + + + + public function sendPeriodicSummaryToUsers() + { + $periods = $this->__getPeriodsForToday(); + $start_time = time(); + $this->io->out(__('Started periodic summary generation for the {0} period', 'Started periodic summary generation for periods: {1}', count($periods), implode(', ', $periods))); + + $UsersTable = $this->fetchTable('Users'); + foreach ($periods as $period) { + $users = $UsersTable->getSubscribedUsersForPeriod($period); + $this->io->out(__('{0} user has subscribed for the `{1}` period', '{2} users has subscribed for the `{3}` period', count($users), count($users), $period)); + foreach ($users as $user) { + $this->io->out(__('Sending `{0}` report to `{1}`', $period, $user['email'])); + $emailTemplate = $UsersTable->generatePeriodicSummary($user['id'], $period, false); + if ($emailTemplate === null) { + continue; // no new event for this user + } + $UsersTable->sendEmail($user, $emailTemplate, false, null); + } + } + $this->io->out(__('All reports sent. Task took {0} seconds', time() - $start_time)); + } + + private function __getPeriodsForToday(): array + { + $today = new Chronos(); + $periods = ['daily']; + if ($today->format('j') == 1) { + $periods[] = 'monthly'; + } + if ($today->format('N') == 1) { + $periods[] = 'weekly'; + } + return $periods; + } + + /** + * @param int $serverId + * @return array + */ + private function getServer($serverId): array + { + $server = $this->Servers->get($serverId); + + if (!$server) { + $this->io->error("Server with ID $serverId doesn't exists."); + die(); + } + + return $server->toArray(); + } +} diff --git a/src/Controller/Admin/AuditLogsController.php b/src/Controller/Admin/AuditLogsController.php index fac12db99..a96227243 100644 --- a/src/Controller/Admin/AuditLogsController.php +++ b/src/Controller/Admin/AuditLogsController.php @@ -199,7 +199,7 @@ class AuditLogsController extends AppController $this->paginate['conditions'] = $this->__createEventIndexConditions($event); $this->set('passedArgsArray', ['eventId' => $eventId, 'org' => $org]); - $params = $this->IndexFilter->harvestParameters(['created', 'org']); + $params = $this->harvestParameters(['created', 'org']); if ($org) { $params['org'] = $org; } diff --git a/src/Controller/AppController.php b/src/Controller/AppController.php index 50a81fc85..d40a817f7 100644 --- a/src/Controller/AppController.php +++ b/src/Controller/AppController.php @@ -1,4 +1,5 @@ ACL->getUser()->Role->perm_site_admin; } diff --git a/src/Controller/Component/ACLComponent.php b/src/Controller/Component/ACLComponent.php index 9c2c8a76d..c168f50ef 100644 --- a/src/Controller/Component/ACLComponent.php +++ b/src/Controller/Component/ACLComponent.php @@ -293,6 +293,93 @@ class ACLComponent extends Component 'update' => [], 'possibleObjectTemplates' => ['*'], ], + 'Feeds' => [ + 'add' => [], + 'cacheFeeds' => [], + 'compareFeeds' => ['host_org_user'], + 'delete' => [], + 'disable' => [], + 'edit' => [], + 'enable' => [], + 'feedCoverage' => ['host_org_user'], + 'fetchFromAllFeeds' => [], + 'fetchFromFeed' => [], + 'fetchSelectedFromFreetextIndex' => [], + 'getEvent' => [], + 'importFeeds' => [], + 'index' => ['host_org_user'], + 'loadDefaultFeeds' => [], + 'previewEvent' => ['host_org_user'], + 'previewIndex' => ['host_org_user'], + 'searchCaches' => ['host_org_user'], + 'toggleSelected' => [], + 'view' => ['host_org_user'], + ], + 'Servers' => [ + 'add' => [], + 'dbSchemaDiagnostic' => [], + 'dbConfiguration' => [], + 'cache' => [], + 'changePriority' => [], + 'checkout' => [], + 'clearWorkerQueue' => [], + 'createSync' => ['perm_sync'], + 'delete' => [], + 'deleteFile' => [], + 'edit' => [], + 'eventBlockRule' => [], + 'fetchServersForSG' => ['perm_sharing_group'], + 'filterEventIndex' => [], + 'getAvailableSyncFilteringRules' => ['*'], + 'getInstanceUUID' => ['perm_sync'], + 'getPyMISPVersion' => ['*'], + 'getRemoteUser' => [], + 'getSetting' => [], + 'getSubmodulesStatus' => [], + 'getSubmoduleQuickUpdateForm' => [], + 'getWorkers' => [], + 'getVersion' => ['perm_auth'], + 'idTranslator' => ['host_org_user'], + 'import' => [], + 'index' => [], + 'ipUser' => ['perm_site_admin'], + 'ondemandAction' => [], + 'postTest' => ['*'], + 'previewEvent' => [], + 'previewIndex' => [], + 'compareServers' => [], + 'pull' => [], + 'purgeSessions' => [], + 'push' => [], + 'queryAvailableSyncFilteringRules' => [], + 'releaseUpdateLock' => [], + 'resetRemoteAuthKey' => [], + 'removeOrphanedCorrelations' => [], + 'restartDeadWorkers' => [], + 'restartWorkers' => [], + 'serverSettings' => [], + 'serverSettingsEdit' => [], + 'serverSettingsReloadSetting' => [], + 'startWorker' => [], + 'startZeroMQServer' => [], + 'statusZeroMQServer' => [], + 'stopWorker' => [], + 'stopZeroMQServer' => [], + 'testConnection' => [], + 'update' => [], + 'updateJSON' => [], + 'updateProgress' => [], + 'updateSubmodule' => [], + 'uploadFile' => [], + 'killAllWorkers' => [], + 'cspReport' => ['*'], + 'pruneDuplicateUUIDs' => [], + 'removeDuplicateEvents' => [], + 'upgrade2324' => [], + 'cleanModelCaches' => [], + 'updateDatabase' => [], + 'rest' => ['perm_auth'], + ], 'Api' => [ 'index' => ['*'] ] diff --git a/src/Controller/Component/CompressedRequestHandlerComponent.php b/src/Controller/Component/CompressedRequestHandlerComponent.php new file mode 100644 index 000000000..5f8a65ab2 --- /dev/null +++ b/src/Controller/Component/CompressedRequestHandlerComponent.php @@ -0,0 +1,80 @@ +getController(); + $contentEncoding = $_SERVER['HTTP_CONTENT_ENCODING'] ?? null; + + if ($contentEncoding === 'application/json') { + return; + } + if (!empty($contentEncoding)) { + if ($contentEncoding === 'br') { + $controller->request->setInput($this->decodeBrotliEncodedContent($controller)); + } else if ($contentEncoding === 'gzip') { + $controller->request->setInput($this->decodeGzipEncodedContent($controller)); + } else { + throw new BadRequestException("Unsupported content encoding '$contentEncoding'."); + } + } + } + + /** + * @return array + */ + public function supportedEncodings() + { + $supportedEncodings = []; + if (function_exists('gzdecode')) { + $supportedEncodings[] = 'gzip'; + } + if (function_exists('brotli_uncompress')) { + $supportedEncodings[] = 'br'; + } + return $supportedEncodings; + } + + /** + * @return string + * @throws Exception + */ + private function decodeGzipEncodedContent(Controller $controller) + { + if (function_exists('gzdecode')) { + $decoded = gzdecode($controller->request->input()); + if ($decoded === false) { + throw new BadRequestException('Invalid compressed data.'); + } + return $decoded; + } else { + throw new BadRequestException("This server doesn't support GZIP compressed requests."); + } + } + + /** + * @param Controller $controller + * @return string + * @throws Exception + */ + private function decodeBrotliEncodedContent(Controller $controller) + { + if (function_exists('brotli_uncompress')) { + $decoded = brotli_uncompress($controller->request->input()); + if ($decoded === false) { + throw new BadRequestException('Invalid compressed data.'); + } + return $decoded; + } else { + throw new BadRequestException("This server doesn't support brotli compressed requests."); + } + } +} diff --git a/src/Controller/FeedsController.php b/src/Controller/FeedsController.php index 86d93ec43..7a280bc7b 100644 --- a/src/Controller/FeedsController.php +++ b/src/Controller/FeedsController.php @@ -14,13 +14,10 @@ use Cake\Core\Configure; use Cake\Event\EventInterface; use Cake\Http\Exception\MethodNotAllowedException; use Cake\Http\Exception\NotFoundException; -use Cake\ORM\Locator\LocatorAwareTrait; use Exception; class FeedsController extends AppController { - use LocatorAwareTrait; - public $paginate = [ 'limit' => 60, 'order' => [ @@ -525,7 +522,7 @@ class FeedsController extends AppController $this->Feeds->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, - BackgroundJobsTool::CMD_SERVER, + BackgroundJobsTool::CMD_FEEDS, [ 'fetchFeed', $this->ACL->getUser()->id, @@ -596,7 +593,7 @@ class FeedsController extends AppController $this->Feeds->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, - BackgroundJobsTool::CMD_SERVER, + BackgroundJobsTool::CMD_FEEDS, [ 'fetchFeed', $this->Auth->user('id'), @@ -972,7 +969,7 @@ class FeedsController extends AppController $this->Feeds->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, - BackgroundJobsTool::CMD_SERVER, + BackgroundJobsTool::CMD_FEEDS, [ 'cacheFeed', $this->Auth->user('id'), diff --git a/src/Controller/ServersController.php b/src/Controller/ServersController.php new file mode 100644 index 000000000..90d7b779f --- /dev/null +++ b/src/Controller/ServersController.php @@ -0,0 +1,2496 @@ + 60, + 'recursive' => -1, + 'contain' => [ + 'User' => [ + 'fields' => ['User.id', 'User.org_id', 'User.email'], + ], + 'Organisation' => [ + 'fields' => ['Organisation.name', 'Organisation.id'], + ], + 'RemoteOrg' => [ + 'fields' => ['RemoteOrg.name', 'RemoteOrg.id'], + ], + ], + 'maxLimit' => 9999, + 'order' => [ + 'Server.priority' => 'ASC' + ], + ]; + + public function initialize(): void + { + $this->loadComponent('CompressedRequestHandler'); + parent::initialize(); + } + + public function beforeFilter(EventInterface $event) + { + $this->Authentication->allowUnauthenticated(['cspReport']); // cspReport must work without authentication + + parent::beforeFilter($event); + $this->Security->setConfig('unlockedActions', ['cspReport']); + // permit reuse of CSRF tokens on some pages. + switch ($this->request->getParam('action')) { + case 'push': + case 'pull': + case 'getVersion': + case 'testConnection': + $this->Security->csrfUseOnce = false; + } + } + + public function index() + { + // Do not fetch server authkey from DB + $fields = array_flip($this->Servers->getSchema()->columns()); + unset($fields['authkey']); + $fields = array_keys($fields); + + $filters = $this->harvestParameters(['search']); + $conditions = []; + if (!empty($filters['search'])) { + $strSearch = '%' . trim(strtolower($filters['search'])) . '%'; + $conditions['OR'][]['LOWER(Server.name) LIKE'] = $strSearch; + $conditions['OR'][]['LOWER(Server.url) LIKE'] = $strSearch; + } + + if ($this->ParamHandler->isRest()) { + $params = [ + 'fields' => $fields, + 'recursive' => -1, + 'contain' => [ + 'Users' => [ + 'fields' => ['id', 'org_id', 'email', 'server_id'], + ], + 'Organisations' => [ + 'fields' => ['id', 'name', 'uuid', 'nationality', 'sector', 'type'], + ], + 'RemoteOrg' => [ + 'fields' => ['RemoteOrg.id', 'RemoteOrg.name', 'RemoteOrg.uuid', 'RemoteOrg.nationality', 'RemoteOrg.sector', 'RemoteOrg.type'], + ], + ], + 'conditions' => $conditions, + ]; + $servers = $this->Servers->find('all', $params); + $servers = $this->Servers->attachServerCacheTimestamps($servers->toArray()); + return $this->RestResponse->viewData($servers, $this->response->getType()); + } else { + $this->paginate['fields'] = $fields; + $this->paginate['conditions'] = $conditions; + $servers = $this->paginate(); + $servers = $this->Servers->attachServerCacheTimestamps($servers); + $this->set('servers', $servers); + $collection = []; + $collection['orgs'] = $this->Servers->Organisation->find( + 'list', + [ + 'fields' => ['id', 'name'], + ] + ); + $TagsTable = $this->fetchTable('Tags'); + $collection['tags'] = $TagsTable->find( + 'list', + [ + 'fields' => ['id', 'name'], + ] + ); + $this->set('collection', $collection); + } + } + + public function previewIndex($id) + { + $urlparams = ''; + $passedArgs = []; + + $server = $this->Servers->get($id); + if (empty($server)) { + throw new NotFoundException('Invalid server ID.'); + } + $validFilters = $this->Servers->validEventIndexFilters; + foreach ($validFilters as $k => $filter) { + if (isset($this->passedArgs[$filter])) { + $passedArgs[$filter] = $this->passedArgs[$filter]; + if ($k != 0) { + $urlparams .= '/'; + } + $urlparams .= $filter . ':' . $this->passedArgs[$filter]; + } + } + $combinedArgs = array_merge($this->passedArgs, $passedArgs); + if (!isset($combinedArgs['sort'])) { + $combinedArgs['sort'] = 'timestamp'; + $combinedArgs['direction'] = 'desc'; + } + if (empty($combinedArgs['page'])) { + $combinedArgs['page'] = 1; + } + if (empty($combinedArgs['limit'])) { + $combinedArgs['limit'] = 60; + } + try { + list($events, $total_count) = $this->Servers->previewIndex($server, $this->ACL->getUser(), $combinedArgs); + } catch (Exception $e) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->throwException(500, $e->getMessage()); + } else { + $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage()); + $this->redirect(['action' => 'index']); + } + } + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($events, $this->response->getType()); + } + + $EventsTable = $this->fetchTable('Events'); + $this->set('threatLevels', $EventsTable->ThreatLevel->listThreatLevels()); + $customPagination = new CustomPaginationTool(); + $params = $customPagination->createPaginationRules($events, $this->passedArgs, $this->alias); + if (!empty($total_count)) { + $params['pageCount'] = ceil($total_count / $params['limit']); + } + $this->params->params['paging'] = ['Servers' => $params]; + if (count($events) > 60) { + $customPagination->truncateByPagination($events, $params); + } + $this->set('events', $events); + $this->set('eventDescriptions', $EventsTable->fieldDescriptions); + $this->set('analysisLevels', $EventsTable->analysisLevels); + $this->set('distributionLevels', $EventsTable->distributionLevels); + + $shortDist = [0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group']; + $this->set('shortDist', $shortDist); + $this->set('id', $id); + $this->set('urlparams', $urlparams); + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('passedArgsArray', $passedArgs); + $this->set('server', $server); + } + + public function previewEvent($serverId, $eventId, $all = false) + { + $server = $this->Servers->get($serverId); + if (empty($server)) { + throw new NotFoundException('Invalid server ID.'); + } + try { + $event = $this->Servers->previewEvent($server, $eventId); + } catch (NotFoundException $e) { + throw new NotFoundException(__("Event '{0}' not found.", $eventId)); + } catch (Exception $e) { + $this->Flash->error(__('Download failed. {0}', $e->getMessage())); + $this->redirect(['action' => 'previewIndex', $serverId]); + } + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($event, $this->response->getType()); + } + + $WarninglistsTable = $this->fetchTable('Warninglists'); + if (isset($event['Event']['Attribute'])) { + $WarninglistsTable->attachWarninglistToAttributes($event['Event']['Attribute']); + } + if (isset($event['Event']['ShadowAttribute'])) { + $WarninglistsTable->attachWarninglistToAttributes($event['Event']['ShadowAttribute']); + } + + $EventsTable = $this->fetchTable('Events'); + $params = $EventsTable->rearrangeEventForView($event, $this->passedArgs, $all); + $this->__removeGalaxyClusterTags($event); + $this->params->params['paging'] = ['Server' => $params]; + $this->set('event', $event); + $this->set('server', $server); + $dataForView = [ + 'Attribute' => ['attrDescriptions' => 'fieldDescriptions', 'distributionDescriptions' => 'distributionDescriptions', 'distributionLevels' => 'distributionLevels'], + 'Event' => ['eventDescriptions' => 'fieldDescriptions', 'analysisLevels' => 'analysisLevels'], + 'Object' => [] + ]; + foreach ($dataForView as $m => $variables) { + if ($m === 'Event') { + $currentModel = $EventsTable; + } elseif ($m === 'Attribute') { + $currentModel = $EventsTable->Attribute; + } elseif ($m === 'Object') { + $currentModel = $EventsTable->Object; + } + foreach ($variables as $alias => $variable) { + $this->set($alias, $currentModel->{$variable}); + } + } + $this->set('threatLevels', $EventsTable->ThreatLevel->listThreatLevels()); + $this->set('title_for_layout', __('Remote event preview')); + } + + private function __removeGalaxyClusterTags(array &$event) + { + $galaxyTagIds = []; + foreach ($event['Galaxy'] as $galaxy) { + foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) { + $galaxyTagIds[$galaxyCluster['tag_id']] = true; + } + } + + if (empty($galaxyTagIds)) { + return; + } + + foreach ($event['Tag'] as $k => $eventTag) { + if (isset($galaxyTagIds[$eventTag['id']])) { + unset($event['Tag'][$k]); + } + } + } + + public function compareServers() + { + list($servers, $overlap) = $this->Servers->serverEventsOverlap(); + $this->set('servers', $servers); + $this->set('overlap', $overlap); + $this->set('title_for_layout', __('Server overlap analysis matrix')); + } + + public function filterEventIndex($id) + { + if (!$this->isSiteAdmin()) { + throw new MethodNotAllowedException('You are not authorised to do that.'); + } + $validFilters = $this->Servers->validEventIndexFilters; + $validatedFilterString = ''; + foreach ($this->passedArgs as $k => $v) { + if (in_array('' . $k, $validFilters)) { + if ($validatedFilterString != '') { + $validatedFilterString .= '/'; + } + $validatedFilterString .= $k . ':' . $v; + } + } + $this->set('id', $id); + $this->set('validFilters', $validFilters); + $this->set('filter', $validatedFilterString); + } + + public function add() + { + if ($this->request->is('post')) { + $data = $this->request->getData(); + if ($this->ParamHandler->isRest()) { + if (!isset($data['Server'])) { + $data = ['Server' => $data]; + } + } + if (!empty($data['Server']['json'])) { + $json = json_decode($data['Server']['json'], true); + } elseif ($this->ParamHandler->isRest()) { + if (empty($data['Server']['remote_org_id'])) { + throw new MethodNotAllowedException('No remote org ID set. Please pass it as remote_org_id'); + } + } + $fail = false; + if (empty(Configure::read('MISP.host_org_id'))) { + $data['Server']['internal'] = 0; + } + + if (!$fail) { + if ($this->ParamHandler->isRest()) { + $defaults = [ + 'push' => 0, + 'pull' => 0, + 'push_sightings' => 0, + 'push_galaxy_clusters' => 0, + 'pull_galaxy_clusters' => 0, + 'caching_enabled' => 0, + 'json' => '[]', + 'self_signed' => 0, + 'remove_missing_tags' => 0 + ]; + foreach ($defaults as $default => $dvalue) { + if (!isset($data['Server'][$default])) { + $data['Server'][$default] = $dvalue; + } + } + } + // force check userid and orgname to be from yourself + $data['Server']['org_id'] = $this->ACL->getUser()->org_id; + if ($this->ParamHandler->isRest()) { + if (empty($data['Server']['remote_org_id'])) { + return $this->RestResponse->saveFailResponse('Servers', 'add', false, ['Organisation' => 'Remote Organisation\'s id/uuid not given (remote_org_id)'], $this->response->getType()); + } + if (Validation::uuid($data['Server']['remote_org_id'])) { + $orgCondition = ['uuid' => $data['Server']['remote_org_id']]; + } else { + $orgCondition = ['id' => $data['Server']['remote_org_id']]; + } + $existingOrgs = $this->Servers->Organisations->find( + 'all', + [ + 'conditions' => $orgCondition, + 'recursive' => -1, + 'fields' => ['id', 'uuid'] + ] + )->first(); + if (empty($existingOrgs)) { + return $this->RestResponse->saveFailResponse('Servers', 'add', false, ['Organisation' => 'Invalid Remote Organisation'], $this->response->getType()); + } + } else { + if ($data['Server']['organisation_type'] < 2) { + $data['Server']['remote_org_id'] = $json['id']; + } else { + $existingOrgs = $this->Servers->Organisation->find( + 'all', + [ + 'conditions' => ['uuid' => $json['uuid']], + 'recursive' => -1, + 'fields' => ['id', 'uuid'] + ] + )->first(); + if (!empty($existingOrgs)) { + $fail = true; + $this->Flash->error(__('That organisation could not be created as the uuid is in use already.')); + } + if (!$fail) { + $this->Servers->Organisation->create(); + $orgSave = $this->Servers->Organisation->save( + [ + 'name' => $json['name'], + 'uuid' => $json['uuid'], + 'local' => 0, + 'created_by' => $this->ACL->getUser()->id + ] + ); + + if (!$orgSave) { + $this->Flash->error(__('Couldn\'t save the new organisation, are you sure that the uuid is in the correct format? Also, make sure the organisation\'s name doesn\'t clash with an existing one.')); + $fail = true; + $data['Server']['external_name'] = $json['name']; + $data['Server']['external_uuid'] = $json['uuid']; + } else { + $data['Server']['remote_org_id'] = $this->Servers->Organisation->id; + $data['Server']['organisation_type'] = 1; + } + } + } + } + if (!$fail) { + if (Configure::read('MISP.host_org_id') == 0 || $data['Server']['remote_org_id'] != Configure::read('MISP.host_org_id')) { + $data['Server']['internal'] = 0; + } + $data['Server']['org_id'] = $this->ACL->getUser()->org_id; + + $serverEntity = $this->Servers->newEntity($data['Server']); + + try { + $this->Servers->saveOrFail($serverEntity); + if (isset($data['Server']['submitted_cert'])) { + $this->__saveCert($data, $this->Servers->id, false); + } + if (isset($data['Server']['submitted_client_cert'])) { + $this->__saveCert($data, $this->Servers->id, true); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($serverEntity->toArray(), $this->response->getType()); + } else { + $this->Flash->success(__('The server has been saved')); + $this->redirect(['action' => 'index']); + } + } catch (Exception $e) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'add', false, $serverEntity->getErrors(), $this->response->getType()); + } else { + $this->Flash->error(__('The server could not be saved. Please, try again.')); + } + } + } + } + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->describe('Servers', 'add', false, $this->response->getType()); + } else { + $organisationOptions = [0 => 'Local organisation', 1 => 'External organisation', 2 => 'New external organisation']; + + $temp = $this->Servers->Organisation->find( + 'all', + [ + 'fields' => ['id', 'name', 'local'], + 'order' => ['lower(Organisation.name) ASC'] + ] + ); + $allOrgs = []; + $localOrganisations = []; + $externalOrganisations = []; + foreach ($temp as $o) { + if ($o['Organisation']['local']) { + $localOrganisations[$o['Organisation']['id']] = $o['Organisation']['name']; + } else { + $externalOrganisations[$o['Organisation']['id']] = $o['Organisation']['name']; + } + $allOrgs[] = ['id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']]; + } + + $allTypes = $this->Servers->getAllTypes(); + + $this->set('host_org_id', Configure::read('MISP.host_org_id')); + $this->set('organisationOptions', $organisationOptions); + $this->set('localOrganisations', $localOrganisations); + $this->set('externalOrganisations', $externalOrganisations); + $this->set('allOrganisations', $allOrgs); + $this->set('allAttributeTypes', $allTypes['attribute']); + $this->set('allObjectTypes', $allTypes['object']); + + $this->set('allTags', $this->__getTags()); + $this->set('host_org_id', Configure::read('MISP.host_org_id')); + $this->set('pull_scope', 'server'); + $this->render('edit'); + } + } + + public function edit($id = null) + { + $server = $this->Servers->get($id); + if (!$server) { + throw new NotFoundException(__('Invalid server')); + } + if ($this->request->is('post') || $this->request->is('put')) { + $data = $this->request->getData(); + if ($this->ParamHandler->isRest()) { + if (!isset($data['Server'])) { + $data = ['Server' => $data]; + } + } + if (empty(Configure::read('MISP.host_org_id'))) { + $server['internal'] = 0; + } + if (isset($data['Server']['json'])) { + $json = json_decode($data['Server']['json'], true); + } else { + $json = null; + } + $fail = false; + + if (!empty($data['Server']['push_rules'])) { + if (!empty($data['Server']['push_rules']['tags'])) { + $TagsTable = $this->fetchTable('Tags'); + foreach ($data['Server']['push_rules']['tags'] as $operator => $list) { + foreach ($list as $i => $tagName) { + if (!is_numeric($tagName)) { // tag added from freetext + $tag_id = $TagsTable->captureTag(['name' => $tagName], $this->ACL->getUser()); + $list[$i] = $tag_id; + } + } + } + } + } + + if (!$fail) { + // say what fields are to be updated + $fieldList = ['id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name', 'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy']; + $server['id'] = $id; + if (isset($data['Server']['authkey']) && "" != $data['Server']['authkey']) { + $fieldList[] = 'authkey'; + } + if (isset($data['Server']['organisation_type']) && isset($json)) { + // adds 'remote_org_id' in the fields to update + $fieldList[] = 'remote_org_id'; + if ($server['organisation_type'] < 2) { + $server['remote_org_id'] = $json['id']; + } else { + $existingOrgs = $this->Servers->Organisation->find( + 'all', + [ + 'conditions' => ['uuid' => $json['uuid']], + 'recursive' => -1, + 'fields' => ['id', 'uuid'] + ] + )->first(); + if (!empty($existingOrgs)) { + $fail = true; + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'edit', false, ['Organisation' => 'Remote Organisation\'s uuid already used'], $this->response->getType()); + } else { + $this->Flash->error(__('That organisation could not be created as the uuid is in use already.')); + } + } + + if (!$fail) { + $this->Servers->Organisation->create(); + $orgSave = $this->Servers->Organisation->save( + [ + 'name' => $json['name'], + 'uuid' => $json['uuid'], + 'local' => 0, + 'created_by' => $this->ACL->getUser()->id + ] + ); + + if (!$orgSave) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'edit', false, $this->Servers->Organisation->validationError, $this->response->getType()); + } else { + $this->Flash->error(__('Couldn\'t save the new organisation, are you sure that the uuid is in the correct format?.')); + } + $fail = true; + $server['external_name'] = $json['name']; + $server['external_uuid'] = $json['uuid']; + } else { + $server['remote_org_id'] = $this->Servers->Organisation->id; + } + } + } + if (empty(Configure::read('MISP.host_org_id')) || $data['Server']['remote_org_id'] != Configure::read('MISP.host_org_id')) { + $server['internal'] = 0; + } + } + } + if (!$fail) { + // Save the data + $this->Servers->patchEntity($server, $data['Server'], ['fieldList' => $fieldList]); + if ($this->Servers->save($server)) { + if (isset($data['Server']['submitted_cert']) && (!isset($data['Server']['delete_cert']) || !$data['Server']['delete_cert'])) { + $this->__saveCert($data, $server->id, false); + } else { + if (isset($data['Server']['delete_cert']) && $data['Server']['delete_cert']) { + $this->__saveCert($data, $server->id, false, true); + } + } + if (isset($data['Server']['submitted_client_cert']) && (!isset($data['Server']['delete_client_cert']) || !$data['Server']['delete_client_cert'])) { + $this->__saveCert($data, $server->id, true); + } else { + if (isset($data['Server']['delete_client_cert']) && $data['Server']['delete_client_cert']) { + $this->__saveCert($data, $server->id, true, true); + } + } + if ($this->ParamHandler->isRest()) { + $server = $this->Servers->get($server->id); + return $this->RestResponse->viewData($server, $this->response->getType()); + } else { + $this->Flash->success(__('The server has been saved')); + $this->redirect(['action' => 'index']); + } + } else { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'edit', false, $this->Servers->validationError, $this->response->getType()); + } else { + $this->Flash->error(__('The server could not be saved. Please, try again.')); + } + } + } + } else { + $this->Servers->read(null, $id); + $server = $this->Servers->get($id); + $server['authkey'] = ''; + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->describe('Servers', 'edit', false, $this->response->getType()); + } else { + $organisationOptions = [0 => 'Local organisation', 1 => 'External organisation', 2 => 'New external organisation']; + + $temp = $this->Servers->Organisations->find( + 'all', + [ + 'fields' => ['id', 'name', 'local'], + 'order' => ['lower(Organisation.name) ASC'] + ] + ); + $allOrgs = []; + $localOrganisations = []; + $externalOrganisations = []; + foreach ($temp as $o) { + if ($o['Organisation']['local']) { + $localOrganisations[$o['Organisation']['id']] = $o['Organisation']['name']; + } else { + $externalOrganisations[$o['Organisation']['id']] = $o['Organisation']['name']; + } + $allOrgs[] = ['id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']]; + } + + $allTypes = $this->Servers->getAllTypes(); + + $oldRemoteSetting = 0; + if (!$server['RemoteOrg']['local']) { + $oldRemoteSetting = 1; + } + $this->set('host_org_id', Configure::read('MISP.host_org_id')); + $this->set('oldRemoteSetting', $oldRemoteSetting); + $this->set('oldRemoteOrg', $this->Servers->data['RemoteOrg']['id']); + + $this->set('organisationOptions', $organisationOptions); + $this->set('localOrganisations', $localOrganisations); + $this->set('externalOrganisations', $externalOrganisations); + $this->set('allOrganisations', $allOrgs); + + $this->set('allTags', $this->__getTags()); + $this->set('allAttributeTypes', $allTypes['attribute']); + $this->set('allObjectTypes', $allTypes['object']); + $this->set('server', $server); + $this->set('id', $id); + $this->set('host_org_id', Configure::read('MISP.host_org_id')); + $this->set('pull_scope', 'server'); + } + } + + public function delete($id = null) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This endpoint expects POST requests.')); + } + $server = $this->Servers->get($id); + if (!$server) { + throw new NotFoundException(__('Invalid server')); + } + if ($this->Servers->delete($server)) { + $message = __('Server deleted'); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'delete', $message, $this->response->getType()); + } else { + $this->Flash->success($message); + $this->redirect(['controller' => 'servers', 'action' => 'index']); + } + } + $message = __('Server was not deleted'); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'delete', $id, $message, $this->response->getType()); + } else { + $this->Flash->error($message); + $this->redirect(['action' => 'index']); + } + } + + public function eventBlockRule() + { + $AdminSettingsTable = $this->fetchTable('AdminSettings'); + + $setting = $AdminSettingsTable->find( + 'all', + [ + 'conditions' => ['setting' => 'eventBlockRule'], + 'recursive' => -1 + ] + )->first(); + if (empty($setting)) { + $setting = ['setting' => 'eventBlockRule']; + if ($this->request->is('post')) { + $AdminSettingsTable->create(); + } + } + if ($this->request->is('post')) { + $data = $this->request->getData(); + if (!empty($data['Server'])) { + $data = $data['Server']; + } + $setting['AdminSetting']['setting'] = 'eventBlockRule'; + $setting['AdminSetting']['value'] = $data['value']; + $settingEntity = $AdminSettingsTable->newEntity($setting); + $result = $AdminSettingsTable->save($settingEntity); + if ($result) { + $message = __('Settings saved'); + } else { + $message = __('Could not save the settings. Invalid input.'); + } + if ($this->ParamHandler->isRest()) { + if ($result) { + return $this->RestResponse->saveFailResponse('Servers', 'eventBlockRule', false, $message, $this->response->getType()); + } else { + return $this->RestResponse->saveSuccessResponse('Servers', 'eventBlockRule', $message, $this->response->getType()); + } + } else { + if ($result) { + $this->Flash->success($message); + $this->redirect('/'); + } else { + $this->Flash->error($message); + } + } + } + $this->set('setting', $setting); + } + + /** + * Pull one or more events with attributes from a remote instance. + * Set $technique to + * full - download everything + * incremental - only new events + * - specific id of the event to pull + */ + public function pull($id = null, $technique = 'full') + { + if (empty($id)) { + if (!empty($this->request->getData()['id'])) { + $id = $this->request->getData()['id']; + } else { + throw new NotFoundException(__('Invalid server')); + } + } + + $s = $this->Servers->get($id); + if (empty($s)) { + throw new NotFoundException(__('Invalid server')); + } + $error = false; + + if (false == $s['pull'] && ($technique === 'full' || $technique === 'incremental')) { + $error = __('Pull setting not enabled for this server.'); + } + if (false == $s['pull_galaxy_clusters'] && ($technique === 'pull_relevant_clusters')) { + $error = __('Pull setting not enabled for this server.'); + } + if (empty($error)) { + if (!Configure::read('BackgroundJobs.enabled')) { + $result = $this->Servers->pull($this->ACL->getUser()->toArray(), $technique, $s->toArray()); + if (is_array($result)) { + $success = __('Pull completed. {0} events pulled, {1} events could not be pulled, {2} proposals pulled, {3} sightings pulled, {4} clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); + } else { + $error = $result; + } + $this->set('successes', $result[0]); + $this->set('fails', $result[1]); + $this->set('pulledProposals', $result[2]); + $this->set('pulledSightings', $result[3]); + } else { + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->ACL->getUser(), + Job::WORKER_DEFAULT, + 'pull', + 'Server: ' . $id, + __('Pulling.') + ); + + $this->Servers->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'pull', + $this->ACL->getUser()->id, + $id, + $technique, + $jobId + ], + false, + $jobId + ); + + $success = __("Pull queued for background execution. Job ID: {0}", $jobId); + } + } + if ($this->ParamHandler->isRest()) { + if (!empty($error)) { + return $this->RestResponse->saveFailResponse('Servers', 'pull', $id, $error, $this->response->getType()); + } else { + return $this->RestResponse->saveSuccessResponse('Servers', 'pull', $id, $this->response->getType(), $success); + } + } else { + if (!empty($error)) { + $this->Flash->error($error); + $this->redirect(['action' => 'index']); + } else { + $this->Flash->success($success); + $this->redirect($this->referer()); + } + } + } + + public function push($id = null, $technique = 'full') + { + $id = $id ?? $this->request->getData('id'); + + $server = $this->Servers->get($id)->toArray(); + + $technique = $technique ?? $this->request->getData('technique'); + + if (!Configure::read('BackgroundJobs.enabled')) { + $HttpSocket = new HttpTool(); + $HttpSocket->configFromServer($server); + $result = $this->Servers->push($id, $technique, false, $HttpSocket, $this->ACL->getUser()->toArray()); + if ($result === false) { + $error = __('The remote server is too outdated to initiate a push towards it. Please notify the hosting organisation of the remote instance.'); + } elseif (!is_array($result)) { + $error = $result; + } + if (!empty($error)) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'push', false, $error, $this->response->getType()); + } else { + $this->Flash->info($error); + $this->redirect(['action' => 'index']); + } + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'push', $id, $this->response->getType(), __('Push complete. {0} events pushed, {1} events could not be pushed.', count($result[0]), count($result[1]))); + } else { + $this->set('successes', $result[0]); + $this->set('fails', $result[1]); + } + } else { + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->ACL->getUser(), + Job::WORKER_DEFAULT, + 'push', + 'Server: ' . $id, + __('Pushing.') + ); + + $this->Servers->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'push', + $this->ACL->getUser()->id, + $id, + $technique, + $jobId + ], + false, + $jobId + ); + + $message = __('Push queued for background execution. Job ID: {0}', $jobId); + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'push', $message, $this->response->getType()); + } + $this->Flash->success($message); + $this->redirect(['action' => 'index']); + } + } + + private function __saveCert($server, $id, $client = false, $delete = false) + { + if ($client) { + $subm = 'submitted_client_cert'; + $attr = 'client_cert_file'; + $ins = '_client'; + } else { + $subm = 'submitted_cert'; + $attr = 'cert_file'; + $ins = ''; + } + if (!$delete) { + $ext = ''; + if (isset($server[$subm]['name'])) { + if ($this->request->getData()[$subm]['size'] != 0) { + if (!$this->Servers->checkFilename($server[$subm]['name'])) { + throw new Exception(__('Filename not allowed')); + } + + if (!is_uploaded_file($server[$subm]['tmp_name'])) { + throw new Exception(__('File not uploaded correctly')); + } + + $ext = pathinfo($server[$subm]['name'], PATHINFO_EXTENSION); + if (!in_array($ext, HttpTool::ALLOWED_CERT_FILE_EXTENSIONS)) { + $this->Flash->error(__('Invalid extension.')); + $this->redirect(['action' => 'index']); + } + + if (!$server[$subm]['size'] > 0) { + $this->Flash->error(__('Incorrect extension or empty file.')); + $this->redirect(['action' => 'index']); + } + + // read certificate file data + $certData = FileAccessTool::readFromFile($server[$subm]['tmp_name'], $server[$subm]['size']); + } else { + return true; + } + } else { + $ext = 'pem'; + $certData = base64_decode($server[$subm]); + } + + // check if the file is a valid x509 certificate + try { + $cert = openssl_x509_parse($certData); + if (!$cert) { + throw new Exception(__('Invalid certificate.')); + } + } catch (Exception $e) { + $this->Flash->error(__('Invalid certificate.')); + $this->redirect(['action' => 'index']); + } + + $destpath = APP . "files" . DS . "certs" . DS; + + FileAccessTool::writeToFile($destpath . $id . $ins . '.' . $ext, $certData); + $s = $this->Servers->get($id); + $s[$attr] = $s['id'] . $ins . '.' . $ext; + $this->Servers->save($s); + } else { + $s = $this->Servers->get($id); + $s[$attr] = ''; + $this->Servers->save($s); + } + return true; + } + + public function serverSettingsReloadSetting($setting, $id) + { + $pathToSetting = explode('.', $setting); + if ( + strpos($setting, 'Plugin.Enrichment') !== false || + strpos($setting, 'Plugin.Import') !== false || + strpos($setting, 'Plugin.Export') !== false || + strpos($setting, 'Plugin.Cortex') !== false || + strpos($setting, 'Plugin.Action') !== false || + strpos($setting, 'Plugin.Workflow') !== false + ) { + $settingObject = $this->Servers->getCurrentServerSettings(); + } else { + $settingObject = $this->Servers->serverSettings; + } + foreach ($pathToSetting as $key) { + if (!isset($settingObject[$key])) { + throw new MethodNotAllowedException(); + } + $settingObject = $settingObject[$key]; + } + $result = $this->Servers->serverSettingReadSingle($settingObject, $setting, $key); + $this->set('setting', $result); + $priorityErrorColours = [0 => 'red', 1 => 'yellow', 2 => 'green']; + $this->set('priorityErrorColours', $priorityErrorColours); + $priorities = [0 => 'Critical', 1 => 'Recommended', 2 => 'Optional', 3 => 'Deprecated']; + $this->set('priorities', $priorities); + $this->set('k', $id); + $this->layout = false; + + $subGroup = 'general'; + if ($pathToSetting[0] === 'Plugin') { + $subGroup = explode('_', $pathToSetting[1])[0]; + } + $this->set('subGroup', $subGroup); + + $this->render('/Elements/healthElements/settings_row'); + } + + public function serverSettings($tab = false) + { + if (!$this->request->is('get')) { + throw new MethodNotAllowedException('Just GET method is allowed.'); + } + $tabs = [ + 'MISP' => ['count' => 0, 'errors' => 0, 'severity' => 5], + 'Encryption' => ['count' => 0, 'errors' => 0, 'severity' => 5], + 'Proxy' => ['count' => 0, 'errors' => 0, 'severity' => 5], + 'Security' => ['count' => 0, 'errors' => 0, 'severity' => 5], + 'Plugin' => ['count' => 0, 'errors' => 0, 'severity' => 5], + 'SimpleBackgroundJobs' => ['count' => 0, 'errors' => 0, 'severity' => 5] + ]; + + $writeableErrors = [0 => __('OK'), 1 => __('not found'), 2 => __('is not writeable')]; + $readableErrors = [0 => __('OK'), 1 => __('not readable')]; + $gpgErrors = [0 => __('OK'), 1 => __('FAIL: settings not set'), 2 => __('FAIL: Failed to load GnuPG'), 3 => __('FAIL: Issues with the key/passphrase'), 4 => __('FAIL: sign failed')]; + $proxyErrors = [0 => __('OK'), 1 => __('not configured (so not tested)'), 2 => __('Getting URL via proxy failed')]; + $zmqErrors = [0 => __('OK'), 1 => __('not enabled (so not tested)'), 2 => __('Python ZeroMQ library not installed correctly.'), 3 => __('ZeroMQ script not running.')]; + $sessionErrors = [ + 0 => __('OK'), + 1 => __('Too many expired sessions in the database, please clear the expired sessions'), + 2 => __('PHP session handler is using the default file storage. This is not recommended, please use the redis or database storage'), + 8 => __('Alternative setting used'), + 9 => __('Test failed') + ]; + $moduleErrors = [0 => __('OK'), 1 => __('System not enabled'), 2 => __('No modules found')]; + $backgroundJobsErrors = [ + 0 => __('OK'), + 1 => __('Not configured (so not tested)'), + 2 => __('Error connecting to Redis.'), + 3 => __('Error connecting to Supervisor.'), + 4 => __('Error connecting to Redis and Supervisor.') + ]; + + $finalSettings = $this->Servers->serverSettingsRead(); + $issues = [ + 'errors' => [ + 0 => [ + 'value' => 0, + 'description' => __('MISP will not operate correctly or will be unsecure until these issues are resolved.') + ], + 1 => [ + 'value' => 0, + 'description' => __('Some of the features of MISP cannot be utilised until these issues are resolved.') + ], + 2 => [ + 'value' => 0, + 'description' => __('There are some optional tweaks that could be done to improve the looks of your MISP instance.') + ], + ], + 'deprecated' => [], + 'overallHealth' => 3, + ]; + $dumpResults = []; + $tempArray = []; + foreach ($finalSettings as $k => $result) { + if ($result['level'] == 3) { + $issues['deprecated']++; + } + $tabs[$result['tab']]['count']++; + if (isset($result['error']) && $result['level'] < 3) { + $issues['errors'][$result['level']]['value']++; + if ($result['level'] < $issues['overallHealth']) { + $issues['overallHealth'] = $result['level']; + } + $tabs[$result['tab']]['errors']++; + if ($result['level'] < $tabs[$result['tab']]['severity']) { + $tabs[$result['tab']]['severity'] = $result['level']; + } + } + if (isset($result['optionsSource']) && is_callable($result['optionsSource'])) { + $result['options'] = $result['optionsSource'](); + } + $dumpResults[] = $result; + if ($result['tab'] == $tab) { + if (isset($result['subGroup'])) { + $tempArray[$result['subGroup']][] = $result; + } else { + $tempArray['general'][] = $result; + } + } + } + $finalSettings = $tempArray; + // Diagnostics portion + $diagnostic_errors = 0; + if ($tab === 'correlations') { + $CorrelationsTable = $this->fetchTable('Correlations'); + $correlation_metrics = $CorrelationsTable->collectMetrics(); + $this->set('correlation_metrics', $correlation_metrics); + } + if ($tab === 'files') { + if (!empty(Configure::read('Security.disable_instance_file_uploads'))) { + throw new MethodNotAllowedException(__('This functionality is disabled.')); + } + $files = $this->Servers->grabFiles(); + $this->set('files', $files); + } + // Only run this check on the diagnostics tab + if ($tab === 'diagnostics' || $tab === 'download' || $this->ParamHandler->isRest()) { + $php_ini = php_ini_loaded_file(); + $this->set('php_ini', $php_ini); + + $attachmentTool = new AttachmentTool(); + try { + $advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus(); + } catch (Exception $e) { + $this->log($e->getMessage(), LOG_NOTICE); + $advanced_attachments = false; + } + + $this->set('advanced_attachments', $advanced_attachments); + + $gitStatus = $this->Servers->getCurrentGitStatus(true); + $this->set('branch', $gitStatus['branch']); + $this->set('commit', $gitStatus['commit']); + $this->set('latestCommit', $gitStatus['latestCommit']); + $this->set('version', $gitStatus['version']); + + $phpSettings = [ + 'max_execution_time' => [ + 'explanation' => 'The maximum duration that a script can run (does not affect the background workers). A too low number will break long running scripts like comprehensive API exports', + 'recommended' => 300, + 'unit' => 'seconds', + ], + 'memory_limit' => [ + 'explanation' => 'The maximum memory that PHP can consume. It is recommended to raise this number since certain exports can generate a fair bit of memory usage', + 'recommended' => 2048, + 'unit' => 'MB' + ], + 'upload_max_filesize' => [ + 'explanation' => 'The maximum size that an uploaded file can be. It is recommended to raise this number to allow for the upload of larger samples', + 'recommended' => 50, + 'unit' => 'MB' + ], + 'post_max_size' => [ + 'explanation' => 'The maximum size of a POSTed message, this has to be at least the same size as the upload_max_filesize setting', + 'recommended' => 50, + 'unit' => 'MB' + ] + ]; + + foreach ($phpSettings as $setting => $settingArray) { + $phpSettings[$setting]['value'] = $this->Servers->getIniSetting($setting); + if ($phpSettings[$setting]['value'] && $settingArray['unit'] && $settingArray['unit'] === 'MB') { + // convert basic unit to M + $phpSettings[$setting]['value'] = (int) floor($phpSettings[$setting]['value'] / 1024 / 1024); + } + } + $this->set('phpSettings', $phpSettings); + + if ($gitStatus['version'] && $gitStatus['version']['upToDate'] === 'older') { + $diagnostic_errors++; + } + + // check if the STIX and Cybox libraries are working and the correct version using the test script stixtest.py + $stix = $this->Servers->stixDiagnostics($diagnostic_errors); + + $yaraStatus = $this->Servers->yaraDiagnostics($diagnostic_errors); + + // if GnuPG is set up in the settings, try to encrypt a test message + $gpgStatus = $this->Servers->gpgDiagnostics($diagnostic_errors); + + // if the message queue pub/sub is enabled, check whether the extension works + $zmqStatus = $this->Servers->zmqDiagnostics($diagnostic_errors); + + // if Proxy is set up in the settings, try to connect to a test URL + $proxyStatus = $this->Servers->proxyDiagnostics($diagnostic_errors); + + // if SimpleBackgroundJobs is set up in the settings, try to connect to Redis + $backgroundJobsStatus = $this->Servers->backgroundJobsDiagnostics($diagnostic_errors); + + // get the DB diagnostics + $dbDiagnostics = $this->Servers->dbSpaceUsage(); + $dbSchemaDiagnostics = $this->Servers->dbSchemaDiagnostic(); + $dbConfiguration = $this->Servers->dbConfiguration(); + + $redisInfo = $this->Servers->redisInfo(); + + $moduleTypes = ['Enrichment', 'Import', 'Export', 'Cortex']; + foreach ($moduleTypes as $type) { + $moduleStatus[$type] = $this->Servers->moduleDiagnostics($diagnostic_errors, $type); + } + + // get php session diagnostics + $sessionStatus = $this->Servers->sessionDiagnostics($diagnostic_errors); + + $AttachmentScansTable = $this->fetchTable('AttachmentScans'); + try { + $attachmentScan = ['status' => true, 'software' => $AttachmentScansTable->diagnostic()]; + } catch (Exception $e) { + $attachmentScan = ['status' => false, 'error' => $e->getMessage()]; + } + + $securityAudit = (new SecurityAudit())->run($this->Server); + + $view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'dbConfiguration', 'redisInfo', 'attachmentScan', 'securityAudit'); + } else { + $view = []; + } + + // check whether the files are writeable + $writeableDirs = $this->Servers->writeableDirsDiagnostics($diagnostic_errors); + $writeableFiles = $this->Servers->writeableFilesDiagnostics($diagnostic_errors); + $readableFiles = $this->Servers->readableFilesDiagnostics($diagnostic_errors); + $extensions = $this->Servers->extensionDiagnostics(); + + // check if the encoding is not set to utf8 + $dbEncodingStatus = $this->Servers->databaseEncodingDiagnostics($diagnostic_errors); + + $view = array_merge($view, compact('diagnostic_errors', 'tabs', 'tab', 'issues', 'finalSettings', 'writeableErrors', 'readableErrors', 'writeableDirs', 'writeableFiles', 'readableFiles', 'extensions', 'dbEncodingStatus')); + $this->set($view); + + $workerIssueCount = 4; + $worker_array = []; + if (Configure::read('BackgroundJobs.enabled')) { + $workerIssueCount = 0; + $worker_array = $this->Servers->workerDiagnostics($workerIssueCount); + } + $this->set('worker_array', $worker_array); + if ($tab === 'download' || $this->ParamHandler->isRest()) { + foreach ($dumpResults as $key => $dr) { + unset($dumpResults[$key]['description']); + } + $dump = [ + 'version' => $gitStatus['version'], + 'phpSettings' => $phpSettings, + 'gpgStatus' => $gpgErrors[$gpgStatus['status']], + 'proxyStatus' => $proxyErrors[$proxyStatus], + 'zmqStatus' => $zmqStatus, + 'stix' => $stix, + 'moduleStatus' => $moduleStatus, + 'writeableDirs' => $writeableDirs, + 'writeableFiles' => $writeableFiles, + 'readableFiles' => $readableFiles, + 'dbDiagnostics' => $dbDiagnostics, + 'dbSchemaDiagnostics' => $dbSchemaDiagnostics, + 'dbConfiguration' => $dbConfiguration, + 'redisInfo' => $redisInfo, + 'finalSettings' => $dumpResults, + 'extensions' => $extensions, + 'workers' => $worker_array, + 'backgroundJobsStatus' => $backgroundJobsErrors[$backgroundJobsStatus] + ]; + foreach ($dump['finalSettings'] as $k => $v) { + if (!empty($v['redacted'])) { + $dump['finalSettings'][$k]['value'] = '*****'; + } + } + $this->response->withStringBody(json_encode($dump, JSON_PRETTY_PRINT)); + $this->response->withType('json'); + $this->response->withDownload('MISP.report.json'); + return $this->response; + } + + $priorities = [0 => 'Critical', 1 => 'Recommended', 2 => 'Optional', 3 => 'Deprecated']; + $this->set('priorities', $priorities); + $this->set('workerIssueCount', $workerIssueCount); + $priorityErrorColours = [0 => 'red', 1 => 'yellow', 2 => 'green']; + $this->set('priorityErrorColours', $priorityErrorColours); + $this->set('phpversion', PHP_VERSION); + $this->set('phpmin', $this->phpmin); + $this->set('phprec', $this->phprec); + $this->set('phptoonew', $this->phptoonew); + $this->set('title_for_layout', __('Diagnostics')); + } + + public function startWorker($type) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + + if (Configure::read('BackgroundJobs.enabled')) { + $message = __('Worker start signal sent'); + $this->Servers->getBackgroundJobsTool()->startWorkerByQueue($type); + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'startWorker', $type, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + // CakeResque + $validTypes = ['default', 'email', 'scheduler', 'cache', 'prio', 'update']; + if (!in_array($type, $validTypes)) { + throw new MethodNotAllowedException('Invalid worker type.'); + } + + $prepend = ''; + if ($type != 'scheduler') { + $workerIssueCount = 0; + $workerDiagnostic = $this->Servers->workerDiagnostics($workerIssueCount); + if ($type == 'update' && isset($workerDiagnostic['update']['ok']) && $workerDiagnostic['update']['ok']) { + $message = __('Only one `update` worker can run at a time'); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'startWorker', false, $message, $this->response->getType()); + } else { + $this->Flash->error($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + shell_exec($prepend . APP . 'Console' . DS . 'cake CakeResque.CakeResque start --interval 5 --queue ' . $type . ' > /dev/null 2>&1 &'); + } else { + shell_exec($prepend . APP . 'Console' . DS . 'cake CakeResque.CakeResque startscheduler -i 5 > /dev/null 2>&1 &'); + } + $message = __('Worker start signal sent'); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'startWorker', $type, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + public function stopWorker($pid) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + + $message = __('Worker stop signal sent'); + + if (Configure::read('BackgroundJobs.enabled')) { + $this->Servers->getBackgroundJobsTool()->stopWorker($pid); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + // CakeResque + $this->Servers->killWorker($pid, $this->ACL->getUser()); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + public function getWorkers() + { + if (Configure::read('BackgroundJobs.enabled')) { + $workerIssueCount = 0; + $worker_array = $this->Servers->workerDiagnostics($workerIssueCount); + } else { + $worker_array = [__('Background jobs not enabled')]; + } + return $this->RestResponse->viewData($worker_array); + } + + public function idTranslator($localId = null) + { + // We retrieve the list of remote servers that we can query + $servers = $this->Servers->find( + 'all', + [ + 'conditions' => ['OR' => ['pull' => true, 'push' => true]], + 'recursive' => -1, + 'order' => ['Server.priority ASC'], + ] + ); + + // We generate the list of servers for the dropdown + $displayServers = []; + foreach ($servers as $s) { + $displayServers[] = [ + 'name' => $s['Server']['name'], + 'value' => $s['Server']['id'], + ]; + } + $this->set('servers', $displayServers); + + if ($localId || $this->request->is('post')) { + $data = $this->request->getData(); + if ($localId && $this->request->is('get')) { + $data['Event']['local'] = 'local'; + $data['Event']['uuid'] = $localId; + } + $EventsTable = $this->fetchTable('Events'); + $remote_events = []; + if (!empty($data['Event']['uuid']) && $data['Event']['local'] === "local") { + $local_event = $EventsTable->fetchSimpleEvent($this->ACL->getUser(), $data['Event']['uuid']); + } else if (!empty($data['Event']['uuid']) && $data['Event']['local'] === "remote" && !empty($data['Server']['id'])) { + //We check on the remote server for any event with this id and try to find a match locally + $conditions = ['AND' => ['Server.id' => $data['Server']['id'], 'Server.pull' => true]]; + $remote_server = $this->Servers->find('all', ['conditions' => $conditions])->first(); + if (!empty($remote_server)) { + try { + $remote_event = $EventsTable->downloadEventMetadataFromServer($data['Event']['uuid'], $remote_server); + } catch (Exception $e) { + $this->Flash->error(__("Issue while contacting the remote server to retrieve event information")); + return; + } + + if (empty($remote_event)) { + $this->Flash->error(__("This event could not be found or you don't have permissions to see it.")); + return; + } + + $local_event = $EventsTable->fetchSimpleEvent($this->ACL->getUser(), $remote_event['uuid']); + // we record it to avoid re-querying the same server in the 2nd phase + if (!empty($local_event)) { + $remote_events[] = [ + "server_id" => $remote_server['Server']['id'], + "server_name" => $remote_server['Server']['name'], + "url" => $remote_server['Server']['url'] . "/events/view/" . $remote_event['id'], + "remote_id" => $remote_event['id'] + ]; + } + } + } + if (empty($local_event)) { + $this->Flash->error(__("This event could not be found or you don't have permissions to see it.")); + return; + } else { + $this->Flash->success(__('The event has been found.')); + } + + // In the second phase, we query all configured sync servers to get their info on the event + foreach ($servers as $server) { + // We check if the server was not already contacted in phase 1 + if (count($remote_events) > 0 && $remote_events[0]['server_id'] == $server['Server']['id']) { + continue; + } + + $exception = null; + try { + $remoteEvent = $EventsTable->downloadEventMetadataFromServer($local_event['Event']['uuid'], $server); + } catch (Exception $e) { + $remoteEvent = null; + $exception = $e->getMessage(); + } + $remoteEventId = isset($remoteEvent['id']) ? $remoteEvent['id'] : null; + $remote_events[] = [ + "server_id" => $server['Server']['id'], + "server_name" => $server['Server']['name'], + "url" => isset($remoteEventId) ? $server['Server']['url'] . "/events/view/" . $remoteEventId : $server['Server']['url'], + "remote_id" => isset($remoteEventId) ? $remoteEventId : false, + "exception" => $exception, + ]; + } + + $this->set('local_event', $local_event); + $this->set('remote_events', $remote_events); + } + $this->set('title_for_layout', __('Event ID translator')); + } + + public function getSubmodulesStatus() + { + $this->set('submodules', $this->Servers->getSubmodulesGitStatus()); + $this->render('ajax/submoduleStatus'); + } + + public function getSetting($settingName) + { + $setting = $this->Servers->getSettingData($settingName); + if (!$setting) { + throw new NotFoundException(__('Setting {0} is invalid.', $settingName)); + } + if (!empty($setting["redacted"])) { + throw new ForbiddenException(__('This setting is redacted.')); + } + if (Configure::check($settingName)) { + $setting['value'] = Configure::read($settingName); + } + return $this->RestResponse->viewData($setting); + } + + public function serverSettingsEdit($settingName, $id = false, $forceSave = false) + { + if (!$this->ParamHandler->isRest()) { + if (!isset($id)) { + throw new MethodNotAllowedException(); + } + $this->set('id', $id); + } + $setting = $this->Servers->getSettingData($settingName); + if ($setting === false) { + throw new NotFoundException(__('Setting {0} is invalid.', $settingName)); + } + if (!empty($setting['cli_only'])) { + throw new MethodNotAllowedException(__('This setting can only be edited via the CLI.')); + } + if ($this->request->is('get')) { + $value = Configure::read($setting['name']); + if (isset($value)) { + $setting['value'] = $value; + } + $setting['setting'] = $setting['name']; + if (isset($setting['optionsSource']) && is_callable($setting['optionsSource'])) { + $setting['options'] = $setting['optionsSource'](); + } + $subGroup = explode('.', $setting['name']); + if ($subGroup[0] === 'Plugin') { + $subGroup = explode('_', $subGroup[1])[0]; + } else { + $subGroup = 'general'; + } + if ($this->ParamHandler->isRest()) { + if (!empty($setting['redacted'])) { + throw new ForbiddenException(__('This setting is redacted.')); + } + return $this->RestResponse->viewData([$setting['name'] => $setting['value']]); + } else { + $this->set('subGroup', $subGroup); + $this->set('setting', $setting); + $this->render('ajax/server_settings_edit'); + } + } else if ($this->request->is('post')) { + $data = $this->request->getData(); + if (!isset($data['Server'])) { + $data = ['Server' => $data]; + } + if (!isset($data['Server']['value']) || !is_scalar($data['Server']['value'])) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'Invalid input. Expected: {"value": "new_setting"}', $this->response->getType()); + } + } + if (!empty($data['Server']['force'])) { + $forceSave = $data['Server']['force']; + } + if (trim($data['Server']['value']) === '*****') { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'No change.', $this->response->getType()); + } else { + return new Response(['body' => json_encode(['saved' => false, 'errors' => 'No change.']), 'status' => 200, 'type' => 'json']); + } + } + $this->autoRender = false; + if (!Configure::read('MISP.system_setting_db') && !is_writeable(APP . 'Config/config.php')) { + $LogsTable = $this->fetchTable('Logs'); + $LogsTable->saveOrFailSilently( + [ + 'org' => $this->ACL->getUser()['Organisation']['name'], + 'model' => 'Server', + 'model_id' => 0, + 'email' => $this->ACL->getUser()->email, + 'action' => 'serverSettingsEdit', + 'user_id' => $this->ACL->getUser()->id, + 'title' => 'Server setting issue', + 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $data['Server']['value'] . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.', + ] + ); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'app/Config.config.php is not writeable to the apache user.', $this->response->getType()); + } else { + return new Response(['body' => json_encode(['saved' => false, 'errors' => 'app/Config.config.php is not writeable to the apache user.']), 'status' => 200, 'type' => 'json']); + } + } + $result = $this->Servers->serverSettingsEditValue($this->ACL->getUser(), $setting, $data['Server']['value'], $forceSave); + if ($result === true) { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'serverSettingsEdit', false, $this->response->getType(), 'Field updated'); + } else { + return new Response(['body' => json_encode(['saved' => true, 'success' => 'Field updated.']), 'status' => 200, 'type' => 'json']); + } + } else { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, $result, $this->response->getType()); + } else { + return new Response(['body' => json_encode(['saved' => false, 'errors' => $result]), 'status' => 200, 'type' => 'json']); + } + } + } + } + + public function killAllWorkers($force = false) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + $this->Servers->killAllWorkers($this->ACL->getUser(), $force); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Server', 'killAllWorkers', false, $this->response->getType(), __('Killing workers.')); + } + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'workers']); + } + + public function restartWorkers() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Servers->getBackgroundJobsTool()->restartWorkers(); + } else { + // CakeResque + $this->Servers->restartWorkers($this->ACL->getUser()); + } + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Server', 'restartWorkers', false, $this->response->getType(), __('Restarting workers.')); + } + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'workers']); + } + + public function restartDeadWorkers() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Servers->getBackgroundJobsTool()->restartDeadWorkers(); + } else { + // CakeResque + $this->Servers->restartDeadWorkers($this->ACL->getUser()); + } + + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Server', 'restartDeadWorkers', false, $this->response->getType(), __('Restarting workers.')); + } + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'workers']); + } + + public function deleteFile($type, $filename) + { + if ($this->request->is('post')) { + $validItems = $this->Servers->getFileRules(); + $existingFile = new SplFileInfo($validItems[$type]['path'] . DS . $filename); + if (!$existingFile->isFile()) { + $this->Flash->error(__('File not found.', true), 'default', [], 'error'); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } + if (FileAccessTool::deleteFile($existingFile->getPathname())) { + $this->Flash->success('File deleted.'); + } else { + $this->Flash->error(__('File could not be deleted.', true), 'default', [], 'error'); + } + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } else { + throw new MethodNotAllowedException('This action expects a POST request.'); + } + } + + public function uploadFile($type) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + if (!empty(Configure::read('Security.disable_instance_file_uploads'))) { + throw new MethodNotAllowedException(__('Feature disabled.')); + } + $validItems = $this->Servers->getFileRules(); + + // Check if there were problems with the file upload + // only keep the last part of the filename, this should prevent directory attacks + $data = $this->request->getData(); + $filename = basename($data['Server']['file']['name']); + if (!preg_match("/" . $validItems[$type]['regex'] . "/", $filename)) { + $this->Flash->error($validItems[$type]['regex_error'], 'default', [], 'error'); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } + if (empty($data['Server']['file']['tmp_name']) || !is_uploaded_file($data['Server']['file']['tmp_name'])) { + $this->Flash->error(__('Upload failed.', true), 'default', [], 'error'); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } + + // check if the file already exists + $existingFile = new SplFileInfo($validItems[$type]['path'] . DS . $filename); + if ($existingFile->isFile()) { + $this->Flash->info(__('File already exists. If you would like to replace it, remove the old one first.', true), 'default', [], 'error'); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } + + $result = move_uploaded_file($data['Server']['file']['tmp_name'], $validItems[$type]['path'] . DS . $filename); + if ($result) { + $this->Flash->success('File uploaded.'); + } else { + $this->Flash->error(__('Upload failed.', true), 'default', [], 'error'); + } + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'files']); + } + + public function fetchServersForSG($idList = '{}') + { + $id_exclusion_list = json_decode($idList, true); + $temp = $this->Servers->find( + 'all', + [ + 'conditions' => [ + 'id !=' => $id_exclusion_list, + ], + 'recursive' => -1, + 'fields' => ['id', 'name', 'url'] + ] + ); + $servers = []; + foreach ($temp as $server) { + $servers[] = ['id' => $server['Server']['id'], 'name' => $server['Server']['name'], 'url' => $server['Server']['url']]; + } + $this->layout = false; + $this->autoRender = false; + $this->set('servers', $servers); + $this->render('ajax/fetch_servers_for_sg'); + } + + public function postTest() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException('Invalid request, expecting a POST request.'); + } + // Fix for PHP-FPM / Nginx / etc + // Fix via https://www.popmartian.com/tipsntricks/2015/07/14/howto-use-php-getallheaders-under-fastcgi-php-fpm-nginx-etc/ + if (!function_exists('getallheaders')) { + $headers = []; + foreach ($_SERVER as $name => $value) { + $name = strtolower($name); + if (substr($name, 0, 5) === 'http_') { + $headers[str_replace('_', '-', substr($name, 5))] = $value; + } + } + } else { + $headers = getallheaders(); + $headers = array_change_key_case($headers, CASE_LOWER); + } + $result = [ + 'body' => $this->request->getData(), + 'headers' => [ + 'Content-type' => isset($headers['content-type']) ? $headers['content-type'] : 0, + 'Accept' => isset($headers['accept']) ? $headers['accept'] : 0, + 'Authorization' => isset($headers['authorization']) ? 'OK' : 0, + ], + ]; + return $this->RestResponse->viewData($result, 'json'); + } + + public function getRemoteUser($id) + { + $user = $this->Servers->getRemoteUser($id); + if ($user === null) { + throw new NotFoundException(__('Invalid server')); + } + return $this->RestResponse->viewData($user); + } + + public function testConnection($id = false) + { + $server = $this->Servers->get($id); + if (!$server) { + throw new NotFoundException(__('Invalid server')); + } + $result = $this->Servers->runConnectionTest($server->toArray()); + if ($result['status'] == 1) { + if (isset($result['info']['version']) && preg_match('/^[0-9]+\.+[0-9]+\.[0-9]+$/', $result['info']['version'])) { + $perm_sync = isset($result['info']['perm_sync']) ? $result['info']['perm_sync'] : false; + $perm_sighting = isset($result['info']['perm_sighting']) ? $result['info']['perm_sighting'] : false; + $local_version = $this->Servers->checkMISPVersion(); + $version = explode('.', $result['info']['version']); + $mismatch = false; + $newer = false; + $parts = ['major', 'minor', 'hotfix']; + foreach ($parts as $k => $v) { + if (!$mismatch) { + if ($version[$k] > $local_version[$v]) { + $mismatch = $v; + $newer = 'remote'; + } elseif ($version[$k] < $local_version[$v]) { + $mismatch = $v; + $newer = 'local'; + } + } + } + if (!$mismatch && $version[0] == 2 && $version[2] < 111) { + $mismatch = 'proposal'; + } + if (!$perm_sync && !$perm_sighting) { + $result['status'] = 7; + return new Response(['body' => json_encode($result), 'type' => 'json']); + } + if (!$perm_sync && $perm_sighting) { + $result['status'] = 8; + return new Response(['body' => json_encode($result), 'type' => 'json']); + } + return $this->RestResponse->viewData( + [ + 'status' => 1, + 'local_version' => implode('.', $local_version), + 'version' => implode('.', $version), + 'mismatch' => $mismatch, + 'newer' => $newer, + 'post' => isset($result['post']) ? $result['post']['status'] : 'too old', + 'response_encoding' => isset($result['post']['content-encoding']) ? $result['post']['content-encoding'] : null, + 'request_encoding' => isset($result['info']['request_encoding']) ? $result['info']['request_encoding'] : null, + 'client_certificate' => $result['client_certificate'], + ], + 'json' + ); + } else { + $result['status'] = 3; + } + } + return new Response(['body' => json_encode($result), 'type' => 'json']); + } + + public function startZeroMQServer() + { + $pubSubTool = $this->Servers->getPubSubTool(); + $result = $pubSubTool->restartServer(); + if ($result === true) { + return new Response(['body' => json_encode(['saved' => true, 'success' => 'ZeroMQ server successfully started.']), 'status' => 200, 'type' => 'json']); + } else { + return new Response(['body' => json_encode(['saved' => false, 'errors' => $result]), 'status' => 200, 'type' => 'json']); + } + } + + public function stopZeroMQServer() + { + $pubSubTool = $this->Servers->getPubSubTool(); + $result = $pubSubTool->killService(); + if ($result === true) { + return new Response(['body' => json_encode(['saved' => true, 'success' => 'ZeroMQ server successfully killed.']), 'status' => 200, 'type' => 'json']); + } else { + return new Response(['body' => json_encode(['saved' => false, 'errors' => 'Could not kill the previous instance of the ZeroMQ script.']), 'status' => 200, 'type' => 'json']); + } + } + + public function statusZeroMQServer() + { + $pubSubTool = $this->Servers->getPubSubTool(); + $result = $pubSubTool->statusCheck(); + if (!empty($result)) { + $this->set('events', $result['publishCount']); + $this->set('messages', $result['messageCount']); + $this->set('time', $result['timestamp']); + $this->set('time2', $result['timestampSettings']); + } + $this->render('ajax/zeromqstatus'); + } + + public function purgeSessions() + { + if ($this->Servers->updateDatabase('cleanSessionTable') == false) { + $this->Flash->error('Could not purge the session table.'); + } + $this->redirect('/servers/serverSettings/diagnostics'); + } + + public function clearWorkerQueue($worker) + { + if (!$this->request->is('Post') || $this->request->is('ajax')) { + throw new MethodNotAllowedException(); + } + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Servers->getBackgroundJobsTool()->purgeQueue($worker); + } else { + // CakeResque + $worker_array = ['cache', 'default', 'email', 'prio']; + if (!in_array($worker, $worker_array)) { + throw new MethodNotAllowedException('Invalid worker'); + } + $redis = RedisTool::init(); + $redis->del('queue:' . $worker); + } + + $this->Flash->success('Queue cleared.'); + $this->redirect($this->referer()); + } + + public function getVersion() + { + $user = $this->ACL->getUser(); + $versionArray = $this->Servers->checkMISPVersion(); + $response = [ + 'version' => $versionArray['major'] . '.' . $versionArray['minor'] . '.' . $versionArray['hotfix'], + 'pymisp_recommended_version' => $this->pyMispVersion, + 'perm_sync' => (bool) $user['Role']['perm_sync'], + 'perm_sighting' => (bool) $user['Role']['perm_sighting'], + 'perm_galaxy_editor' => (bool) $user['Role']['perm_galaxy_editor'], + 'request_encoding' => $this->CompressedRequestHandler->supportedEncodings(), + 'filter_sightings' => true, // check if Sightings::filterSightingUuidsForPush method is supported + ]; + return $this->RestResponse->viewData($response, 'json'); + } + + /** + * @deprecated Use field `pymisp_recommended_version` from getVersion instead + */ + public function getPyMISPVersion() + { + $this->set('response', ['version' => $this->pyMispVersion]); + $this->set('_serialize', 'response'); + } + + public function checkout() + { + $result = $this->Servers->checkoutMain(); + } + + public function update($branch = false) + { + if ($this->request->is('post')) { + $filterData = [ + 'request' => $this->request, + 'named_params' => $this->params['named'], + 'paramArray' => ['branch'], + 'ordered_url_params' => [], + 'additional_delimiters' => PHP_EOL + ]; + $exception = false; + $settings = $this->harvestParameters($filterData, $exception); + $status = $this->Servers->getCurrentGitStatus(); + $raw = []; + if (empty($status['branch'])) { // do not try to update if you are not on branch + $msg = 'Update failed, you are not on branch'; + $raw[] = $msg; + $update = $msg; + } else { + if ($settings === false) { + $settings = []; + } + $update = $this->Servers->update($status, $raw, $settings); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData(['results' => $raw], $this->response->getType()); + } else { + return new Response(['body' => $update, 'type' => 'txt']); + } + } else { + $this->set('isUpdatePossible', $this->Servers->isUpdatePossible()); + $this->set('branch', $this->Servers->getCurrentBranch()); + $this->render('ajax/update'); + } + } + + public function ondemandAction() + { + $AdminSettingsTable = $this->fetchTable('AdminSettings'); + $actions = $this->Servers->actions_description; + $default_fields = [ + 'title' => '', + 'description' => '', + 'liveOff' => false, + 'recommendBackup' => false, + 'exitOnError' => false, + 'requirements' => '', + 'url' => $this->baseurl . '/' + ]; + foreach ($actions as $id => $action) { + foreach ($default_fields as $field => $value) { + if (!isset($action[$field])) { + $actions[$id][$field] = $value; + } + } + $done = $AdminSettingsTable->getSetting($id); + $actions[$id]['done'] = ($done == '1'); + } + $this->set('actions', $actions); + $this->set('updateLocked', $this->Servers->isUpdateLocked()); + } + + public function getSubmoduleQuickUpdateForm($submodule_path = false) + { + $this->set('submodule', base64_decode($submodule_path)); + $this->render('ajax/submodule_quick_update_form'); + } + + public function updateSubmodule() + { + if ($this->request->is('post')) { + $request = $this->request->getData(); + $submodule = $request['Server']['submodule']; + $res = $this->Servers->updateSubmodule($this->ACL->getUser(), $submodule); + return new Response(['body' => json_encode($res), 'type' => 'json']); + } else { + throw new MethodNotAllowedException(); + } + } + + public function getInstanceUUID() + { + return $this->RestResponse->viewData(['uuid' => Configure::read('MISP.uuid')], $this->response->getType()); + } + + public function cache($id = 'all') + { + if (Configure::read('BackgroundJobs.enabled')) { + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->ACL->getUser(), + Job::WORKER_DEFAULT, + 'cache_servers', + intval($id) ? $id : 'all', + __('Starting server caching.') + ); + + $this->Servers->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheServer', + $this->ACL->getUser()->id, + $id, + $jobId + ], + false, + $jobId + ); + + $message = 'Server caching job initiated.'; + } else { + $result = $this->Servers->cacheServerInitiator($this->ACL->getUser(), $id); + if (!$result) { + $this->Flash->error(__('Caching the servers has failed.')); + $this->redirect(['action' => 'index']); + } + $message = __('Caching the servers has successfully completed.'); + } + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveSuccessResponse('Server', 'cache', false, $this->response->getType(), $message); + } else { + $this->Flash->info($message); + $this->redirect(['action' => 'index']); + } + } + + public function updateJSON() + { + $results = $this->Servers->updateJSON(); + return $this->RestResponse->viewData($results, $this->response->getType()); + } + + public function createSync() + { + if ($this->isSiteAdmin()) { + throw new MethodNotAllowedException('Site admin accounts cannot be used to create server sync configurations.'); + } + $baseurl = Configure::read('MISP.external_baseurl'); + if (empty($baseurl)) { + $baseurl = Configure::read('MISP.baseurl'); + if (empty($baseurl)) { + $baseurl = Router::url('/', true); + } + } + $host_org_id = Configure::read('MISP.host_org_id'); + if (empty($host_org_id)) { + throw new MethodNotAllowedException(__('Cannot create sync config - no host org ID configured for the instance.')); + } + $OrganisationsTable = $this->fetchTable('Organisations'); + $host_org = $OrganisationsTable->find( + 'all', + [ + 'conditions' => ['Organisation.id' => $host_org_id], + 'recursive' => -1, + 'fields' => ['name', 'uuid'] + ] + )->first(); + if (empty($host_org)) { + throw new MethodNotAllowedException(__('Configured host org not found. Please make sure that the setting is current on the instance.')); + } + if (Configure::read('Security.advanced_authkeys')) { + $AuthKeysTable = $this->fetchTable('AuthKeys'); + $authkey = $AuthKeysTable->createnewkey($this->ACL->getUser()->id, null, __('Auto generated sync key - {0}', date('Y-m-d H:i:s'))); + } else { + $UsersTable = $this->fetchTable('Users'); + $authkey = $UsersTable->find( + 'column', + [ + 'conditions' => ['User.id' => $this->ACL->getUser()->id], + 'recursive' => -1, + 'fields' => ['User.authkey'] + ] + ); + $authkey = $authkey[0]; + } + $server = [ + 'Server' => [ + 'url' => $baseurl, + 'uuid' => Configure::read('MISP.uuid'), + 'authkey' => h($authkey), + 'Organisation' => [ + 'name' => $host_org['Organisation']['name'], + 'uuid' => $host_org['Organisation']['uuid'], + ] + ] + ]; + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($server, $this->response->getType()); + } else { + $this->set('server', $server); + } + } + + public function import() + { + if ($this->request->is('post')) { + $server = $this->request->getData(); + if (isset($server['Server'])) { + $server = $server['Server']; + } + if (isset($server['json'])) { + $server = json_decode($server['json'], true)['Server']; + } + $OrganisationsTable = $this->fetchTable('Organisations'); + $org_id = $OrganisationsTable->captureOrg($server['Organisation'], $this->ACL->getUser()); + $toSave = [ + 'push' => 0, + 'pull' => 0, + 'caching_enabled' => 0, + 'json' => '[]', + 'push_rules' => [], + 'pull_rules' => [], + 'self_signed' => 0, + 'org_id' => $this->ACL->getUser()->org_id, + 'remote_org_id' => $org_id, + 'name' => empty($server['name']) ? $server['url'] : $server['name'], + 'url' => $server['url'], + 'authkey' => $server['authkey'] + ]; + $serverEntity = $this->Servers->newEntity($toSave); + $result = $this->Servers->save($serverEntity); + if ($result) { + if ($this->ParamHandler->isRest()) { + $server = $this->Servers->get($serverEntity->id); + return $this->RestResponse->viewData($server, $this->response->getType()); + } else { + $this->Flash->success(__('The server has been saved')); + $this->redirect(['action' => 'index', $serverEntity->id]); + } + } else { + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Servers->validationErrors, $this->response->getType()); + } else { + $this->Flash->error(__('Could not save the server. Error: {0}', json_encode($this->Servers->validationErrors))); + $this->redirect(['action' => 'index']); + } + } + } + } + + public function resetRemoteAuthKey($id) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This endpoint expects POST requests.')); + } + $result = $this->Servers->resetRemoteAuthkey($id); + if ($result !== true) { + if (!$this->ParamHandler->isRest()) { + $this->Flash->error($result); + $this->redirect(['action' => 'index']); + } else { + $message = __('Could not update API key.'); + return $this->RestResponse->saveFailResponse('Servers', 'resetRemoteAuthKey', $id, $message, $this->response->getType()); + } + } else { + $message = __('API key updated.'); + if (!$this->ParamHandler->isRest()) { + $this->Flash->success($message); + $this->redirect(['action' => 'index']); + } else { + return $this->RestResponse->saveSuccessResponse('Servers', 'resetRemoteAuthKey', $message, $this->response->getType()); + } + } + } + + public function changePriority($id = false, $direction = 'down') + { + $this->Servers->id = $id; + if (!$this->Servers->exists()) { + throw new InvalidArgumentException(__('ID has to be a valid server connection')); + } + if ($direction !== 'up' && $direction !== 'down') { + throw new InvalidArgumentException(__('Invalid direction. Valid options: ', 'up', 'down')); + } + $success = $this->Servers->reprioritise($id, $direction); + if ($success) { + $message = __('Priority changed.'); + return $this->RestResponse->saveSuccessResponse('Servers', 'changePriority', $message, $this->response->getType()); + } else { + $message = __('Priority could not be changed.'); + return $this->RestResponse->saveFailResponse('Servers', 'changePriority', $id, $message, $this->response->getType()); + } + } + + public function releaseUpdateLock() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(__('This endpoint expects POST requests.')); + } + $this->Servers->changeLockState(false); + $this->Servers->resetUpdateFailNumber(); + $this->redirect(['action' => 'updateProgress']); + } + + public function dbSchemaDiagnostic() + { + $dbSchemaDiagnostics = $this->Servers->dbSchemaDiagnostic(); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($dbSchemaDiagnostics, $this->response->getType()); + } else { + $this->set('checkedTableColumn', $dbSchemaDiagnostics['checked_table_column']); + $this->set('dbSchemaDiagnostics', $dbSchemaDiagnostics['diagnostic']); + $this->set('dbIndexDiagnostics', $dbSchemaDiagnostics['diagnostic_index']); + $this->set('expectedDbVersion', $dbSchemaDiagnostics['expected_db_version']); + $this->set('actualDbVersion', $dbSchemaDiagnostics['actual_db_version']); + $this->set('error', $dbSchemaDiagnostics['error']); + $this->set('remainingLockTime', $dbSchemaDiagnostics['remaining_lock_time']); + $this->set('updateFailNumberReached', $dbSchemaDiagnostics['update_fail_number_reached']); + $this->set('updateLocked', $dbSchemaDiagnostics['update_locked']); + $this->set('dataSource', $dbSchemaDiagnostics['dataSource']); + $this->set('columnPerTable', $dbSchemaDiagnostics['columnPerTable']); + $this->set('indexes', $dbSchemaDiagnostics['indexes']); + $this->render('/Elements/healthElements/db_schema_diagnostic'); + } + } + + public function dbConfiguration() + { + $dbConfiguration = $this->Servers->dbConfiguration(); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($dbConfiguration, $this->response->getType()); + } else { + $this->set('dbConfiguration', $dbConfiguration); + $this->render('/Elements/healthElements/db_config_diagnostic'); + } + } + + public function cspReport() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException('This action expects a POST request.'); + } + + $report = JsonTool::decode((string)$this->request->getBody()); + if (!isset($report['csp-report'])) { + throw new RuntimeException("Invalid report"); + } + + $message = 'CSP reported violation'; + $remoteIp = $this->_remoteIp(); + if ($remoteIp) { + $message .= ' from IP ' . $remoteIp; + } + $report = JsonTool::encode($report['csp-report'], true); + if (strlen($report) > 1024 * 1024) { // limit report to 1 kB + $report = substr($report, 0, 1024 * 1024) . '...'; + } + $this->log("$message: $report"); + + return new Response(['status' => 204]); + } + + /** + * List all tags for the rule picker. + * + * @return array + */ + private function __getTags() + { + $TagsTable = $this->fetchTable('Tags'); + $list = $TagsTable->find( + 'list', + [ + 'recursive' => -1, + 'order' => ['LOWER(TRIM(Tag.name))' => 'ASC'], + 'fields' => ['name'], + ] + ); + $allTags = []; + foreach ($list as $id => $name) { + $allTags[] = ['id' => $id, 'name' => trim($name)]; + } + return $allTags; + } + + public function removeOrphanedCorrelations() + { + $count = $this->Servers->removeOrphanedCorrelations(); + $message = __('{0} orphaned correlation removed', $count); + if ($this->ParamHandler->isRest()) { + return $this->RestResponse->viewData($message, $this->response->getType()); + } else { + $this->Flash->success($message); + $this->redirect(['action' => 'serverSettings', 'diagnostics']); + } + } + + public function queryAvailableSyncFilteringRules($serverID) + { + if (!$this->ParamHandler->isRest()) { + throw new MethodNotAllowedException(__('This method can only be access via REST')); + } + $server = $this->Servers->get($serverID); + if (!$server) { + throw new NotFoundException(__('Invalid server')); + } + $syncFilteringRules = $this->Servers->queryAvailableSyncFilteringRules($server); + return $this->RestResponse->viewData($syncFilteringRules); + } + + public function getAvailableSyncFilteringRules() + { + if (!$this->ParamHandler->isRest()) { + throw new MethodNotAllowedException(__('This method can only be access via REST')); + } + $syncFilteringRules = $this->Servers->getAvailableSyncFilteringRules($this->ACL->getUser()); + return $this->RestResponse->viewData($syncFilteringRules); + } + + public function pruneDuplicateUUIDs() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + $AttributesTable = $this->fetchTable('Attributes'); + $duplicates = $AttributesTable->find( + 'all', + [ + 'fields' => ['Attribute.uuid', 'count(*) as occurance'], + 'recursive' => -1, + 'group' => ['Attribute.uuid HAVING COUNT(*) > 1'], + ] + ); + $counter = 0; + foreach ($duplicates as $duplicate) { + $attributes = $AttributesTable->find( + 'all', + [ + 'recursive' => -1, + 'conditions' => ['uuid' => $duplicate['Attribute']['uuid']] + ] + ); + foreach ($attributes as $k => $attribute) { + if ($k > 0) { + $AttributesTable->delete($attribute['Attribute']['id']); + $counter++; + } + } + } + $this->Servers->updateDatabase('makeAttributeUUIDsUnique'); + $this->Flash->success('Done. Deleted ' . $counter . ' duplicate attribute(s).'); + $this->redirect(['controller' => 'pages', 'action' => 'display', 'administration']); + } + + public function removeDuplicateEvents() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + $EventsTable = $this->fetchTable('Events'); + $duplicates = $EventsTable->find( + 'all', + [ + 'fields' => ['Event.uuid', 'count(*) as occurance'], + 'recursive' => -1, + 'group' => ['Event.uuid HAVING COUNT(*) > 1'], + ] + ); + $counter = 0; + + // load this so we can remove the blocklist item that will be created, this is the one case when we do not want it. + if (Configure::read('MISP.enableEventBlocklisting') !== false) { + $EventsTableBlocklist = $this->fetchTable('EventBlocklist'); + } + + foreach ($duplicates as $duplicate) { + $events = $EventsTable->find( + 'all', + [ + 'recursive' => -1, + 'conditions' => ['uuid' => $duplicate['Event']['uuid']] + ] + ); + foreach ($events as $k => $event) { + if ($k > 0) { + $uuid = $event['Event']['uuid']; + $EventsTable->delete($event['Event']['id']); + $counter++; + // remove the blocklist entry that we just created with the event deletion, if the feature is enabled + // We do not want to block the UUID, since we just deleted a copy + if (Configure::read('MISP.enableEventBlocklisting') !== false) { + $EventsTableBlocklist->deleteAll(['EventBlocklist.event_uuid' => $uuid]); + } + } + } + } + $this->Servers->updateDatabase('makeEventUUIDsUnique'); + $this->Flash->success('Done. Removed ' . $counter . ' duplicate events.'); + $this->redirect(['controller' => 'pages', 'action' => 'display', 'administration']); + } + + public function upgrade2324() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + if (!Configure::read('BackgroundJobs.enabled')) { + $this->Servers->upgrade2324($this->ACL->getUser()->id); + $this->Flash->success('Done. For more details check the audit logs.'); + $this->redirect(['controller' => 'pages', 'action' => 'display', 'administration']); + } else { + + /** @var JobsTable $JobsTable */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $this->ACL->getUser(), + Job::WORKER_DEFAULT, + 'upgrade_24', + 'Old database', + __('Job created.') + ); + + $this->Servers->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobUpgrade24', + $jobId, + $this->ACL->getUser()->id, + ], + true, + $jobId + ); + + $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).')); + $this->redirect(['controller' => 'pages', 'action' => 'display', 'administration']); + } + } + + public function cleanModelCaches() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + $this->Servers->cleanCacheFiles(); + $this->Flash->success('Caches cleared.'); + $this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'diagnostics']); + } + + public function updateDatabase($command) + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException(); + } + if (is_numeric($command)) { + $command = intval($command); + } + $this->Servers->updateDatabase($command); + $this->Flash->success('Done.'); + $this->redirect(['controller' => 'pages', 'action' => 'display', 'administration']); + } + + public function ipUser($input = false) + { + $params = $this->harvestParameters(['ip']); + if (!empty($params['ip'])) { + $input = $params['ip']; + } + $redis = RedisTool::init(); + if (!is_array($input)) { + $input = [$input]; + } + $users = []; + foreach ($input as $ip) { + if (!filter_var($ip, FILTER_VALIDATE_IP)) { + continue; + } + $user_id = $redis->get('misp:ip_user:' . $ip); + if (empty($user_id)) { + continue; + } + $UsersTable = $this->fetchTable('Users'); + $user = $UsersTable->find( + 'all', + [ + 'recursive' => -1, + 'conditions' => ['User.id' => $user_id], + 'contain' => ['Organisation.name'] + ] + )->first(); + if (empty($user)) { + throw new NotFoundException(__('User not found (perhaps it has been removed?).')); + } + $users[$ip] = [ + 'id' => $user['User']['id'], + 'email' => $user['User']['email'], + ]; + } + return $this->RestResponse->viewData($users, $this->response->getType()); + } + + /** + * @deprecated + * @return void + */ + public function rest() + { + $this->redirect(['controller' => 'api', 'action' => 'rest']); + } +} diff --git a/src/Http/Exception/HttpSocketJsonException.php b/src/Http/Exception/HttpSocketJsonException.php new file mode 100644 index 000000000..e646c93fa --- /dev/null +++ b/src/Http/Exception/HttpSocketJsonException.php @@ -0,0 +1,27 @@ +response = $response; + parent::__construct($message, 0, $previous); + } + + /** + * @return HttpSocketResponseExtended + */ + public function getResponse() + { + return $this->response; + } +} diff --git a/src/Lib/Tools/AWSS3Client.php b/src/Lib/Tools/AWSS3Client.php new file mode 100644 index 000000000..61059db9a --- /dev/null +++ b/src/Lib/Tools/AWSS3Client.php @@ -0,0 +1,158 @@ + false, + 'bucket_name' => 'my-malware-bucket', + 'region' => 'eu-west-1', + 'aws_access_key' => '', + 'aws_secret_key' => '', + 'aws_endpoint' => '', + 'aws_compatible' => false, + 'aws_ca' => '', + 'aws_validate_ca' => true + ); + + // We have 2 situations + // Either we're running on EC2 and we can assume an IAM role + // Or we're not and need explicitly set AWS key + if (strlen($settings['aws_access_key']) > 0) { + putenv('AWS_ACCESS_KEY_ID=' . $settings['aws_access_key']); + } + if (strlen($settings['aws_secret_key']) > 0) { + putenv('AWS_SECRET_ACCESS_KEY=' . $settings['aws_secret_key']); + } + + foreach ($settings as $key => $setting) { + $temp = Configure::read('Plugin.S3_' . $key); + if ($temp) { + $settings[$key] = $temp; + } + } + return $settings; + } + + public function initTool() + { + $settings = $this->__getSetSettings(); + $s3Config = array( + 'version' => 'latest', + 'region' => $settings['region'], + ); + if ($settings['aws_compatible']) { + $s3Config = array( + 'version' => 'latest', + 'region' => $settings['region'], + // MinIO compatibility + // Reference: https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html + 'endpoint' => $settings['aws_endpoint'], + 'use_path_style_endpoint' => true, + 'credentials' => [ + 'key' => $settings['aws_access_key'], + 'secret' => $settings['aws_secret_key'], + ], + ); + } + // This line should points to server certificate + // Generically, this verify is set to false so that any certificate is valid + // Reference: + // - https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html + // - https://docs.guzzlephp.org/en/5.3/clients.html#verify + // Example: + // -- Verify certificate + // 'http' => ['verify' => '/usr/lib/ssl/certs/minio.pem'], + // -- Do not verify certificate, securitywise, this option is not recommended, however due to + // internal deployment scheme it is acceptable risk to set this to false + // 'http' => ['verify' => false], + // -- Verify againts built in CA certificates + // 'http' => ['verify' => true], + if ($settings['aws_validate_ca']) { + $s3Config['http']['verify'] = true; + if (!empty($settings['aws_ca'])) { + $s3Config['http']['verify'] = $settings['aws_ca']; + } + } else { + $s3Config['http']['verify'] = false; + } + $s3Client = new S3Client($s3Config); + $this->__client = $s3Client; + $this->__settings = $settings; + return $s3Client; + } + + public function exist($key) + { + return $this->__client->doesObjectExist([ + 'Bucket' => $this->__settings['bucket_name'], + 'Key' => $key, + ]); + } + + public function upload($key, $data) + { + $this->__client->putObject([ + 'Bucket' => $this->__settings['bucket_name'], + 'Key' => $key, + 'Body' => $data + ]); + } + + public function download($key) + { + try { + $result = $this->__client->getObject([ + 'Bucket' => $this->__settings['bucket_name'], + 'Key' => $key + ]); + + return $result['Body']; + } catch (AwsException $e) { + throw new NotFoundException('Could not download object ' . $e->getMessage()); + } + } + + public function delete($key) + { + $this->__client->deleteObject([ + 'Bucket' => $this->__settings['bucket_name'], + 'Key' => $key + ]); + } + + public function deleteDirectory($prefix) + { + $keys = $this->__client->listObjectsV2([ + 'Bucket' => $this->__settings['bucket_name'], + 'Prefix' => $prefix + ]); + + $toDelete = array_map( + function ($key) { + return ['Key' => $key['Key']]; + }, + is_array($keys['Contents']) ? $keys['Contents'] : [] + ); + + if (sizeof($toDelete) != 0) { + $this->__client->deleteObjects([ + 'Bucket' => $this->__settings['bucket_name'], + 'Delete' => [ + 'Objects' => $toDelete + ] + ]); + } + } +} diff --git a/src/Lib/Tools/AttachmentTool.php b/src/Lib/Tools/AttachmentTool.php new file mode 100644 index 000000000..44b3e18c2 --- /dev/null +++ b/src/Lib/Tools/AttachmentTool.php @@ -0,0 +1,514 @@ +_exists(false, $eventId, $attributeId, $path_suffix); + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $path_suffix + * @return bool + * @throws Exception + */ + public function shadowExists($eventId, $attributeId, $path_suffix = '') + { + return $this->_exists(true, $eventId, $attributeId, $path_suffix); + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $path_suffix + * @return bool + * @throws Exception + */ + protected function _exists($shadow, $eventId, $attributeId, $path_suffix = '') + { + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix); + return $s3->exist($path); + } else { + try { + $this->_getFile($shadow, $eventId, $attributeId, $path_suffix); + } catch (NotFoundException $e) { + return false; + } + } + + return true; + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $path_suffix + * @return string + * @throws Exception + */ + public function getContent($eventId, $attributeId, $path_suffix = '') + { + return $this->_getContent(false, $eventId, $attributeId, $path_suffix); + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $path_suffix + * @return string + * @throws Exception + */ + public function getShadowContent($eventId, $attributeId, $path_suffix = '') + { + return $this->_getContent(true, $eventId, $attributeId, $path_suffix); + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $path_suffix + * @return string + * @throws Exception + */ + protected function _getContent($shadow, $eventId, $attributeId, $path_suffix = '') + { + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix); + return $s3->download($path); + } else { + $file = $this->_getFile($shadow, $eventId, $attributeId, $path_suffix); + $result = $file->read(); + if ($result === false) { + throw new Exception("Could not read file '{$file->path}'."); + } + return $result; + } + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return File + * @throws Exception + */ + public function getFile($eventId, $attributeId, $pathSuffix = '') + { + return $this->_getFile(false, $eventId, $attributeId, $pathSuffix); + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return File + * @throws Exception + */ + public function getShadowFile($eventId, $attributeId, $pathSuffix = '') + { + return $this->_getFile(true, $eventId, $attributeId, $pathSuffix); + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return File + * @throws Exception + */ + protected function _getFile($shadow, $eventId, $attributeId, $pathSuffix = '') + { + $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix); + + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $content = $s3->download($path); + $file = FileAccessTool::writeToTempFile($content); + } else { + $filepath = $this->attachmentDir() . DS . $path; + $file = FileAccessTool::createFile($filepath); + } + + return $file; + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $data + * @param string $pathSuffix + * @return bool + * @throws Exception + */ + public function save($eventId, $attributeId, $data, $pathSuffix = '') + { + return $this->_save(false, $eventId, $attributeId, $data, $pathSuffix); + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $data + * @param string $pathSuffix + * @return bool + * @throws Exception + */ + public function saveShadow($eventId, $attributeId, $data, $pathSuffix = '') + { + return $this->_save(true, $eventId, $attributeId, $data, $pathSuffix); + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $data + * @param string $pathSuffix + * @return bool + * @throws Exception + */ + protected function _save($shadow, $eventId, $attributeId, $data, $pathSuffix = '') + { + $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix); + + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $s3->upload($path, $data); + } else { + $path = $this->attachmentDir() . DS . $path; + FileAccessTool::writeToFile($path, $data, true); + } + + return true; + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return bool + * @throws Exception + */ + public function delete($eventId, $attributeId, $pathSuffix = '') + { + return $this->_delete(false, $eventId, $attributeId, $pathSuffix); + } + + /** + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return bool + * @throws Exception + */ + public function deleteShadow($eventId, $attributeId, $pathSuffix = '') + { + return $this->_delete(true, $eventId, $attributeId, $pathSuffix); + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return bool Return true if file was deleted, `false` if file doesn't exists. + * @throws Exception + */ + protected function _delete($shadow, $eventId, $attributeId, $pathSuffix = '') + { + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix); + $s3->delete($path); + } else { + try { + $file = $this->_getFile($shadow, $eventId, $attributeId, $pathSuffix); + } catch (NotFoundException $e) { + return false; + } + + if (!$file->delete()) { + throw new Exception(__('Delete of file attachment failed. Please report to administrator.')); + } + } + + return true; + } + + /** + * Deletes all attributes and shadow attributes files. + * + * @param int $eventId + * @return bool + * @throws Exception + */ + public function deleteAll($eventId) + { + if ($this->attachmentDirIsS3()) { + $s3 = $this->loadS3Client(); + $s3->deleteDirectory($eventId); + } else { + $dirPath = $this->attachmentDir(); + + foreach (array($dirPath, $dirPath . DS . 'shadow') as $dirPath) { + $folder = new SplFileInfo($dirPath . DS . $eventId); + if (!FileAccessTool::deleteFileIfExists($folder->getPath())) { + throw new Exception("Delete of directory '{$folder->getPath()()}' failed."); + } + } + } + + return true; + } + + /** + * It is not possible to use PHP extensions for compressing. The reason is, that extensions support just AES encrypted + * files, but these files are not supported in Windows and in Python. So the only solution is to use 'zip' command. + * + * @param string $originalFilename + * @param string $content + * @param string $md5 + * @return string Content of zipped file + * @throws Exception + */ + public function encrypt($originalFilename, $content, $md5) + { + $tempDir = $this->tempDir(); + + FileAccessTool::writeToFile($tempDir . DS . $md5, $content); + FileAccessTool::writeToFile($tempDir . DS . $md5 . '.filename.txt', $originalFilename); + + $zipFile = $tempDir . DS . $md5 . '.zip'; + + $exec = [ + 'zip', + '-j', // junk (don't record) directory names + '-P', // use standard encryption + self::ZIP_PASSWORD, + $zipFile, + $tempDir . DS . $md5, + $tempDir . DS . $md5 . '.filename.txt', + ]; + + try { + ProcessTool::execute($exec); + return FileAccessTool::readFromFile($zipFile); + } catch (Exception $e) { + throw new Exception("Could not create encrypted ZIP file '$zipFile'.", 0, $e); + } finally { + FileAccessTool::deleteFile($tempDir . DS . $md5); + FileAccessTool::deleteFile($tempDir . DS . $md5 . '.filename.txt'); + FileAccessTool::deleteFile($zipFile); + } + } + + /** + * @param string $content + * @param array $hashTypes + * @return array + * @throws InvalidArgumentException + */ + public function computeHashes($content, array $hashTypes = array()) + { + $validHashes = array('md5', 'sha1', 'sha256'); + $hashes = []; + foreach ($hashTypes as $hashType) { + if (!in_array($hashType, $validHashes)) { + throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type."); + } + $hashes[$hashType] = hash($hashType, $content); + } + return $hashes; + } + + /** + * @param string $pythonBin + * @param string $filePath + * @return array + * @throws Exception + */ + public function advancedExtraction($filePath) + { + return $this->executeAndParseJsonOutput([ + ProcessTool::pythonBin(), + self::ADVANCED_EXTRACTION_SCRIPT_PATH, + '-p', + $filePath, + ]); + } + + /** + * @param string $pythonBin + * @return array + * @throws Exception + */ + public function checkAdvancedExtractionStatus() + { + return $this->executeAndParseJsonOutput([ProcessTool::pythonBin(), self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']); + } + + /** + * @param string $data + * @param int $maxWidth + * @param int $maxHeight + * @param string $outputFormat Can be 'png' or 'webp' + * @return string + * @throws Exception + */ + public function resizeImage($data, $maxWidth, $maxHeight, $outputFormat = 'png') + { + $image = imagecreatefromstring($data); + if ($image === false) { + throw new Exception("Image is not valid."); + } + + $currentWidth = imagesx($image); + $currentHeight = imagesy($image); + + // Compute thumbnail size with keeping ratio + if ($currentWidth > $currentHeight) { + $newWidth = min($currentWidth, $maxWidth); + $divisor = $currentWidth / $newWidth; + $newHeight = floor($currentHeight / $divisor); + } else { + $newHeight = min($currentHeight, $maxHeight); + $divisor = $currentHeight / $newHeight; + $newWidth = floor($currentWidth / $divisor); + } + + $imageThumbnail = imagecreatetruecolor($newWidth, $newHeight); + + // Allow transparent background + imagealphablending($imageThumbnail, false); + imagesavealpha($imageThumbnail, true); + $transparent = imagecolorallocatealpha($imageThumbnail, 255, 255, 255, 127); + imagefilledrectangle($imageThumbnail, 0, 0, $newWidth, $newHeight, $transparent); + + // Resize image + imagecopyresampled($imageThumbnail, $image, 0, 0, 0, 0, $newWidth, $newHeight, $currentWidth, $currentHeight); + imagedestroy($image); + + // Output image to string + ob_start(); + if ($outputFormat === 'webp') { + if (!function_exists('imagewebp')) { + throw new InvalidArgumentException("Webp image format is not supported."); + } + imagewebp($imageThumbnail); + } elseif ($outputFormat === 'png') { + imagepng($imageThumbnail, null, 9); + } else { + throw new InvalidArgumentException("Unsupported image format $outputFormat."); + } + $imageData = ob_get_clean(); + imagedestroy($imageThumbnail); + + return $imageData; + } + + private function tempFileName() + { + $randomName = (new RandomTool())->random_str(false, 12); + return $this->tempDir() . DS . $randomName; + } + + /** + * @return string + */ + private function tempDir() + { + return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir(); + } + + /** + * @return string + */ + private function attachmentDir() + { + return Configure::read('MISP.attachments_dir') ?: (APP . 'files'); + } + + /** + * Naive way to detect if we're working in S3 + * @return bool + */ + public function attachmentDirIsS3() + { + $attachmentsDir = Configure::read('MISP.attachments_dir'); + return $attachmentsDir && substr($attachmentsDir, 0, 2) === "s3"; + } + + /** + * @return AWSS3Client + */ + private function loadS3Client() + { + if ($this->s3client) { + return $this->s3client; + } + + $client = new AWSS3Client(); + $client->initTool(); + $this->s3client = $client; + return $client; + } + + /** + * @param bool $shadow + * @param int $eventId + * @param int $attributeId + * @param string $pathSuffix + * @return string + */ + private function getPath($shadow, $eventId, $attributeId, $pathSuffix) + { + $path = $shadow ? ('shadow' . DS) : ''; + return $path . $eventId . DS . $attributeId . $pathSuffix; + } + + /** + * @param array $command + * @return array + * @throws Exception + */ + private function executeAndParseJsonOutput(array $command) + { + $output = ProcessTool::execute($command); + try { + return JsonTool::decode($output); + } catch (Exception $e) { + throw new Exception("Command output is not valid JSON.", 0, $e); + } + } +} diff --git a/src/Lib/Tools/BackgroundJobsTool.php b/src/Lib/Tools/BackgroundJobsTool.php index 703a3f3c5..24486bdf7 100644 --- a/src/Lib/Tools/BackgroundJobsTool.php +++ b/src/Lib/Tools/BackgroundJobsTool.php @@ -72,13 +72,15 @@ class BackgroundJobsTool public const CMD_EVENT = 'event', - CMD_SERVER = 'server', + CMD_SERVER = 'servers', + CMD_FEEDS = 'feeds', CMD_ADMIN = 'admin', CMD_WORKFLOW = 'workflow'; public const ALLOWED_COMMANDS = [ self::CMD_EVENT, self::CMD_SERVER, + self::CMD_FEEDS, self::CMD_ADMIN, self::CMD_WORKFLOW ]; @@ -385,7 +387,7 @@ class BackgroundJobsTool foreach ($procs as $proc) { if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) { $name = explode("_", $proc->offsetGet('name'))[0]; - if ($name === $queue && $proc->offsetGet('state') != \Supervisor\Process::RUNNING) { + if ($name === $queue && $proc->offsetGet('state') != \Supervisor\ProcessStates::Running) { return $this->getSupervisor()->startProcess( sprintf( '%s:%s', @@ -512,7 +514,7 @@ class BackgroundJobsTool */ public function getSupervisorStatus(): bool { - return $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING; + return $this->getSupervisor()->getState()['statecode'] === \Supervisor\ProcessStates::Running; } /** @@ -662,12 +664,6 @@ class BackgroundJobsTool ) ); - if (class_exists('Supervisor\Connector\XmlRpc')) { - // for compatibility with older versions of supervisor - $connector = new \Supervisor\Connector\XmlRpc($client); - return new \Supervisor\Supervisor($connector); - } - return new \Supervisor\Supervisor($client); } @@ -725,9 +721,9 @@ class BackgroundJobsTool private function convertProcessStatus(int $stateId): int { switch ($stateId) { - case \Supervisor\Process::RUNNING: + case \Supervisor\ProcessStates::Running: return Worker::STATUS_RUNNING; - case \Supervisor\Process::UNKNOWN: + case \Supervisor\ProcessStates::Unknown: return Worker::STATUS_UNKNOWN; default: return Worker::STATUS_FAILED; @@ -750,7 +746,7 @@ class BackgroundJobsTool } } - static function getInstance() + public static function getInstance() { if (!self::$instance) { self::$instance = new BackgroundJobsTool(Configure::read('BackgroundJobs')); diff --git a/src/Lib/Tools/EncryptedValue.php b/src/Lib/Tools/EncryptedValue.php index ad1f76aee..dc496d47b 100644 --- a/src/Lib/Tools/EncryptedValue.php +++ b/src/Lib/Tools/EncryptedValue.php @@ -29,12 +29,14 @@ class EncryptedValue implements JsonSerializable * @throws JsonException * @throws Exception */ - public function decrypt($key=false) + public function decrypt($key = false) { if (!$key) { $key = Configure::read('Security.encryption_key'); } - if (!$key) return ''; + if (!$key) { + return ''; + } $decrypt = BetterSecurity::decrypt(substr($this->value, 2), $key); return $this->isJson ? JsonTool::decode($decrypt) : $decrypt; } @@ -55,7 +57,7 @@ class EncryptedValue implements JsonSerializable * @return string * @throws Exception */ - public static function encryptIfEnabled($value, $key=false) + public static function encryptIfEnabled($value, $key = false) { if (!$key) { $key = Configure::read('Security.encryption_key'); @@ -72,15 +74,16 @@ class EncryptedValue implements JsonSerializable * @return string * @throws Exception */ - public static function decryptIfEncrypted($value, $key=false) + public static function decryptIfEncrypted($value, $key = false) { - if(is_resource($value)) + if (is_resource($value)) { $value = stream_get_contents($value); + } if (EncryptedValue::isEncrypted($value)) { $self = new EncryptedValue($value); - return $self->decrypt($key); + return $self->decrypt($key); } else { - return $value; + return trim($value, "\x00"); } } diff --git a/src/Lib/Tools/HttpTool.php b/src/Lib/Tools/HttpTool.php index 3868279a3..4fbcf38df 100644 --- a/src/Lib/Tools/HttpTool.php +++ b/src/Lib/Tools/HttpTool.php @@ -13,6 +13,8 @@ use Cake\I18n\FrozenTime; class HttpTool extends CakeClient { + public const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt']; + /** * Create a new MISP specific HTTP Client * {@inheritdoc} In addition brings some MISP specifics to the game. @@ -406,4 +408,16 @@ class HttpTool extends CakeClient return $output; } + + /** + * @return array|null + */ + public function getMetaData() + { + // TODO: [3.x-MIGRATION] + // if ($this->connection) { + // return stream_get_meta_data($this->connection); + // } + return null; + } } diff --git a/src/Lib/Tools/SecurityAudit.php b/src/Lib/Tools/SecurityAudit.php new file mode 100644 index 000000000..c440af156 --- /dev/null +++ b/src/Lib/Tools/SecurityAudit.php @@ -0,0 +1,600 @@ +config['password']; + if (empty($databasePassword)) { + $output['Database'][] = ['error', __('Database password not set.')]; + } else if (strlen($databasePassword) < self::STRONG_PASSWORD_LENGTH) { + $output['Database'][] = ['warning', __('Database password is too short, should be at least %s chars long.', self::STRONG_PASSWORD_LENGTH)]; + } + + if (!Configure::read('Security.encryption_key')) { + $output['Database'][] = ['warning', __('Sensitive information like keys to remote server are stored in database unencrypted. Set `Security.encryption_key` to encrypt these values.')]; + } + + $passwordPolicyLength = Configure::read('Security.password_policy_length') ?: $server->serverSettings['Security']['password_policy_length']['value']; + if ($passwordPolicyLength < 8) { + $output['Password'][] = ['error', __('Minimum password length is set to %s, it is highly advised to increase it.', $passwordPolicyLength)]; + } elseif ($passwordPolicyLength < 12) { + $output['Password'][] = ['warning', __('Minimum password length is set to %s, consider raising to at least 12 characters.', $passwordPolicyLength)]; + } + + if (empty(Configure::read('Security.require_password_confirmation'))) { + $output['Password'][] = [ + 'warning', + __('Password confirmation is not enabled. %s', $server->serverSettings['Security']['require_password_confirmation']['description']), + ]; + } + if (!empty(Configure::read('Security.auth')) && !Configure::read('Security.auth_enforced')) { + $output['Login'][] = [ + 'hint', + __('External authentication is enabled, but local accounts will still work. You can disable the ability to log in via local accounts by setting `Security.auth_enforced` to `true`.'), + ]; + } + + if (!Configure::read('Security.alert_on_suspicious_logins')) { + $output['Login'][] = [ + 'warning', + __('Warning about suspicious logins is disabled. You can enable alert by setting `Security.alert_on_suspicious_logins` to `true`.'), + ]; + } + + if (empty(Configure::read('Security.disable_browser_cache'))) { + $output['Browser'][] = [ + 'warning', + __('Browser cache is enabled. An attacker could obtain sensitive data from the user cache. You can disable the cache by setting `Security.disable_browser_cache` to `true`.'), + ]; + } + if (empty(Configure::read('Security.check_sec_fetch_site_header'))) { + $output['Browser'][] = [ + 'warning', + __('The MISP server is not checking `Sec-Fetch` HTTP headers. This is a protection mechanism against CSRF used by modern browsers. You can enable this check by setting `Security.check_sec_fetch_site_header` to `true`.'), + ]; + } + if (empty(Configure::read('Security.csp_enforce'))) { + $output['Browser'][] = [ + 'warning', + __('Content security policies (CSP) are not enforced. Consider enabling them by setting `Security.csp_enforce` to `true`.'), + 'https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP', + ]; + } + if (!env('HTTPS') && strpos(Configure::read('MISP.baseurl'), 'https://') === 0) { + $output['Browser'][] = [ + 'error', + __('MISP base URL is set to https://, but MISP thinks that the connection is insecure. This usually happens when a server is running behind a reverse proxy. By setting `Security.force_https` to `true`, session cookies will be set as Secure and CSP headers will upgrade insecure requests.'), + ]; + } + $sessionConfig = Configure::read('Session'); + if (isset($sessionConfig['ini']['session.cookie_secure']) && !$sessionConfig['ini']['session.cookie_secure']) { + $output['Browser'][] = ['error', __('Setting session cookies as not secure is never a good idea.')]; + } + + if (empty(Configure::read('Security.advanced_authkeys'))) { + $output['Auth Key'][] = ['warning', __('Consider enabling Advanced Auth Keys, they provide increased security by only storing the API key hashes.')]; + } + if (Configure::read('Security.allow_unsafe_apikey_named_param')) { + $output['Auth Key'][] = ['error', __('It is possible to pass API keys via the URL, meaning that the keys can be logged by proxies.')]; + } + if (empty(Configure::read('Security.do_not_log_authkeys'))) { + $output['Auth Key'][] = ['warning', __('Auth Key logging is not disabled. Auth Keys in cleartext can be visible in the Audit log.')]; + } + + $salt = Configure::read('Security.salt'); + if (empty($salt)) { + $output['Security salt'][] = ['error', __('Salt is not set.')]; + } else if (strlen($salt) < 32) { + $output['Security salt'][] = ['warning', __('Salt is too short, should contain at least 32 characters.')]; + } else if ($salt === "Rooraenietu8Eeyoemail($output); + + /* + * These settings are dangerous and break both the transparency and potential introduce sync issues + if (!Configure::read('Security.hide_organisation_index_from_users')) { + $output['MISP'][] = [ + 'hint', + __('Any user can see list of all organisations. You can disable that by setting `Security.hide_organisation_index_from_users` to `true`. %s', $server->serverSettings['Security']['hide_organisation_index_from_users']['description']), + ]; + } + if (!Configure::read('Security.hide_organisations_in_sharing_groups')) { + $output['MISP'][] = [ + 'hint', + __('Any user can see list of all organisations in sharing group that user can see. You can disable that by setting `Security.hide_organisations_in_sharing_groups` to `true`. %s', $server->serverSettings['Security']['hide_organisations_in_sharing_groups']['description']), + ]; + } + */ + + if (!$systemOnly) { + $this->feeds($output); + $this->remoteServers($output); + } + + try { + $cakeVersion = $this->getCakeVersion(); + if (version_compare($cakeVersion, '2.10.21', '<')) { + $output['Dependencies'][] = ['warning', __('CakePHP version %s is outdated.', $cakeVersion)]; + } + } catch (RuntimeException $e) { + } + + if (version_compare(PHP_VERSION, '7.4.0', '<')) { + $output['PHP'][] = [ + 'warning', + __('PHP version %s is not supported anymore. It can be still supported by your distribution.', PHP_VERSION), + 'https://www.php.net/supported-versions.php' + ]; + } + + if (ini_get('expose_php')) { + $output['PHP'][] = [ + 'hint', + __('PHP `expose_php` setting is enabled. That means that PHP version will be send in `X-Powered-By` header. This can help attackers.'), + ]; + } + + if (extension_loaded('xdebug')) { + $output['PHP'][] = [ + 'error', + __('The xdebug extension can reveal code and data to an attacker.'), + ]; + } + + if (ini_get('session.use_strict_mode') != 1) { + $output['PHP'][] = [ + 'warning', + __('Session strict mode is disabled.'), + 'https://www.php.net/manual/en/session.configuration.php#ini.session.use-strict-mode', + ]; + } + if (empty(ini_get('session.cookie_httponly'))) { + $output['PHP'][] = ['error', __('Session cookie is not set as HTTP only. Session cookie can be accessed from JavaScript.')]; + } + if (!in_array(strtolower(ini_get('session.cookie_samesite')), ['strict', 'lax'])) { + $output['PHP'][] = [ + 'error', + __('Session cookie SameSite parameter is not defined or set to None.'), + 'https://developer.mozilla.org/en-us/docs/Web/HTTP/Headers/Set-Cookie/SameSite', + ]; + } + $sidLength = ini_get('session.sid_length'); + if ($sidLength !== false && $sidLength < 32) { + $output['PHP'][] = [ + 'warning', + __('Session ID length is set to %s, at least 32 is recommended.', $sidLength), + 'https://www.php.net/manual/en/session.configuration.php#ini.session.sid-length', + ]; + } + $sidBits = ini_get('session.sid_bits_per_character'); + if ($sidBits !== false && $sidBits <= 4) { + $output['PHP'][] = [ + 'warning', + __('Session ID bit per character is set to %s, at least 5 is recommended.', $sidBits), + 'https://www.php.net/manual/en/session.configuration.php#ini.session.sid-bits-per-character', + ]; + } + + $this->system($output); + + return $output; + } + + /** + * @return array|string[][] + * @throws Exception + */ + public function tlsConnections() + { + $urls = [ + 'TLSv1.0' => ['url' => 'https://tls-v1-0.badssl.com:1010/'], + 'TLSv1.1' => ['url' => 'https://tls-v1-1.badssl.com:1011/'], + 'TLSv1.2' => ['url' => 'https://tls-v1-2.badssl.com:1012/', 'expected' => true], + 'TLSv1.3' => [ + 'url' => 'https://check-tls.akamai.io/v1/tlsinfo.json', + 'expected' => true, + 'process' => function (HttpSocketHttpException $response) { + return $response->getResponse()->getJson()['tls_version'] === 'tls1.3'; + } + ], + 'DH480' => ['url' => 'https://dh480.badssl.com/', 'expected' => false], + 'DH512' => ['url' => 'https://dh512.badssl.com/', 'expected' => false], + 'DH1024' => ['url' => 'https://dh1024.badssl.com/', 'expected' => false], + 'DH2048' => ['url' => 'https://dh2048.badssl.com/'], + 'RC4-MD5' => ['url' => 'https://rc4-md5.badssl.com/', 'expected' => false], + 'RC4' => ['url' => 'https://rc4.badssl.com/', 'expected' => false], + '3DES' => ['url' => 'https://3des.badssl.com/', 'expected' => false], + 'NULL' => ['url' => 'https://null.badssl.com/', 'expected' => false], + 'SHA1 2016' => ['url' => 'https://sha1-2016.badssl.com/', 'expected' => false], + 'SHA1 2017' => ['url' => 'https://sha1-2017.badssl.com/', 'expected' => false], + 'SHA1 intermediate' => ['url' => 'https://sha1-intermediate.badssl.com/', 'expected' => false], + 'Invalid expected sct' => ['url' => 'https://invalid-expected-sct.badssl.com/', 'expected' => false], + 'Expired' => ['url' => 'https://expired.badssl.com/', 'expected' => false], + 'Wrong host' => ['url' => 'https://wrong.host.badssl.com/', 'expect' => false], + 'Self-signed' => ['url' => 'https://self-signed.badssl.com/', 'expected' => false], + 'Untrusted-root' => ['url' => 'https://untrusted-root.badssl.com/', 'expected' => false], + 'Revoked' => ['url' => 'https://revoked.badssl.com/'], + 'Pinning test' => ['url' => 'https://pinning-test.badssl.com/'], + 'Bad DNSSEC' => ['url' => 'http://rhybar.cz', 'expected' => false], + ]; + foreach ($urls as &$details) { + $httpSocket = new HttpTool(); + try { + $response = $httpSocket->get($details['url']); + if (isset($details['process'])) { + $details['success'] = $details['process']($response); + } else { + $details['success'] = true; + } + } catch (Exception $e) { + $details['success'] = false; + $details['exception'] = $e; + } + } + return $urls; + } + + private function feeds(array &$output) + { + /** @var Feed $feed */ + $FeedsTable = $this->fetchTable('Feeds'); + $enabledFeeds = $FeedsTable->find('list', [ + 'conditions' => [ + 'input_source' => 'network', + 'OR' => [ + 'enabled' => true, + 'caching_enabled' => true, + ] + ], + 'fields' => ['name', 'url'], + ]); + foreach ($enabledFeeds as $feedName => $feedUrl) { + if (substr($feedUrl, 0, strlen('http://')) === 'http://') { + $output['Feeds'][] = ['warning', __('Feed %s uses insecure (HTTP) connection.', $feedName)]; + } + } + } + + private function remoteServers(array &$output) + { + $ServersTable = $this->fetchTable('Servers'); + + $enabledServers = $ServersTable->find('all', [ + 'conditions' => ['OR' => [ + 'push' => true, + 'pull' => true, + 'push_sightings' => true, + 'caching_enabled' => true, + ]], + 'fields' => ['id', 'name', 'url', 'self_signed', 'cert_file', 'client_cert_file'], + ]); + foreach ($enabledServers as $enabledServer) { + if (substr($enabledServer['Server']['url'], 0, strlen('http://')) === 'http://') { + $output['Remote servers'][] = ['warning', __('Server %s uses insecure (HTTP) connection.', $enabledServer['Server']['name'])]; + } else if ($enabledServer['Server']['self_signed']) { + $output['Remote servers'][] = ['warning', __('Server %s uses self signed certificate. This is considered insecure.', $enabledServer['Server']['name'])]; + } + + try { + $parsed = HttpTool::getServerClientCertificateInfo($enabledServer); + if (isset($parsed['public_key_size_ok']) && !$parsed['public_key_size_ok']) { + $algo = $parsed['public_key_type'] . " " . $parsed['public_key_size']; + $output['Remote servers'][] = ['warning', __('Server %s uses weak client certificate (%s).', $enabledServer['Server']['name'], $algo)]; + } + } catch (Exception $e) { + } + + try { + $parsed = HttpTool::getServerCaCertificateInfo($enabledServer); + if (isset($parsed['public_key_size_ok']) && !$parsed['public_key_size_ok']) { + $algo = $parsed['public_key_type'] . " " . $parsed['public_key_size']; + $output['Remote servers'][] = ['warning', __('Server %s uses weak CA certificate (%s).', $enabledServer['Server']['name'], $algo)]; + } + } catch (Exception $e) { + } + } + } + + private function email(array &$output) + { + $canSignPgp = Configure::read('GnuPG.sign'); + $canSignSmime = Configure::read('SMIME.enabled') && + !empty(Configure::read('SMIME.cert_public_sign')) && + !empty(Configure::read('SMIME.key_sign')); + + if (!$canSignPgp && !$canSignSmime) { + $output['Email'][] = [ + 'warning', + __('Email signing (PGP or S/MIME) is not enabled.') + ]; + } + + if ($canSignPgp) { + $gpgKeyPassword = Configure::read('GnuPG.password'); + if (empty($gpgKeyPassword)) { + $output['Email'][] = ['error', __('PGP private key password is empty.')]; + } else if (strlen($gpgKeyPassword) < self::STRONG_PASSWORD_LENGTH) { + $output['Email'][] = ['warning', __('PGP private key password is too short, should be at least %s chars long.', self::STRONG_PASSWORD_LENGTH)]; + } + } + + if (!Configure::read('GnuPG.bodyonlyencrypted')) { + $output['Email'][] = [ + 'hint', + __('Full email body with all event information will be sent, even without encryption.') + ]; + } + + if ($canSignPgp && !Configure::read('GnuPG.obscure_subject')) { + $output['Email'][] = [ + 'hint', + __('Even for encrypted emails, the email subject will be sent unencrypted. You can change that behaviour by setting `GnuPG.obscure_subject` to `true`.'), + ]; + } + + $email = new CakeEmail(); + $emailConfig = $email->config(); + if ($emailConfig['transport'] === 'Smtp' && $emailConfig['port'] == 25 && empty($emailConfig['tls'])) { + $output['Email'][] = [ + 'warning', + __('STARTTLS is not enabled.'), + 'https://en.wikipedia.org/wiki/Opportunistic_TLS', + ]; + } + } + + private function system(array &$output) + { + $kernelBuildTime = $this->getKernelBuild(); + if ($kernelBuildTime) { + $diff = (new Chronos())->diff($kernelBuildTime); + $diffDays = $diff->format('a'); + if ($diffDays > 300) { + $output['System'][] = [ + 'warning', + __('Kernel build time was %s days ago. This usually means that the system kernel is not updated.', $diffDays), + ]; + } + } + + // uptime + try { + $since = ProcessTool::execute(['uptime', '-s']); + $since = new Chronos($since); + $diff = (new Chronos())->diff($since); + $diffDays = $diff->format('a'); + if ($diffDays > 100) { + $output['System'][] = [ + 'warning', + __('Uptime of this server is %s days. This usually means that the system kernel is outdated.', $diffDays), + ]; + } + } catch (Exception $e) { + } + + // Python version + try { + $pythonVersion = ProcessTool::execute([ProcessTool::pythonBin(), '-V']); + $parts = explode(' ', $pythonVersion); + if ($parts[0] !== 'Python') { + throw new Exception("Invalid python version response: $pythonVersion"); + } + + if (version_compare($parts[1], '3.6', '<')) { + $output['System'][] = [ + 'warning', + __('You are using Python %s. This version is not supported anymore, but it can be still supported by your distribution.', $parts[1]), + 'https://endoflife.date/python', + ]; + } else if (version_compare($parts[1], '3.7', '<')) { + $output['System'][] = [ + 'hint', + __('You are using Python %s. This version will not be supported beyond 23 Dec 2021, but it can be that it is still supported by your distribution.', $parts[1]), + 'https://endoflife.date/python', + ]; + } + } catch (Exception $e) { + } + + $linuxVersion = $this->getLinuxVersion(); + if ($linuxVersion) { + list($name, $version) = $linuxVersion; + if ($name === 'Ubuntu') { + if (in_array($version, ['14.04', '16.04', '19.10', '20.10', '21.04', '21.10'], true)) { + $output['System'][] = [ + 'warning', + __('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $version), + 'https://endoflife.date/ubuntu', + ]; + } + } else if ($name === 'CentOS Linux' && $version == 8) { + $output['System'][] = [ + 'warning', + __('You are using CentOS 8. This version doesn\'t receive security support anymore. Please migrate to CentOS 8 Stream.'), + 'https://endoflife.date/centos', + ]; + } + } + } + + /** + * @return ChronosInterface|false + */ + private function getKernelBuild() + { + if (PHP_OS !== 'Linux') { + return false; + } + $version = php_uname('v'); + if (substr($version, 0, 7) !== '#1 SMP ') { + return false; + } + try { + return new Chronos('@' . substr($version, 7)); + } catch (Exception $e) { + return false; + } + } + + /** + * @return array|false + */ + private function getLinuxVersion() + { + if (PHP_OS !== 'Linux') { + return false; + } + if (!is_readable('/etc/os-release')) { + return false; + } + $content = file_get_contents('/etc/os-release'); + if ($content === false) { + return false; + } + $parsed = parse_ini_string($content); + if ($parsed === false) { + return false; + } + if (!isset($parsed['NAME']) || !isset($parsed['VERSION_ID'])) { + return false; + } + return [$parsed['NAME'], $parsed['VERSION_ID']]; + } + + /** + * @return string + */ + private function getCakeVersion() + { + $filePath = CAKE_CORE_INCLUDE_PATH . '/Cake/VERSION.txt'; + $version = file_get_contents($filePath); + if (!$version) { + throw new RuntimeException("Could not open CakePHP version file '$filePath'."); + } + foreach (explode("\n", $version) as $line) { + if ($line[0] === '/') { + continue; + } + return trim($line); + } + throw new RuntimeException("CakePHP version not found in file '$filePath'."); + } +} diff --git a/src/Lib/Tools/ServerSyncTool.php b/src/Lib/Tools/ServerSyncTool.php index b6310ba12..28ac33b91 100644 --- a/src/Lib/Tools/ServerSyncTool.php +++ b/src/Lib/Tools/ServerSyncTool.php @@ -3,9 +3,16 @@ namespace App\Lib\Tools; use App\Http\Exception\HttpSocketHttpException; +use Cake\Core\Configure; +use Cake\Http\Client\Response; +use Cake\ORM\Locator\LocatorAwareTrait; +use Exception; +use InvalidArgumentException; class ServerSyncTool { + use LocatorAwareTrait; + const FEATURE_BR = 'br', FEATURE_GZIP = 'gzip', FEATURE_ORG_RULE = 'org_rule', @@ -24,7 +31,7 @@ class ServerSyncTool /** @var array */ private $request; - /** @var HttpSocketExtended */ + /** @var HttpTool */ private $socket; /** @var CryptographicKey */ @@ -41,15 +48,17 @@ class ServerSyncTool */ public function __construct(array $server, array $request) { - if (!isset($server['Server'])) { + if (!isset($server)) { throw new InvalidArgumentException("Invalid server provided."); } $this->server = $server; $this->request = $request; - $syncTool = new SyncTool(); - $this->socket = $syncTool->setupHttpSocket($server); + $HttpTool = new HttpTool(); + $HttpTool->configFromServer($server); + + $this->socket = $HttpTool; } /** @@ -60,14 +69,14 @@ class ServerSyncTool */ public function eventExists(array $event) { - $url = $this->server['Server']['url'] . '/events/view/' . $event['Event']['uuid']; + $url = $this->server['url'] . '/events/view/' . $event['uuid']; $start = microtime(true); $exists = $this->socket->head($url, [], $this->request); $this->log($start, 'HEAD', $url, $exists); - if ($exists->code == '404') { + if ($exists->getStatusCode() == '404') { return false; } - if ($exists->code == '200') { + if ($exists->getStatusCode() == '200') { return true; } throw new HttpSocketHttpException($exists, $url); @@ -135,7 +144,7 @@ class ServerSyncTool // There is bug in MISP API, that returns response code 404 with Location if event already exists } else if ($e->getResponse()->getHeader('Location')) { - $urlPath = $e->getResponse()->getHeader('Location'); + $urlPath = $e->getResponse()->getHeader('Location')[0]; $pieces = explode('/', $urlPath); $lastPart = end($pieces); return $this->updateEvent($event, $lastPart); @@ -153,7 +162,7 @@ class ServerSyncTool */ public function createEvent(array $event) { - $logMessage = "Pushing Event #{$event['Event']['id']} to Server #{$this->serverId()}"; + $logMessage = "Pushing Event #{$event['id']} to Server #{$this->serverId()}"; return $this->post("/events/add/metadata:1", $event, $logMessage); } @@ -167,9 +176,9 @@ class ServerSyncTool public function updateEvent(array $event, $eventId = null) { if ($eventId === null) { - $eventId = $event['Event']['uuid']; + $eventId = $event['uuid']; } - $logMessage = "Pushing Event #{$event['Event']['id']} to Server #{$this->serverId()}"; + $logMessage = "Pushing Event #{$event['id']} to Server #{$this->serverId()}"; return $this->post("/events/edit/$eventId/metadata:1", $event, $logMessage); } @@ -239,12 +248,15 @@ class ServerSyncTool */ public function fetchSightingsForEvents(array $eventUuids) { - return $this->post('/sightings/restSearch/event', [ - 'returnFormat' => 'json', - 'last' => 0, // fetch all - 'includeUuid' => true, - 'uuid' => $eventUuids, - ])->json()['response']; + return $this->post( + '/sightings/restSearch/event', + [ + 'returnFormat' => 'json', + 'last' => 0, // fetch all + 'includeUuid' => true, + 'uuid' => $eventUuids, + ] + )->getJson()['response']; } /** @@ -260,8 +272,8 @@ class ServerSyncTool return []; } - $response = $this->post('/sightings/filterSightingUuidsForPush/' . $event['Event']['uuid'], $sightingUuids); - return $response->json(); + $response = $this->post('/sightings/filterSightingUuidsForPush/' . $event['uuid'], $sightingUuids); + return $response->getJson(); } /** @@ -304,7 +316,7 @@ class ServerSyncTool } $response = $this->get('/servers/getVersion'); - $info = $response->json(); + $info = $response->getJson(); if (!isset($info['version'])) { throw new Exception("Invalid response when fetching server version: `version` field missing."); } @@ -354,7 +366,15 @@ class ServerSyncTool */ public function serverId() { - return $this->server['Server']['id']; + return $this->server['id']; + } + + /** + * @return string + */ + public function serverName() + { + return $this->server['name']; } /** @@ -362,7 +382,7 @@ class ServerSyncTool */ public function pullRules() { - return $this->decodeRule('pull_rules'); + return $this->server['pull_rules']; } /** @@ -370,7 +390,7 @@ class ServerSyncTool */ public function pushRules() { - return $this->decodeRule('push_rules'); + return $this->server['push_rules']; } /** @@ -431,7 +451,7 @@ class ServerSyncTool */ private function get($url) { - $url = $this->server['Server']['url'] . $url; + $url = $this->server['url'] . $url; $start = microtime(true); $response = $this->socket->get($url, [], $this->request); $this->log($start, 'GET', $url, $response); @@ -453,7 +473,7 @@ class ServerSyncTool */ private function post($url, $data, $logMessage = null, $etag = null) { - $protectedMode = !empty($data['Event']['protected']); + $protectedMode = !empty($data['protected']); $data = JsonTool::encode($data); if ($logMessage && !empty(Configure::read('Security.sync_audit'))) { @@ -491,11 +511,11 @@ class ServerSyncTool $data = gzencode($data, 1); } } - $url = $this->server['Server']['url'] . $url; + $url = $this->server['url'] . $url; $start = microtime(true); $response = $this->socket->post($url, $data, $request); $this->log($start, 'POST', $url, $response); - if ($etag && $response->isNotModified()) { + if ($etag && $response->getStatusCode() === 304) { return $response; // if etag was provided and response code is 304, it is valid response } if (!$response->isOk()) { @@ -516,7 +536,7 @@ class ServerSyncTool } if (!$this->cryptographicKey) { - $this->cryptographicKey = ClassRegistry::init('CryptographicKey'); + $this->cryptographicKey = $this->fetchTable('CryptographicKeys'); } $signature = $this->cryptographicKey->signWithInstanceKey($data); if (empty($signature)) { @@ -525,16 +545,6 @@ class ServerSyncTool return base64_encode($signature); } - /** - * @param string $key - * @return array - */ - private function decodeRule($key) - { - $rules = $this->server['Server'][$key]; - return json_decode($rules, true); - } - /** * @param array $params * @return string @@ -558,14 +568,17 @@ class ServerSyncTool * @param float $start Microtime when request was send * @param string $method HTTP method * @param string $url - * @param HttpSocketResponse $response + * @param Response $response */ - private function log($start, $method, $url, HttpSocketResponse $response) + private function log($start, $method, $url, Response $response) { $duration = round(microtime(true) - $start, 3); - $responseSize = strlen($response->body); - $ce = $response->getHeader('Content-Encoding'); - $logEntry = '[' . date('Y-m-d H:i:s', intval($start)) . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n"; - file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX); + $responseSize = strlen($response->getBody()); + $ce = ''; + if ($response->getHeader('Content-Encoding')) { + $ce = $response->getHeader('Content-Encoding')[0]; + } + $logEntry = '[' . date('Y-m-d H:i:s', intval($start)) . "] \"$method $url\" {$response->getStatusCode()} $responseSize $duration $ce\n"; + file_put_contents(APP . '../logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX); } } diff --git a/src/Model/Behavior/JsonFieldsBehavior.php b/src/Model/Behavior/JsonFieldsBehavior.php index 43ecaccef..40827c548 100644 --- a/src/Model/Behavior/JsonFieldsBehavior.php +++ b/src/Model/Behavior/JsonFieldsBehavior.php @@ -25,8 +25,10 @@ class JsonFieldsBehavior extends Behavior $config = $this->getConfig(); foreach ($config['fields'] as $field => $fieldConfig) { - if (!$entity->has($field) && array_key_exists('default', $fieldConfig)) { + if (!isset($data[$field]) && array_key_exists('default', $fieldConfig)) { $entity->set($field, $fieldConfig['default']); + } else { + $entity->set($field, $data[$field] ?? []); } } } @@ -44,6 +46,17 @@ class JsonFieldsBehavior extends Behavior } } + public function afterSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + $config = $this->getConfig(); + + foreach ($config['fields'] as $field => $fieldConfig) { + if ($entity[$field] !== null) { + $entity[$field] = JsonTool::decode($entity[$field]); + } + } + } + public function beforeFind(EventInterface $event, Query $query, ArrayObject $options) { $config = $this->getConfig(); diff --git a/src/Model/Entity/Event.php b/src/Model/Entity/Event.php index fb25e861e..4efe7f9fe 100644 --- a/src/Model/Entity/Event.php +++ b/src/Model/Entity/Event.php @@ -12,4 +12,117 @@ class Event extends AppModel public const ANALYSIS_LEVELS = [ 0 => 'Initial', 1 => 'Ongoing', 2 => 'Completed' ]; + + /** + * @return array[] + */ + public static function exportTypes() + { + return [ + 'json' => [ + 'extension' => '.json', + 'type' => 'JSON', + 'scope' => 'Event', + 'requiresPublished' => 0, + 'params' => ['includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'json'], + 'description' => __('Click this to download all events and attributes that you have access to in MISP JSON format.'), + ], + 'xml' => [ + 'extension' => '.xml', + 'type' => 'XML', + 'scope' => 'Event', + 'params' => ['includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'xml'], + 'requiresPublished' => 0, + 'description' => __('Click this to download all events and attributes that you have access to in MISP XML format.'), + ], + 'csv_sig' => [ + 'extension' => '.csv', + 'type' => 'CSV_Sig', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => ['published' => 1, 'to_ids' => 1, 'returnFormat' => 'csv'], + 'description' => __('Click this to download all attributes that are indicators and that you have access to (except file attachments) in CSV format.'), + ], + 'csv_all' => [ + 'extension' => '.csv', + 'type' => 'CSV_All', + 'scope' => 'Event', + 'requiresPublished' => 0, + 'params' => ['ignore' => 1, 'returnFormat' => 'csv'], + 'description' => __('Click this to download all attributes that you have access to (except file attachments) in CSV format.'), + ], + 'suricata' => [ + 'extension' => '.rules', + 'type' => 'Suricata', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'suricata'], + 'description' => __('Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ], + 'snort' => [ + 'extension' => '.rules', + 'type' => 'Snort', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'snort'], + 'description' => __('Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ], + 'bro' => [ + 'extension' => '.intel', + 'type' => 'Bro', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'bro'], + 'description' => __('Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ], + 'stix' => [ + 'extension' => '.xml', + 'type' => 'STIX', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'stix', 'includeAttachments' => 1], + 'description' => __('Click this to download a STIX document containing the STIX version of all events and attributes that you have access to.') + ], + 'stix2' => [ + 'extension' => '.json', + 'type' => 'STIX2', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'stix2', 'includeAttachments' => 1], + 'description' => __('Click this to download a STIX2 document containing the STIX2 version of all events and attributes that you have access to.') + ], + 'rpz' => [ + 'extension' => '.txt', + 'type' => 'RPZ', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'rpz'], + 'description' => __('Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.') + ], + 'text' => [ + 'extension' => '.txt', + 'type' => 'TEXT', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'text', 'includeAttachments' => 1], + 'description' => __('Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.') + ], + 'yara' => [ + 'extension' => '.yara', + 'type' => 'Yara', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'yara'], + 'description' => __('Click this to download Yara rules generated from all relevant attributes.') + ], + 'yara-json' => [ + 'extension' => '.json', + 'type' => 'Yara', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => ['returnFormat' => 'yara-json'], + 'description' => __('Click this to download Yara rules generated from all relevant attributes. Rules are returned in a JSON format with information about origin (generated or parsed) and validity.') + ], + ]; + } } diff --git a/src/Model/Entity/Server.php b/src/Model/Entity/Server.php index 444dee7d8..0f0be8d98 100644 --- a/src/Model/Entity/Server.php +++ b/src/Model/Entity/Server.php @@ -2,13 +2,8 @@ namespace App\Model\Entity; -use App\Lib\Tools\BackgroundJobsTool; -use App\Lib\Tools\BetterSecurity; -use App\Lib\Tools\EncryptedValue; -use App\Lib\Tools\RedisTool; use App\Model\Entity\AppModel; use Cake\ORM\Locator\LocatorAwareTrait; -use Exception; class Server extends AppModel { @@ -18,2817 +13,39 @@ class Server extends AppModel SETTING_RECOMMENDED = 1, SETTING_OPTIONAL = 2; - public function &__get(string $field) - { - if ($field === 'serverSettings') { - return $this->serverSettings = $this->generateServerSettings(); - } else if ($field === 'command_line_functions') { - return $this->command_line_functions = $this->generateCommandLineFunctions(); - } - return parent::__get($field); - } - - /** - * Generate just when required - * @return array[] - */ - private function generateServerSettings() - { - return [ - 'MISP' => [ - 'branch' => 1, - 'baseurl' => [ - 'level' => 0, - 'description' => __('The base url of the application (in the format https://www.mymispinstance.com or https://myserver.com/misp). Several features depend on this setting being correctly set to function.'), - 'value' => '', - 'errorMessage' => __('The currently set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'), - 'test' => 'testBaseURL', - 'type' => 'string', - 'null' => true - ], - 'external_baseurl' => [ - 'level' => 0, - 'description' => __('The base url of the application (in the format https://www.mymispinstance.com) as visible externally/by other MISPs. MISP will encode this URL in sharing groups when including itself. If this value is not set, the baseurl is used as a fallback.'), - 'value' => '', - 'test' => 'testURL', - 'type' => 'string', - ], - 'live' => [ - 'level' => 0, - 'description' => __('Unless set to true, the instance will only be accessible by site admins.'), - 'value' => false, - 'test' => 'testLive', - 'type' => 'boolean', - ], - 'language' => [ - 'level' => 0, - 'description' => __('Select the language MISP should use. The default is english.'), - 'value' => 'eng', - 'test' => 'testLanguage', - 'type' => 'string', - 'optionsSource' => function () { - return $this->loadAvailableLanguages(); - }, - 'afterHook' => 'cleanCacheFiles' - ], - 'default_attribute_memory_coefficient' => [ - 'level' => 1, - 'description' => __('This values controls the internal fetcher\'s memory envelope when it comes to attributes. The number provided is the amount of attributes that can be loaded for each MB of PHP memory available in one shot. Consider lowering this number if your instance has a lot of attribute tags / attribute galaxies attached.'), - 'value' => 80, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ], - 'default_event_memory_divisor' => [ - 'level' => 1, - 'description' => __('This value controls the divisor for attribute weighting when it comes to loading full events. Meaning that it will load coefficient / divisor number of attributes per MB of memory available. Consider raising this number if you have a lot of correlations or highly contextualised events (large number of event level galaxies/tags).'), - 'value' => 3, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ], - 'disable_event_locks' => [ - 'level' => 1, - 'description' => __('Disable the event locks that are executed periodically when a user browses an event view. It can be useful to leave event locks enabled to warn users that someone else is editing the same event, but generally it\'s extremely verbose and can cause issues in certain setups, so it\'s recommended to disable this.'), - 'value' => false, - 'test' => 'testBoolTrue', - 'type' => 'boolean', - 'null' => true - ], - 'correlation_engine' => [ - 'level' => 0, - 'description' => __('Choose which correlation engine to use. MISP defaults to the default engine, maintaining all data in the database whilst enforcing ACL rules on any non site-admin user. This is recommended for any MISP instnace with multiple organisations. If you are an endpoint MISP, consider switching to the much leaner and faster No ACL engine.'), - 'value' => 'default', - 'test' => 'testForCorrelationEngine', - 'type' => 'string', - 'null' => true, - 'options' => [ - 'Default' => __('Default Correlation Engine'), - 'NoAcl' => __('No ACL Engine') - ], - ], - 'correlation_limit' => [ - 'level' => 0, - 'description' => __('Set a value for the maximum number of correlations a value should have before MISP will refuse to correlate it (extremely over-correlating values are rarely useful from a correlation perspective).'), - 'value' => 100, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ], - 'enable_advanced_correlations' => [ - 'level' => 0, - 'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'server_settings_skip_backup_rotate' => [ - 'level' => 1, - 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'python_bin' => [ - 'level' => 1, - 'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT), - 'value' => false, - 'null' => false, - 'test' => 'testForBinExec', - 'beforeHook' => 'beforeHookBinExec', - 'type' => 'string', - 'cli_only' => 1 - ], - 'ca_path' => [ - 'level' => 1, - 'description' => __('MISP will default to the bundled mozilla certificate bundle shipped with the framework, which is rather stale. If you wish to use an alternate bundle, just set this setting using the path to the bundle to use. This setting can only be modified via the CLI.'), - 'value' => APP . 'Lib/cakephp/lib/Cake/Config/cacert.pem', - 'null' => true, - 'test' => 'testForCABundle', - 'type' => 'string', - 'cli_only' => 1 - ], - 'disable_auto_logout' => [ - 'level' => 1, - 'description' => __('In some cases, a heavily used MISP instance can generate unwanted blackhole errors due to a high number of requests hitting the server. Disable the auto logout functionality to ease the burden on the system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'ssdeep_correlation_threshold' => [ - 'level' => 1, - 'description' => __('Set the ssdeep score at which to consider two ssdeep hashes as correlating [1-100]'), - 'value' => 40, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'max_correlations_per_event' => [ - 'level' => 1, - 'description' => __('Sets the maximum number of correlations that can be fetched with a single event. For extreme edge cases this can prevent memory issues. The default value is 5k.'), - 'value' => 5000, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ], - 'maintenance_message' => [ - 'level' => 2, - 'description' => __('The message that users will see if the instance is not live.'), - 'value' => 'Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at $email.', - 'errorMessage' => __('If this is not set the default value will be used.'), - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'name' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'version' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'disable_cached_exports' => [ - 'level' => 1, - 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), - 'value' => false, - 'null' => true, - 'test' => 'testDisableCache', - 'type' => 'boolean', - 'afterHook' => 'disableCacheAfterHook', - ], - 'disable_threat_level' => [ - 'level' => 1, - 'description' => __('Disable displaying / modifications to the threat level altogether on the instance (deprecated field).'), - 'value' => false, - 'null' => true, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'header' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footermidleft' => [ - 'level' => 2, - 'description' => __('Footer text prepending the "Powered by MISP" text.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footermidright' => [ - 'level' => 2, - 'description' => __('Footer text following the "Powered by MISP" text.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footerpart1' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footerpart2' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footer' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footerversion' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'footer_logo' => [ - 'level' => 2, - 'description' => __('If set, this setting allows you to display a logo on the right side of the footer. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ], - 'home_logo' => [ - 'level' => 2, - 'description' => __('If set, this setting allows you to display a logo as the home icon. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ], - 'main_logo' => [ - 'level' => 2, - 'description' => __('If set, the image specified here will replace the main MISP logo on the login screen. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ], - 'org' => [ - 'level' => 1, - 'description' => __('The organisation tag of the hosting organisation. This is used in the e-mail subjects.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'host_org_id' => [ - 'level' => 0, - 'description' => __('The hosting organisation of this instance. If this is not selected then replication instances cannot be added.'), - 'value' => '0', - 'test' => 'testLocalOrgStrict', - 'type' => 'numeric', - 'optionsSource' => function () { - return $this->loadLocalOrganisations(true); - }, - ], - 'uuid' => [ - 'level' => 0, - 'description' => __('The MISP instance UUID. This UUID is used to identify this instance.'), - 'value' => '0', - 'errorMessage' => __('No valid UUID set'), - 'test' => 'testUuid', - 'type' => 'string' - ], - 'logo' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'showorg' => [ - 'level' => 0, - 'description' => __('Setting this setting to \'false\' will hide all organisation names / logos.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'threatlevel_in_email_subject' => [ - 'level' => 2, - 'description' => __('Put the event threat level in the notification E-mail subject.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'email_subject_TLP_string' => [ - 'level' => 2, - 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), - 'value' => 'tlp:amber', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'email_subject_tag' => [ - 'level' => 2, - 'description' => __('If this tag is set on an event it\'s value will be sent in the E-mail subject. If the tag is not set the email_subject_TLP_string will be used.'), - 'value' => 'tlp', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'email_subject_include_tag_name' => [ - 'level' => 2, - 'description' => __('Include in name of the email_subject_tag in the subject. When false only the tag value is used.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'email_from_name' => [ - 'level' => 2, - 'description' => __('Notification e-mail sender name.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'taxii_sync' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'taxii_client_path' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'background_jobs' => [ - 'level' => 1, - 'description' => __('Enables the use of MISP\'s background processing.'), - 'value' => '', - 'test' => 'testBoolTrue', - 'type' => 'boolean', - ], - 'attachments_dir' => [ - 'level' => 2, - 'description' => __('Directory where attachments are stored. MISP will NOT migrate the existing data if you change this setting. The only safe way to change this setting is in config.php, when MISP is not running, and after having moved/copied the existing data to the new location. This directory must already exist and be writable and readable by the MISP application.'), - 'value' => APP . '/files', # GUI display purpose only. - 'null' => false, - 'test' => 'testForWritableDir', - 'type' => 'string', - 'cli_only' => 1 - ], - 'download_attachments_on_load' => [ - 'level' => 2, - 'description' => __('Always download attachments when loaded by a user in a browser'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'osuser' => [ - 'level' => 0, - 'description' => __('The Unix user MISP (php) is running as'), - 'value' => 'www-data', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'email' => [ - 'level' => 0, - 'description' => __('The e-mail address that MISP should use for all notifications'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'disable_emailing' => [ - 'level' => 0, - 'description' => __('You can disable all e-mailing using this setting. When enabled, no outgoing e-mails will be sent by MISP.'), - 'value' => false, - 'null' => true, - 'test' => 'testDisableEmail', - 'type' => 'boolean', - ], - 'publish_alerts_summary_only' => [ - 'level' => 1, - 'description' => __('This setting is deprecated. Please use `MISP.event_alert_metadata_only` instead.'), - 'value' => false, - 'null' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'contact' => [ - 'level' => 1, - 'description' => __('The e-mail address that MISP should include as a contact address for the instance\'s support team.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'dns' => [ - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'cveurl' => [ - 'level' => 1, - 'description' => __('Turn Vulnerability type attributes into links linking to the provided CVE lookup'), - 'value' => 'https://cve.circl.lu/cve/', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'cweurl' => [ - 'level' => 1, - 'description' => __('Turn Weakness type attributes into links linking to the provided CWE lookup'), - 'value' => 'https://cve.circl.lu/cwe/', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'disablerestalert' => [ - 'level' => 1, - 'description' => __('This setting controls whether notification e-mails will be sent when an event is created via the REST interface. It might be a good idea to disable this setting when first setting up a link to another instance to avoid spamming your users during the initial pull. Quick recap: True = Emails are NOT sent, False = Emails are sent on events published via sync / REST.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'extended_alert_subject' => [ - 'level' => 1, - 'description' => __('Enabling this flag will allow the event description to be transmitted in the alert e-mail\'s subject. Be aware that this is not encrypted by GnuPG, so only enable it if you accept that part of the event description will be sent out in clear-text.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'forceHTTPSforPreLoginRequestedURL' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('If enabled, any requested URL before login will have their HTTP part replaced by HTTPS. This can be usefull if MISP is running behind a reverse proxy responsible for SSL and communicating unencrypted with MISP.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'event_alert_metadata_only' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Send just event metadata (attributes and objects will be omitted) for event alert.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'default_event_distribution' => [ - 'level' => 0, - 'description' => __('The default distribution setting for events (0-3).'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => [ - '0' => __('Your organisation only'), - '1' => __('This community only'), - '2' => __('Connected communities'), - '3' => __('All communities') - ], - ], - 'default_attribute_distribution' => [ - 'level' => 0, - 'description' => __('The default distribution setting for attributes, set it to \'event\' if you would like the attributes to default to the event distribution level. (0-3 or "event")'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => [ - '0' => __('Your organisation only'), - '1' => __('This community only'), - '2' => __('Connected communities'), - '3' => __('All communities'), - 'event' => __('Inherit from event') - ], - ], - 'default_event_threat_level' => [ - 'level' => 1, - 'description' => __('The default threat level setting when creating events.'), - 'value' => 4, - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => ['1' => 'High', '2' => 'Medium', '3' => 'Low', '4' => 'undefined'], - ], - 'default_event_tag_collection' => [ - 'level' => 0, - 'description' => __('The tag collection to be applied to all events created manually.'), - 'value' => 0, - 'test' => 'testTagCollections', - 'type' => 'numeric', - 'optionsSource' => function () { - return $this->loadTagCollections(); - } - ], - 'default_publish_alert' => [ - 'level' => 0, - 'description' => __('The default setting for publish alerts when creating users.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'tagging' => [ - 'level' => 1, - 'description' => __('Enable the tagging feature of MISP. This is highly recommended.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'full_tags_on_event_index' => [ - 'level' => 2, - 'description' => __('Show the full tag names on the event index.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => [0 => 'Minimal tags', 1 => 'Full tags', 2 => 'Shortened tags'], - ], - 'disable_taxonomy_consistency_checks' => [ - 'level' => 0, - 'description' => __('*WARNING* This will disable taxonomy tags conflict checks when browsing attributes and objects, does not impact checks when adding tags. It can dramatically increase the performance when loading events with lots of tagged attributes or objects.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'welcome_text_top' => [ - 'level' => 2, - 'description' => __('Used on the login page, before the MISP logo'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'welcome_text_bottom' => [ - 'level' => 2, - 'description' => __('Used on the login page, after the MISP logo'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'welcome_logo' => [ - 'level' => 2, - 'description' => __('Used on the login page, to the left of the MISP logo, upload it as a custom image in the file management tool.'), - 'value' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ], - 'welcome_logo2' => [ - 'level' => 2, - 'description' => __('Used on the login page, to the right of the MISP logo, upload it as a custom image in the file management tool.'), - 'value' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ], - 'title_text' => [ - 'level' => 2, - 'description' => __('Used in the page title, after the name of the page'), - 'value' => 'MISP', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'take_ownership_xml_import' => [ - 'level' => 2, - 'description' => __('Allows users to take ownership of an event uploaded via the "Add MISP XML" button. This allows spoofing the creator of a manually imported event, also breaking possibly breaking the original intended releasability. Synchronising with an instance that has a different creator for the same event can lead to unwanted consequences.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'terms_download' => [ - 'level' => 2, - 'description' => __('Choose whether the terms and conditions should be displayed inline (false) or offered as a download (true)'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'terms_file' => [ - 'level' => 2, - 'description' => __('The filename of the terms and conditions file. Make sure that the file is located in your MISP/app/files/terms directory'), - 'value' => '', - 'test' => 'testForTermsFile', - 'type' => 'string' - ], - 'showorgalternate' => [ - 'level' => 2, - 'description' => __('True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'unpublishedprivate' => [ - 'level' => 2, - 'description' => __('True will deny access to unpublished events to users outside the organization of the submitter except site admins.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'newUserText' => [ - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent to the user after account creation (has to be sent manually from the administration interface). Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $org = the organisation that the instance belongs to, as set in MISP.org, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), - 'value' => 'Dear new MISP user,\n\nWe would hereby like to welcome you to the $org MISP community.\n\n Use the credentials below to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nPassword: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'test' => 'testPasswordResetText', - 'type' => 'string' - ], - 'passwordResetText' => [ - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent to the users when a password reset is triggered. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), - 'value' => 'Dear MISP user,\n\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nYour temporary password: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'test' => 'testPasswordResetText', - 'type' => 'string' - ], - 'enableEventBlocklisting' => [ - 'level' => 1, - 'description' => __('Since version 2.3.107 you can start blocklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'), - 'value' => true, - 'type' => 'boolean', - 'test' => 'testBool' - ], - 'enableOrgBlocklisting' => [ - 'level' => 1, - 'description' => __('Blocklisting organisation UUIDs to prevent the creation of any event created by the blocklisted organisation.'), - 'value' => true, - 'type' => 'boolean', - 'test' => 'testBool' - ], - 'log_client_ip' => [ - 'level' => 1, - 'description' => __('If enabled, all log entries will include the IP address of the user.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'beforeHook' => 'ipLogBeforeHook' - ], - 'log_client_ip_header' => [ - 'level' => 1, - 'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy in front of your MISP instance. Prepend the variable with "HTTP_", for example "HTTP_X_FORWARDED_FOR".'), - 'value' => 'REMOTE_ADDR', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ], - 'store_api_access_time' => [ - 'level' => 1, - 'description' => __('If enabled, MISP will capture the last API access time following a successful authentication using API keys, stored against a user under the last_api_access field.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'log_auth' => [ - 'level' => 1, - 'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'log_skip_db_logs_completely' => [ - 'level' => 0, - 'description' => __('This functionality allows you to completely disable any logs from being saved in your SQL backend. This is HIGHLY advised against, you lose all the functionalities provided by the audit log subsystem along with the event history (as these are built based on the logs on the fly). Only enable this if you understand and accept the associated risks.'), - 'value' => false, - 'errorMessage' => __('Logging has now been disabled - your audit logs will not capture failed authentication attempts, your event history logs are not being populated and no system maintenance messages are being logged.'), - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'log_skip_access_logs_in_application_logs' => [ - 'level' => 0, - 'description' => __('Skip adding the access log entries to the /logs/ application logs. This is **HIGHLY** recommended as your instance will be logging these entries twice otherwise, however, for compatibility reasons for auditing we maintain this behaviour until confirmed otherwise.'), - 'value' => false, - 'errorMessage' => __('Access logs are logged twice. This is generally not recommended, make sure you update your tooling.'), - 'test' => 'testBoolTrue', - 'type' => 'boolean', - 'null' => true - ], - 'log_paranoid' => [ - 'level' => 0, - 'description' => __('If this functionality is enabled all page requests will be logged. Keep in mind this is extremely verbose and will become a burden to your database.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'log_paranoid_api' => [ - 'level' => 0, - 'description' => __('If this functionality is enabled all API requests will be logged.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'log_paranoid_skip_db' => [ - 'level' => 0, - 'description' => __('You can decide to skip the logging of the paranoid logs to the database. Logs will be just published to ZMQ or Kafka.'), - 'value' => false, - 'test' => 'testParanoidSkipDb', - 'type' => 'boolean', - 'null' => true - ], - 'log_paranoid_include_post_body' => [ - 'level' => 0, - 'description' => __('If paranoid logging is enabled, include the POST body in the entries.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'log_paranoid_include_sql_queries' => [ - 'level' => 0, - 'description' => __('If paranoid logging is enabled, include the SQL queries in the entries.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'log_user_ips' => [ - 'level' => 0, - 'description' => __('Log user IPs on each request. 30 day retention for lookups by IP to get the last authenticated user ID for the given IP, whilst on the reverse, indefinitely stores all associated IPs for a user ID.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'log_user_ips_authkeys' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('Log user IP and key usage on each API request. All logs for given keys are deleted after one year when this key is not used.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'log_new_audit' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('Enable new audit log system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'log_new_audit_compress' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Compress log changes by brotli algorithm. This will reduce log database size.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'delegation' => [ - 'level' => 1, - 'description' => __('This feature allows users to create org only events and ask another organisation to take ownership of the event. This allows organisations to remain anonymous by asking a partner to publish an event for them.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'discussion_disable' => [ - 'level' => 1, - 'description' => __('Completely disable ability for user to add discussion to events.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'showCorrelationsOnIndex' => [ - 'level' => 1, - 'description' => __('When enabled, the number of correlations visible to the currently logged in user will be visible on the event index UI. This comes at a performance cost but can be very useful to see correlating events at a glance.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'showProposalsCountOnIndex' => [ - 'level' => 1, - 'description' => __('When enabled, the number of proposals for the events are shown on the index.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'showSightingsCountOnIndex' => [ - 'level' => 1, - 'description' => __('When enabled, the aggregate number of attribute sightings within the event becomes visible to the currently logged in user on the event index UI.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'showDiscussionsCountOnIndex' => [ - 'level' => 1, - 'description' => __('When enabled, the aggregate number of discussion posts for the event becomes visible to the currently logged in user on the event index UI.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'showEventReportCountOnIndex' => [ - 'level' => 1, - 'description' => __('When enabled, the aggregate number of event reports for the event becomes visible to the currently logged in user on the event index UI.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'disableUserSelfManagement' => [ - 'level' => 1, - 'description' => __('When enabled only Org and Site admins can edit a user\'s profile.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'disable_user_login_change' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled only Site admins can change user email. This should be enabled if you manage user logins by external system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'disable_user_password_change' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled only Site admins can change user password. This should be enabled if you manage user passwords by external system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'disable_user_add' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled, Org Admins could not add new users. This should be enabled if you manage users by external system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'block_event_alert' => [ - 'level' => 1, - 'description' => __('Enable this setting to start blocking alert e-mails for events with a certain tag. Define the tag in MISP.block_event_alert_tag.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'block_event_alert_tag' => [ - 'level' => 1, - 'description' => __('If the MISP.block_event_alert setting is set, alert e-mails for events tagged with the tag defined by this setting will be blocked.'), - 'value' => 'no-alerts="true"', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => false, - ], - 'event_alert_republish_ban' => [ - 'level' => 1, - 'description' => __('Enable this setting to start blocking alert e-mails for events that have already been published since a specified amount of time. This threshold is defined by MISP.event_alert_republish_ban_threshold'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'event_alert_republish_ban_threshold' => [ - 'level' => 1, - 'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control how long no alerting by email will be done. Expected format: integer, in minutes'), - 'value' => 5, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ], - 'event_alert_republish_ban_refresh_on_retry' => [ - 'level' => 1, - 'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control if a ban time should be reset if emails are tried to be sent during the ban.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'user_email_notification_ban' => [ - 'level' => 1, - 'description' => __('Enable this setting to start blocking users to send too many e-mails notification since a specified amount of time. This threshold is defined by MISP.user_email_notification_ban_threshold'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'user_email_notification_ban_time_threshold' => [ - 'level' => 1, - 'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how long no notification by email will be done. Expected format: integer, in minutes'), - 'value' => 120, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ], - 'user_email_notification_ban_amount_threshold' => [ - 'level' => 1, - 'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how many notification by email can be send for the timeframe defined in MISP.user_email_notification_ban_time_threshold. Expected format: integer'), - 'value' => 10, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ], - 'org_alert_threshold' => [ - 'level' => 1, - 'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'), - 'value' => 0, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true, - ], - 'block_old_event_alert' => [ - 'level' => 1, - 'description' => __('Enable this setting to start blocking alert e-mails for old events. The exact timing of what constitutes an old event is defined by MISP.block_old_event_alert_age.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'block_old_event_alert_age' => [ - 'level' => 1, - 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control how old an event can be for it to be alerted on. The "timestamp" field of the event is used. Expected format: integer, in days'), - 'value' => false, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ], - 'block_old_event_alert_by_date' => [ - 'level' => 1, - 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control the threshold for the event.date field, indicating how old an event can be for it to be alerted on. The "date" field of the event is used. Expected format: integer, in days'), - 'value' => false, - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ], - 'tmpdir' => [ - 'level' => 1, - 'description' => __('Please indicate the temp directory you wish to use for certain functionalities in MISP. By default this is set to %s and will be used among others to store certain temporary files extracted from imports during the import process.', APP . 'tmp'), - 'value' => APP . 'tmp', - 'test' => 'testForPath', - 'type' => 'string', - 'null' => true, - 'cli_only' => 1 - ], - 'custom_css' => [ - 'level' => 2, - 'description' => __('If you would like to customise the CSS, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'), - 'value' => '', - 'test' => 'testForStyleFile', - 'type' => 'string', - 'null' => true, - ], - 'proposals_block_attributes' => [ - 'level' => 0, - 'description' => __('Enable this setting to allow blocking attributes from to_ids sensitive exports if a proposal has been made to it to remove the IDS flag or to remove the attribute altogether. This is a powerful tool to deal with false-positives efficiently.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ], - 'incoming_tags_disabled_by_default' => [ - 'level' => 1, - 'description' => __('Enable this settings if new tags synced / added via incoming events from any source should not be selectable by users by default.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false - ], - 'completely_disable_correlation' => [ - 'level' => 0, - 'description' => __('*WARNING* This setting will completely disable the correlation on this instance and remove any existing saved correlations. Enabling this will trigger a full recorrelation of all data which is an extremely long and costly procedure. Only enable this if you know what you\'re doing.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true, - 'afterHook' => 'correlationAfterHook', - ], - 'allow_disabling_correlation' => [ - 'level' => 0, - 'description' => __('*WARNING* This setting will give event creators the possibility to disable the correlation of individual events / attributes that they have created.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'redis_host' => [ - 'level' => 0, - 'description' => __('The host running the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), - 'value' => '127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'redis_port' => [ - 'level' => 0, - 'description' => __('The port used by the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), - 'value' => 6379, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'redis_database' => [ - 'level' => 0, - 'description' => __('The database on the redis server to be used for generic MISP tasks. If you run more than one MISP instance, please make sure to use a different database on each instance.'), - 'value' => 13, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'redis_password' => [ - 'level' => 0, - 'description' => __('The password on the redis server (if any) to be used for generic MISP tasks.'), - 'value' => '', - 'test' => null, - 'type' => 'string', - 'redacted' => true - ], - 'redis_serializer' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Redis serializer method. WARNING: Changing this setting will drop some cached data.'), - 'value' => 'JSON', - 'test' => null, - 'type' => 'string', - 'null' => true, - 'options' => [ - 'JSON' => 'JSON', - 'igbinary' => 'igbinary', - ], - 'afterHook' => function () { - $keysToDelete = ['taxonomies_cache:*', 'misp:warninglist_cache', 'misp:wlc:*', 'misp:event_lock:*', 'misp:event_index:*', 'misp:dashboard:*']; - RedisTool::deleteKeysByPattern(RedisTool::init(), $keysToDelete); - return true; - }, - ], - 'event_view_filter_fields' => [ - 'level' => 2, - 'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'), - 'value' => 'id, uuid, value, comment, type, category, Tag.name', - 'test' => null, - 'type' => 'string', - ], - 'manage_workers' => [ - 'level' => 2, - 'description' => __('Set this to false if you would like to disable MISP managing its own worker processes (for example, if you are managing the workers with a systemd unit).'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'deadlock_avoidance' => [ - 'level' => 1, - 'description' => __('Only enable this if you have some tools using MISP with extreme high concurency. General performance will be lower as normal as certain transactional queries are avoided in favour of shorter table locks.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'updateTimeThreshold' => [ - 'level' => 1, - 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'), - 'value' => '7200', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ], - 'attribute_filters_block_only' => [ - 'level' => 1, - 'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'attachment_scan_module' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'), - 'value' => '', - 'type' => 'string', - 'null' => true, - ], - 'attachment_scan_hash_only' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'attachment_scan_timeout' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('How long to wait for scan results in seconds.'), - 'value' => 30, - 'test' => 'testForPositiveInteger', - 'type' => 'numeric', - 'null' => true, - ], - 'warning_for_all' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('Enable warning list triggers regardless of the IDS flag value.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'system_setting_db' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('Enable storing setting in database.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => true, - ], - 'menu_custom_right_link' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Custom right menu URL.'), - 'value' => null, - 'type' => 'string', - 'null' => true, - ], - 'menu_custom_right_link_html' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Custom right menu text (it is possible to use HTML).'), - 'value' => null, - 'type' => 'string', - 'null' => true, - ], - 'enable_synchronisation_filtering_on_type' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Allows server synchronisation connections to be filtered on Attribute type or Object name. Warning: This feature can potentially cause your synchronisation partners to receive incomplete versions of the events you are propagating on behalf of others. This means that even if they would be receiving the unfiltered version through another instance, your filtered version might be the one they receive on a first-come-first-serve basis.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true, - ], - 'download_gpg_from_homedir' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Fetch GPG instance key from GPG homedir.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => true, - ], - 'enable_clusters_mirroring_from_attributes_to_event' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Add a checkbox when attaching a cluster to an Attribute which, when checked, will also create the same clusters on the attribute\'s event.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true, - ], - 'thumbnail_in_redis' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Store image thumbnails in Redis instead of file system.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'self_update' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => true, - ], - 'online_version_check' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('Enable the online MISP version check when loading the Diagnostics page.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => true, - ], - ], - 'GnuPG' => [ - 'branch' => 1, - 'binary' => [ - 'level' => 2, - 'description' => __('The location of the GnuPG executable. If you would like to use a different GnuPG executable than /usr/bin/gpg, you can set it here. If the default is fine, just keep the setting suggested by MISP.'), - 'value' => '/usr/bin/gpg', - 'test' => 'testForGPGBinary', - 'type' => 'string', - 'cli_only' => 1 - ], - 'onlyencrypted' => [ - 'level' => 0, - 'description' => __('Allow (false) unencrypted e-mails to be sent to users that don\'t have a GnuPG key.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'bodyonlyencrypted' => [ - 'level' => 2, - 'description' => __('Allow (false) the body of unencrypted e-mails to contain details about the event.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'sign' => [ - 'level' => 2, - 'description' => __('Enable the signing of GnuPG emails. By default, GnuPG emails are signed'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'email' => [ - 'level' => 0, - 'description' => __('The e-mail address that the instance\'s GnuPG key is tied to.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'password' => [ - 'level' => 1, - 'description' => __('The password (if it is set) of the GnuPG key of the instance.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'redacted' => true - ], - 'homedir' => [ - 'level' => 0, - 'description' => __('The location of the GnuPG homedir.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'obscure_subject' => [ - 'level' => 2, - 'description' => __('When enabled, the subject in signed and encrypted e-mails will not be sent in unencrypted form.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'key_fetching_disabled' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('When disabled, user could not fetch his PGP key from CIRCL key server. Key fetching requires internet connection.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - ], - 'SMIME' => [ - 'branch' => 1, - 'enabled' => [ - 'level' => 2, - 'description' => __('Enable S/MIME encryption. The encryption posture of the GnuPG.onlyencrypted and GnuPG.bodyonlyencrypted settings are inherited if S/MIME is enabled.'), - 'value' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'email' => [ - 'level' => 2, - 'description' => __('The e-mail address that the instance\'s S/MIME key is tied to.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'cert_public_sign' => [ - 'level' => 2, - 'description' => __('The location of the public half of the signing certificate.'), - 'value' => '/var/www/MISP/.smime/email@address.com.pem', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'key_sign' => [ - 'level' => 2, - 'description' => __('The location of the private half of the signing certificate.'), - 'value' => '/var/www/MISP/.smime/email@address.com.key', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'password' => [ - 'level' => 2, - 'description' => __('The password (if it is set) of the S/MIME key of the instance.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'redacted' => true - ], - ], - 'Proxy' => [ - 'branch' => 1, - 'host' => [ - 'level' => 2, - 'description' => __('The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'port' => [ - 'level' => 2, - 'description' => __('The TCP port for the HTTP proxy.'), - 'value' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - ], - 'method' => [ - 'level' => 2, - 'description' => __('The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'user' => [ - 'level' => 2, - 'description' => __('The authentication username for the HTTP proxy.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'password' => [ - 'level' => 2, - 'description' => __('The authentication password for the HTTP proxy.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'redacted' => true - ], - ], - 'Security' => [ - 'branch' => 1, - 'disable_form_security' => [ - 'level' => 0, - 'description' => __('Disabling this setting will remove all form tampering protection. Do not set this setting pretty much ever. You were warned.'), - 'value' => false, - 'errorMessage' => 'This setting leaves your users open to CSRF attacks. Please consider disabling this setting.', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'csp_enforce' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('Enforce CSP. Content Security Policy (CSP) is an added layer of security that helps to detect and mitigate certain types of attacks, including Cross Site Scripting (XSS) and data injection attacks. When disabled, violations will be just logged.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'salt' => [ - 'level' => 0, - 'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'), - 'value' => '', - 'test' => 'testSalt', - 'type' => 'string', - 'editable' => false, - 'redacted' => true - ], - 'log_each_individual_auth_fail' => [ - 'level' => 1, - 'description' => __('By default API authentication failures that happen within the same hour for the same key are omitted and a single log entry is generated. This allows administrators to more easily keep track of attackers that try to brute force API authentication, by reducing the noise generated by expired API keys. On the other hand, this makes little sense for internal MISP instances where detecting the misconfiguration of tools becomes more interesting, so if you fall into the latter category, enable this feature.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'advanced_authkeys' => [ - 'level' => 0, - 'description' => __('Advanced authkeys will allow each user to create and manage a set of authkeys for themselves, each with individual expirations and comments. API keys are stored in a hashed state and can no longer be recovered from MISP. Users will be prompted to note down their key when creating a new authkey. You can generate a new set of API keys for all users on demand in the diagnostics page, or by triggering %s.', sprintf('%s', $this->baseurl, __('the advanced upgrade'))), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'advanced_authkeys_validity' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Maximal key lifetime in days. Use can limit that validity even more. Just newly created keys will be affected. When not set, key validity is not limited.'), - 'value' => '', - 'type' => 'numeric', - 'test' => 'testForNumeric', - 'null' => true, - ], - 'authkey_keep_session' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('When enabled, session is kept between API requests.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'auth_enforced' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('This optionally can be enabled if an external auth provider is used. When set to true, it will disable the default form authentication.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'rest_client_enable_arbitrary_urls' => [ - 'level' => 0, - 'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => 1 - ], - 'rest_client_baseurl' => [ - 'level' => 1, - 'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'), - 'value' => false, - 'test' => null, - 'type' => 'string' - ], - 'syslog' => [ - 'level' => 0, - 'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'syslog_to_stderr' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Write syslog messages also to standard error output.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'syslog_ident' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Syslog message identifier.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'do_not_log_authkeys' => [ - 'level' => 0, - 'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'mandate_ip_allowlist_advanced_authkeys' => [ - 'level' => 2, - 'description' => __('If enabled, setting an ip allowlist will be mandatory when adding or editing an advanced authkey.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'disable_browser_cache' => [ - 'level' => 0, - 'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'check_sec_fetch_site_header' => [ - 'level' => 0, - 'description' => __('If enabled, any POST, PUT or AJAX request will be allow just when Sec-Fetch-Site header is not defined or contains "same-origin".'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'force_https' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('If enabled, MISP server will consider all requests as secure. This is usually useful when you run MISP behind reverse proxy that terminates HTTPS.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'otp_required' => [ - 'level' => 2, - 'description' => __('Require authentication with OTP. Users that do not have (T/H)OTP configured will be forced to create a token at first login. You cannot use it in combination with external authentication plugins.'), - 'value' => false, - 'test' => 'testBool', - 'beforeHook' => 'otpBeforeHook', - 'type' => 'boolean', - 'null' => true - ], - 'otp_issuer' => [ - 'level' => 2, - 'description' => __('If OTP is enabled, set the issuer string to an arbitrary value. Otherwise, MISP will default to "[MISP.org] MISP".'), - 'value' => false, - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'email_otp_enabled' => [ - 'level' => 2, - 'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'), - 'value' => false, - 'test' => 'testBool', - 'beforeHook' => 'email_otpBeforeHook', - 'type' => 'boolean', - 'null' => true - ], - 'email_otp_length' => [ - 'level' => 2, - 'description' => __('Define the length of the OTP code sent by email'), - 'value' => '6', - 'type' => 'numeric', - 'test' => 'testForNumeric', - 'null' => true, - ], - 'email_otp_validity' => [ - 'level' => 2, - 'description' => __('Define the validity (in minutes) of the OTP code sent by email'), - 'value' => '5', - 'type' => 'numeric', - 'test' => 'testForNumeric', - 'null' => true, - ], - 'email_otp_text' => [ - 'level' => 2, - 'bigField' => true, - 'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'), - 'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ], - 'email_otp_exceptions' => [ - 'level' => 2, - 'bigField' => true, - 'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ], - 'allow_self_registration' => [ - 'level' => 1, - 'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'allow_password_forgotten' => [ - 'level' => 1, - 'description' => __('Enabling this setting will allow users to request automated password reset tokens via mail and initiate a reset themselves. Users with no encryption keys will not be able to use this feature.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'self_registration_message' => [ - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent shown to anyone trying to self-register.'), - 'value' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.', - 'test' => false, - 'type' => 'string' - ], - 'password_policy_length' => [ - 'level' => 2, - 'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'), - 'value' => '12', - 'test' => 'testPasswordLength', - 'type' => 'numeric', - ], - 'password_policy_complexity' => [ - 'level' => 2, - 'description' => __('Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Default (simple 3 out of 4 or minimum 16 characters): /^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/'), - 'value' => '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/', - 'test' => 'testPasswordRegex', - 'type' => 'string', - ], - 'require_password_confirmation' => [ - 'level' => 1, - 'description' => __('Enabling this setting will require users to submit their current password on any edits to their profile (including a triggered password change). For administrators, the confirmation will be required when changing the profile of any user. Could potentially mitigate an attacker trying to change a compromised user\'s password in order to establish persistance, however, enabling this feature will be highly annoying to users.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'sanitise_attribute_on_delete' => [ - 'level' => 1, - 'description' => __('Enabling this setting will sanitise the contents of an attribute on a soft delete'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'hide_organisation_index_from_users' => [ - 'level' => 1, - 'description' => __('Enabling this setting will block the organisation index from being visible to anyone besides site administrators on the current instance. Keep in mind that users can still see organisations that produce data via events, proposals, event history log entries, etc.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'hide_organisations_in_sharing_groups' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('Enabling this setting will block the organisation list from being visible in sharing group besides user with sharing group permission.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'disable_local_feed_access' => [ - 'level' => 0, - 'description' => __('Disabling this setting will allow the creation/modification of local feeds (as opposed to network feeds). Enabling this setting will restrict feed sources to be network based only. When disabled, keep in mind that a malicious site administrator could get access to any arbitrary file on the system that the apache user has access to. Make sure that proper safe-guards are in place. This setting can only be modified via the CLI.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => 1 - ], - 'allow_unsafe_apikey_named_param' => [ - 'level' => 0, - 'description' => __('Allows passing the API key via the named url parameter "apikey" - highly recommended not to enable this, but if you have some dodgy legacy tools that cannot pass the authorization header it can work as a workaround. Again, only use this as a last resort.'), - 'value' => false, - 'errorMessage' => __('You have enabled the passing of API keys via URL parameters. This is highly recommended against, do you really want to reveal APIkeys in your logs?...'), - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'allow_cors' => [ - 'level' => 1, - 'description' => __('Allow cross-origin requests to this instance, matching origins given in Security.cors_origins. Set to false to totally disable'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'cors_origins' => [ - 'level' => 1, - 'description' => __('Set the origins from which MISP will allow cross-origin requests. Useful for external integration. Comma seperate if you need more than one.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'sync_audit' => [ - 'level' => 1, - 'description' => __('Enable this setting to create verbose logs of synced event data for debugging reasons. Logs are saved in your MISP directory\'s app/files/scripts/tmp/ directory.'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ], - 'user_monitoring_enabled' => [ - 'level' => 1, - 'description' => __('Enables the functionality to monitor users - thereby enabling all logging functionalities for a single user. This functionality is intrusive and potentially heavy on the system - use it with care.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'username_in_response_header' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('When enabled, logged in username will be included in X-Username HTTP response header. This is useful for request logging on webserver/proxy side.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'encryption_key' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Encryption key used to store sensitive data (like authkeys) in database encrypted. If empty, data are stored unencrypted. Requires PHP 7.1 or newer.'), - 'value' => '', - 'test' => function ($value) { - if (strlen($value) < 32) { - return __('Encryption key must be at least 32 chars long.'); - } - return true; - }, - // $table->behaviors()->has('EncryptedFields'); - 'afterHook' => function ($setting, $new, $old) { - // LATER change code to automatically search for xxTables with EncryptedFieldsBehalvior and re-encrypte all the keys using changeKey($old) of the Behavior - // although at first sight this is complex and requires filesystem listings - /** @var SystemSetting $systemSetting */ - $systemSetting = $this->fetchTable('SystemSettings'); - $systemSetting->changeKey($old); - - /** Server */ - $this->changeKey($old); - - /** @var Cerebrate $cerebrate */ - $cerebrate = $this->fetchTable('Cerebrates'); - $cerebrate->changeKey($old); - return true; - }, - 'type' => 'string', - 'null' => true, - 'cli_only' => true, - 'redacted' => true, - ], - 'min_tls_version' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Minimal required TLS version when connecting to external resources.'), - 'value' => '', - 'type' => 'string', - 'null' => true, - 'options' => [ - '' => __('All versions'), - 'tlsv1_0' => 'TLSv1.0', - 'tlsv1_1' => 'TLSv1.1', - 'tlsv1_2' => 'TLSv1.2', - 'tlsv1_3' => 'TLSv1.3', - ], - ], - 'enable_svg_logos' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('When enabled, organisations logos in svg format are allowed.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'disable_instance_file_uploads' => [ - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled, the "Manage files" menu is disabled on the server settings. You can still copy files via ssh to the appropriate location and link them using MISP.settings.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => true - ], - 'disclose_user_emails' => [ - 'level' => 0, - 'description' => __('Enable this setting to allow for the user e-mail addresses to be shown to non site-admin users. Keep in mind that in broad communities this can be abused.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - ], - 'SecureAuth' => [ - 'branch' => 1, - 'amount' => [ - 'level' => 0, - 'description' => __('The number of tries a user can try to login and fail before the bruteforce protection kicks in.'), - 'value' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - ], - 'expire' => [ - 'level' => 0, - 'description' => __('The duration (in seconds) of how long the user will be locked out when the allowed number of login attempts are exhausted.'), - 'value' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - ], - ], - 'Session' => [ - 'branch' => 1, - 'autoRegenerate' => [ - 'level' => 0, - 'description' => __('Set to true to automatically regenerate sessions after x number of requests. This might lead to the user getting de-authenticated and is frustrating in general, so only enable it if you really need to regenerate sessions. (Not recommended)'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - ], - 'checkAgent' => [ - 'level' => 0, - 'description' => __('Set to true to check for the user agent string in each request. This can lead to occasional logouts (not recommended).'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - ], - 'defaults' => [ - 'level' => 0, - 'description' => __('The session type used by MISP. The default setting is php, which will use the session settings configured in php.ini for the session data (supported options: php, database). The recommended option is php and setting your PHP up to use redis sessions via your php.ini. Just add \'session.save_handler = redis\' and "session.save_path = \'tcp://localhost:6379\'" (replace the latter with your redis connection) to '), - 'value' => '', - 'test' => 'testForSessionDefaults', - 'type' => 'string', - 'options' => ['php' => 'php', 'database' => 'database', 'cake' => 'cake', 'cache' => 'cache'], - ], - 'timeout' => [ - 'level' => 0, - 'description' => __('The timeout duration of sessions (in MINUTES). 0 does not mean infinite for the PHP session handler, instead sessions will invalidate immediately.'), - 'value' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'cookieTimeout' => [ - 'level' => 0, - 'description' => __('The expiration of the cookie (in MINUTES). The session timeout gets refreshed frequently, however the cookies do not. Generally it is recommended to have a much higher cookie_timeout than timeout.'), - 'value' => '', - 'test' => 'testForCookieTimeout', - 'type' => 'numeric' - ] - ], - 'Plugin' => [ - 'branch' => 1, - 'RPZ_policy' => [ - 'level' => 2, - 'description' => __('The default policy action for the values added to the RPZ.'), - 'value' => 1, - 'test' => 'testForRPZBehaviour', - 'type' => 'numeric', - 'options' => [0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only'], - ], - 'RPZ_walled_garden' => [ - 'level' => 2, - 'description' => __('The default walled garden used by the RPZ export if the Local-Data policy setting is picked for the export.'), - 'value' => '127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'RPZ_serial' => [ - 'level' => 2, - 'description' => __('The serial in the SOA portion of the zone file. (numeric, best practice is yyyymmddrr where rr is the two digit sub-revision of the file. $date will automatically get converted to the current yyyymmdd, so $date00 is a valid setting). Setting it to $time will give you an unixtime-based serial (good then you need more than 99 revisions per day).'), - 'value' => '$date00', - 'test' => 'testForRPZSerial', - 'type' => 'string', - ], - 'RPZ_refresh' => [ - 'level' => 2, - 'description' => __('The refresh specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '2h', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ], - 'RPZ_retry' => [ - 'level' => 2, - 'description' => __('The retry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '30m', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ], - 'RPZ_expiry' => [ - 'level' => 2, - 'description' => __('The expiry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '30d', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ], - 'RPZ_minimum_ttl' => [ - 'level' => 2, - 'description' => __('The minimum TTL specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '1h', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ], - 'RPZ_ttl' => [ - 'level' => 2, - 'description' => __('The TTL of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '1w', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ], - 'RPZ_ns' => [ - 'level' => 2, - 'description' => __('Nameserver'), - 'value' => 'localhost.', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'RPZ_ns_alt' => [ - 'level' => 2, - 'description' => __('Alternate nameserver'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'RPZ_email' => [ - 'level' => 2, - 'description' => __('The e-mail address specified in the SOA portion of the zone file.'), - 'value' => 'root.localhost', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'Kafka_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'Kafka_brokers' => [ - 'level' => 2, - 'description' => __('A comma separated list of Kafka bootstrap brokers'), - 'value' => 'kafka:9092', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'Kafka_rdkafka_config' => [ - 'level' => 2, - 'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'), - 'value' => '/etc/rdkafka.ini', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'Kafka_include_attachments' => [ - 'level' => 2, - 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_event_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_event_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing event creations/edits/deletions.'), - 'value' => 'misp_event', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_event_publish_notifications_enable' => [ - 'level' => 2, - 'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_event_publish_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing event information on publish.'), - 'value' => 'misp_event_publish', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_object_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_object_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing object creations/edits/deletions.'), - 'value' => 'misp_object', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_object_reference_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_object_reference_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing object reference creations/deletions.'), - 'value' => 'misp_object_reference', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_attribute_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_attribute_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing attribute creations/edits/soft deletions.'), - 'value' => 'misp_attribute', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_shadow_attribute_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_shadow_attribute_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing proposal creations/edits/deletions.'), - 'value' => 'misp_shadow_attribute', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_tag_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_tag_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => 'misp_tag', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_sighting_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new sightings.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_sighting_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing sightings.'), - 'value' => 'misp_sighting', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_user_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified users.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_user_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing new/modified users.'), - 'value' => 'misp_user', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_organisation_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified organisations.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_organisation_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing new/modified organisations.'), - 'value' => 'misp_organisation', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Kafka_audit_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Kafka_audit_notifications_topic' => [ - 'level' => 2, - 'description' => __('Topic for publishing log entries.'), - 'value' => 'misp_audit', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'ZeroMQ_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_host' => [ - 'level' => 2, - 'description' => __('The host that the pub/sub feature will use.'), - 'value' => '127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_port' => [ - 'level' => 2, - 'description' => __('The port that the pub/sub feature will use.'), - 'value' => 50000, - 'test' => 'testForZMQPortNumber', - 'type' => 'numeric', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_username' => [ - 'level' => 2, - 'description' => __('The username that client need to use to connect to ZeroMQ.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_password' => [ - 'level' => 2, - 'description' => __('The password that client need to use to connect to ZeroMQ.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - 'redacted' => true - ], - 'ZeroMQ_redis_host' => [ - 'level' => 2, - 'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'), - 'value' => 'localhost', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_redis_port' => [ - 'level' => 2, - 'description' => __('The port that Redis is listening on.'), - 'value' => 6379, - 'test' => 'testForPortNumber', - 'type' => 'numeric', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_redis_password' => [ - 'level' => 2, - 'description' => __('The password, if set for Redis.'), - 'value' => '', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - 'redacted' => true - ], - 'ZeroMQ_redis_database' => [ - 'level' => 2, - 'description' => __('The database to be used for queuing messages for the pub/sub functionality.'), - 'value' => 1, - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_redis_namespace' => [ - 'level' => 2, - 'description' => __('The namespace to be used for queuing messages for the pub/sub functionality.'), - 'value' => 'mispq', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ], - 'ZeroMQ_include_attachments' => [ - 'level' => 2, - 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_event_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_object_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_object_reference_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_attribute_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_tag_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_sighting_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new sightings to the ZMQ pubsub feed.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_user_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified users to the ZMQ pubsub feed.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_organisation_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified organisations to the ZMQ pubsub feed.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_audit_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of log entries to the ZMQ pubsub feed. Keep in mind, this can get pretty verbose depending on your logging settings.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ZeroMQ_warninglist_notifications_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified warninglist to the ZMQ pubsub feed.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ElasticSearch_logging_enable' => [ - 'level' => 2, - 'description' => __('Enabled logging to an ElasticSearch instance'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'ElasticSearch_connection_string' => [ - 'level' => 2, - 'description' => __('The URL(s) at which to access ElasticSearch - comma separate if you want to have more than one.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'ElasticSearch_log_index' => [ - 'level' => 2, - 'description' => __('The index in which to place logs'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_enable' => [ - 'level' => 2, - 'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'S3_aws_compatible' => [ - 'level' => 2, - 'description' => __('Use external AWS compatible system such as MinIO'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'S3_aws_ca' => [ - 'level' => 2, - 'description' => __('AWS TLS CA, set to empty to use CURL internal trusted certificates or path for custom trusted CA'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_aws_validate_ca' => [ - 'level' => 2, - 'description' => __('Validate CA'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'S3_aws_endpoint' => [ - 'level' => 2, - 'description' => __('Uses external AWS compatible endpoint such as MinIO'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_bucket_name' => [ - 'level' => 2, - 'description' => __('Bucket name to upload to, please make sure that the bucket exists. We will not create the bucket for you'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_region' => [ - 'level' => 2, - 'description' => __('Region in which your S3 bucket resides'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_aws_access_key' => [ - 'level' => 2, - 'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'S3_aws_secret_key' => [ - 'level' => 2, - 'description' => __('AWS secret key to use when uploading samples'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Sightings_policy' => [ - 'level' => 1, - 'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner organisation alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and any organisation that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'), - 'value' => 0, - 'type' => 'numeric', - 'options' => [ - 0 => __('Event Owner Organisation'), - 1 => __('Sighting reporters'), - 2 => __('Everyone'), - 3 => __('Event Owner + host org sightings'), - ], - ], - 'Sightings_anonymise' => [ - 'level' => 1, - 'description' => __('Enabling the anonymisation of sightings will simply aggregate all sightings instead of showing the organisations that have reported a sighting. Users will be able to tell the number of sightings their organisation has submitted and the number of sightings for other organisations'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'Sightings_anonymise_as' => [ - 'level' => 1, - 'description' => __('When pushing sightings to another server, report all sightings from this instance as this organisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'), - 'value' => '0', - 'test' => 'testLocalOrg', - 'type' => 'numeric', - 'optionsSource' => function () { - return $this->loadLocalOrganisations(); - }, - ], - 'Sightings_range' => [ - 'level' => 1, - 'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'), - 'value' => 365, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'Sightings_sighting_db_enable' => [ - 'level' => 1, - 'description' => __('Enable SightingDB integration.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Sightings_enable_realtime_publish' => [ - 'level' => 1, - 'description' => __('By default, sightings will not be immediately pushed to connected instances, as this can have a heavy impact on the performance of sighting attributes. Enable realtime publishing to trigger the publishing of sightings immediately as they are added.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'CustomAuth_enable' => [ - 'level' => 2, - 'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'beforeHook' => 'customAuthBeforeHook' - ], - 'CustomAuth_header' => [ - 'level' => 2, - 'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'), - 'value' => 'AUTHORIZATION', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_use_header_namespace' => [ - 'level' => 2, - 'description' => __('Use a header namespace for the auth header - default setting is enabled'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'CustomAuth_header_namespace' => [ - 'level' => 2, - 'description' => __('The default header namespace for the auth header - default setting is HTTP_'), - 'value' => 'HTTP_', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_required' => [ - 'level' => 2, - 'description' => __('If this setting is enabled then the only way to authenticate will be using the custom header. Alternatively, you can run in mixed mode that will log users in via the header if found, otherwise users will be redirected to the normal login page.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'CustomAuth_only_allow_source' => [ - 'level' => 2, - 'description' => __('If you are using an external tool to authenticate with MISP and would like to only allow the tool\'s url as a valid point of entry then set this field. '), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_name' => [ - 'level' => 2, - 'description' => __('The name of the authentication method, this is cosmetic only and will be shown on the user creation page and logs.'), - 'value' => 'External authentication', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_disable_logout' => [ - 'level' => 2, - 'description' => __('Disable the logout button for users authenticate with the external auth mechanism.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Enrichment_services_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the enrichment services'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Enrichment_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the enrichment services'), - 'value' => 10, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Import_services_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the import services'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Import_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the import services'), - 'value' => 10, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Import_services_url' => [ - 'level' => 1, - 'description' => __('The url used to access the import services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Import_services_port' => [ - 'level' => 1, - 'description' => __('The port used to access the import services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => '6666', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ], - 'Export_services_url' => [ - 'level' => 1, - 'description' => __('The url used to access the export services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Export_services_port' => [ - 'level' => 1, - 'description' => __('The port used to access the export services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => '6666', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ], - 'Export_services_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the export services'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Export_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the export services'), - 'value' => 10, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Action_services_url' => [ - 'level' => 1, - 'description' => __('The url used to access the action services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Action_services_port' => [ - 'level' => 1, - 'description' => __('The port used to access the action services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => '6666', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ], - 'Action_services_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the action services'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Action_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the action services'), - 'value' => 10, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Enrichment_hover_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Enrichment_hover_popover_only' => [ - 'level' => 0, - 'description' => __('When enabled, users have to click on the magnifier icon to show the enrichment'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Enrichment_hover_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the hover services'), - 'value' => 5, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Enrichment_services_url' => [ - 'level' => 1, - 'description' => __('The url used to access the enrichment services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Enrichment_services_port' => [ - 'level' => 1, - 'description' => __('The port used to access the enrichment services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => 6666, - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ], - 'Workflow_enable' => [ - 'level' => 1, - 'description' => __('Enable/disable workflow feature. [experimental]'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Workflow_debug_url' => [ - 'level' => 1, - 'description' => __('Set the debug URL where info about workflow execution will be POSTed'), - 'value' => 'http://127.0.0.1:27051', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Cortex_services_url' => [ - 'level' => 1, - 'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'), - 'value' => 'http://127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'Cortex_services_port' => [ - 'level' => 1, - 'description' => __('The port used to access Cortex. By default, this is port 9000'), - 'value' => 9000, - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ], - 'Cortex_services_enable' => [ - 'level' => 0, - 'description' => __('Enable/disable the Cortex services'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'Cortex_authkey' => [ - 'level' => 1, - 'description' => __('Set an authentication key to be passed to Cortex'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'Cortex_timeout' => [ - 'level' => 1, - 'description' => __('Set a timeout for the Cortex services'), - 'value' => 120, - 'test' => 'testForEmpty', - 'type' => 'numeric' - ], - 'Cortex_ssl_verify_peer' => [ - 'level' => 1, - 'description' => __('Set to false to disable SSL verification. This is not recommended.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'Cortex_ssl_verify_host' => [ - 'level' => 1, - 'description' => __('Set to false if you wish to ignore hostname match errors when validating certificates.'), - 'value' => true, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'Cortex_ssl_allow_self_signed' => [ - 'level' => 1, - 'description' => __('Set to true to enable self-signed certificates to be accepted. This requires Cortex_ssl_verify_peer to be enabled.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'Cortex_ssl_cafile' => [ - 'level' => 1, - 'description' => __('Set to the absolute path of the Certificate Authority file that you wish to use for verifying SSL certificates.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_custom_password_reset' => [ - 'level' => 2, - 'description' => __('Provide your custom authentication users with an external URL to the authentication system to reset their passwords.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CustomAuth_custom_logout' => [ - 'level' => 2, - 'description' => __('Provide a custom logout URL for your users that will log them out using the authentication system you use.'), - 'value' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ], - 'CyCat_enable' => [ - 'level' => 1, - 'description' => __('Enable lookups for additional relations via CyCat.'), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ], - 'CyCat_url' => [ - 'level' => 2, - 'description' => __('URL to use for CyCat lookups, if enabled.'), - 'value' => 'https://api.cycat.org', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ] - ], - 'SimpleBackgroundJobs' => [ - 'branch' => 1, - 'enabled' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('Enables or disables background jobs with Supervisor backend. Please read %s before setting this to `true`.', '' . __('this guide') . ''), - 'value' => false, - 'test' => 'testBool', - 'type' => 'boolean' - ], - 'redis_host' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The host running the redis server to be used for background jobs.'), - 'value' => '127.0.0.1', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'redis_port' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The port used by the redis server to be used for background jobs.'), - 'value' => 6379, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'redis_database' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The database on the redis server to be used for background jobs. If you run more than one MISP instance, please make sure to use a different database or redis_namespace on each instance.'), - 'value' => 1, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'redis_password' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The password on the redis server (if any) to be used for background jobs.'), - 'value' => '', - 'test' => null, - 'type' => 'string', - 'redacted' => true - ], - 'redis_namespace' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The namespace to be used for the background jobs related keys.'), - 'value' => 'background_jobs', - 'test' => null, - 'type' => 'string' - ], - 'redis_serializer' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Redis serializer method. WARNING: Changing this setting in production will break your jobs.'), - 'value' => 'JSON', - 'test' => null, - 'type' => 'string', - 'null' => true, - 'options' => [ - 'JSON' => 'JSON', - 'igbinary' => 'igbinary', - ], - 'afterHook' => function () { - BackgroundJobsTool::getInstance()->restartWorkers(); - return true; - }, - ], - 'max_job_history_ttl' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The time in seconds the job statuses history will be kept.'), - 'value' => 86400, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'supervisor_host' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The host where the Supervisor XML-RPC API is running.'), - 'value' => 'localhost', - 'test' => 'testForEmpty', - 'type' => 'string' - ], - 'supervisor_port' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The port where the Supervisor XML-RPC API is running.'), - 'value' => 9001, - 'test' => 'testForNumeric', - 'type' => 'numeric' - ], - 'supervisor_user' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The user of the Supervisor XML-RPC API.'), - 'value' => 'supervisor', - 'test' => null, - 'type' => 'string' - ], - 'supervisor_password' => [ - 'level' => self::SETTING_CRITICAL, - 'description' => __('The password of the Supervisor XML-RPC API.'), - 'value' => '', - 'test' => null, - 'type' => 'string', - 'redacted' => true - ], - ], - 'debug' => [ - 'level' => 0, - 'description' => __('The debug level of the instance, always use 0 for production instances.'), - 'value' => '', - 'test' => 'testDebug', - 'type' => 'numeric', - 'options' => [0 => 'Debug off', 1 => 'Debug on', 2 => 'Debug + SQL dump'], - ], - 'site_admin_debug' => [ - 'level' => 0, - 'description' => __('The debug level of the instance for site admins. This feature allows site admins to run debug mode on a live instance without exposing it to other users. The most verbose option of debug and site_admin_debug is used for site admins.'), - 'value' => '', - 'test' => 'testDebugAdmin', - 'type' => 'boolean', - 'null' => true - ], - 'LinOTPAuth' => [ - 'branch' => 1, - 'enabled' => [ - 'level' => 2, - 'description' => __('Enable / Disable LinOTP'), - 'value' => true, - 'type' => 'boolean', - ], - 'baseUrl' => [ - 'level' => 2, - 'description' => __('The default LinOTP URL.'), - 'value' => 'https://', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'realm' => [ - 'level' => 2, - 'description' => __('The LinOTP realm to authenticate against.'), - 'value' => 'lino', - 'test' => 'testForEmpty', - 'type' => 'string', - ], - 'verifyssl' => [ - 'level' => 2, - 'description' => __('Set to false to skip SSL/TLS verify'), - 'value' => true, - 'test' => 'testBoolTrue', - 'type' => 'boolean', - ], - 'mixedauth' => [ - 'level' => 2, - 'description' => __('Set to true to enforce OTP usage'), - 'value' => false, - 'test' => 'testBoolFalse', - 'type' => 'boolean', - ], - ], - ]; - } + public const SYNC_TEST_ERROR_CODES = [ + 2 => 'Server unreachable', + 3 => 'Unexpected error', + 4 => 'Authentication failed', + 5 => 'Password change required', + 6 => 'Terms not accepted' + ]; + public const ACTIONS_DESCRIPTIONS = [ + 'verifyGnuPGkeys' => [ + 'title' => 'Verify GnuPG keys', + 'description' => "Run a full validation of all GnuPG keys within this instance's userbase. The script will try to identify possible issues with each key and report back on the results.", + 'url' => '/users/verifyGPG/' + ], + 'databaseCleanupScripts' => [ + 'title' => 'Database Cleanup Scripts', + 'description' => 'If you run into an issue with an infinite upgrade loop (when upgrading from version ~2.4.50) that ends up filling your database with upgrade script log messages, run the following script.', + 'url' => '/logs/pruneUpdateLogs/' + ], + 'releaseUpdateLock' => [ + 'title' => 'Release update lock', + 'description' => 'If your your database is locked and is not updating, unlock it here.', + 'ignore_disabled' => true, + 'url' => '/servers/releaseUpdateLock/' + ], + 'normalizeCustomTagsToTaxonomyFormat' => [ + 'title' => 'Normalize custom tags to taxonomy format', + 'description' => 'Transform all custom tags existing in a taxonomy into the taxonomy version', + 'url' => '/taxonomies/normalizeCustomTagsToTaxonomyFormat/' + ], + ]; + // TODO: [3.x-MIGRATION] Move the usage to each individual command help private function generateCommandLineFunctions() { return [ @@ -2977,5 +194,4 @@ class Server extends AppModel } return $options; } - } diff --git a/src/Model/Entity/SystemSetting.php b/src/Model/Entity/SystemSetting.php new file mode 100644 index 000000000..ce00626be --- /dev/null +++ b/src/Model/Entity/SystemSetting.php @@ -0,0 +1,51 @@ +checkMISPVersion()); + $commit = $this->checkMISPCommit(); + + $authkey = $server['authkey']; + if (EncryptedValue::isEncrypted($authkey)) { + $authkey = (string)new EncryptedValue($authkey); + } + + return [ + 'headers' => [ + 'Authorization' => $authkey, + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + 'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit), + ] + ]; + } + + /** + * @param string $name + * @return bool + */ + protected function pubToZmq($name) + { + static $zmqEnabled; + if ($zmqEnabled === null) { + $zmqEnabled = (bool)Configure::read('Plugin.ZeroMQ_enable'); + } + if ($zmqEnabled) { + return Configure::read("Plugin.ZeroMQ_{$name}_notifications_enable"); + } + return false; + } } diff --git a/src/Model/Table/CryptographicKeysTable.php b/src/Model/Table/CryptographicKeysTable.php index 792345d68..2fdc0d98f 100644 --- a/src/Model/Table/CryptographicKeysTable.php +++ b/src/Model/Table/CryptographicKeysTable.php @@ -56,44 +56,44 @@ class CryptographicKeysTable extends AppTable 'uuid', 'uuid', [ - 'rule' => 'uuid', - 'message' => 'Please provide a valid RFC 4122 UUID' + 'rule' => 'uuid', + 'message' => 'Please provide a valid RFC 4122 UUID' ] ) ->add( 'type', 'inList', [ - 'rule' => ['inList', self::VALID_TYPES], - 'message' => 'Invalid key type' + 'rule' => ['inList', self::VALID_TYPES], + 'message' => 'Invalid key type' ] ) ->add( 'key_data', 'notBlankKey', [ - 'rule' => 'notBlank', - 'message' => 'No key data received.' + 'rule' => 'notBlank', + 'message' => 'No key data received.' ] ) ->add( 'key_data', 'validKey', [ - 'rule' => function ($value, $context) { - return $this->validateKey($context['data']['type'], $value); - }, - 'message' => 'Invalid key.' + 'rule' => function ($value, $context) { + return $this->validateKey($context['data']['type'], $value); + }, + 'message' => 'Invalid key.' ] ) ->add( 'key_data', 'uniqueKeyForElement', [ - 'rule' => function ($value, $context) { - return $this->uniqueKeyForElement($value, $context); - }, - 'message' => 'This key is already assigned to the target.' + 'rule' => function ($value, $context) { + return $this->uniqueKeyForElement($value, $context); + }, + 'message' => 'This key is already assigned to the target.' ] ); @@ -189,13 +189,13 @@ class CryptographicKeysTable extends AppTable return $this->find( 'column', [ - 'conditions' => [ - 'CryptographicKey.parent_type' => 'Event', - 'CryptographicKey.parent_id' => $eventIds, - 'CryptographicKey.fingerprint' => $instanceKey, - ], - 'fields' => ['CryptographicKey.parent_id'], - 'recursive' => -1, + 'conditions' => [ + 'CryptographicKeys.parent_type' => 'Event', + 'CryptographicKeys.parent_id' => $eventIds, + 'CryptographicKeys.fingerprint' => $instanceKey, + ], + 'fields' => ['CryptographicKeys.parent_id'], + 'recursive' => -1, ] ); } @@ -299,10 +299,10 @@ class CryptographicKeysTable extends AppTable { return $this->find()->where( [ - 'parent_type' => $context['data']['parent_type'], - 'parent_id' => $context['data']['parent_id'], - 'key_data' => $value, - 'type' => $context['data']['type'], + 'parent_type' => $context['data']['parent_type'], + 'parent_id' => $context['data']['parent_id'], + 'key_data' => $value, + 'type' => $context['data']['type'], ] )->all()->isEmpty(); } @@ -343,19 +343,19 @@ class CryptographicKeysTable extends AppTable $existingKeys = $this->find( 'all', [ - 'recursive' => -1, - 'conditions' => [ - 'parent_type' => $type, - 'parent_id' => $parent_id, - ], - 'fields' => [ - 'id', - 'type', - 'parent_type', - 'parent_id', - 'revoked', - 'fingerprint', - ] + 'recursive' => -1, + 'conditions' => [ + 'parent_type' => $type, + 'parent_id' => $parent_id, + ], + 'fields' => [ + 'id', + 'type', + 'parent_type', + 'parent_id', + 'revoked', + 'fingerprint', + ] ] )->first(); $toRemove = []; @@ -377,13 +377,13 @@ class CryptographicKeysTable extends AppTable foreach ($cryptographicKeys as $cryptographicKey) { $cryptoKeyEntity = $this->newEntity( [ - 'uuid' => $cryptographicKey['uuid'], - 'key_data' => $cryptographicKey['key_data'], - 'fingerprint' => $cryptographicKey['fingerprint'], - 'revoked' => $cryptographicKey['revoked'], - 'parent_type' => $cryptographicKey['parent_type'], - 'parent_id' => $parent_id, - 'type' => $cryptographicKey['type'] + 'uuid' => $cryptographicKey['uuid'], + 'key_data' => $cryptographicKey['key_data'], + 'fingerprint' => $cryptographicKey['fingerprint'], + 'revoked' => $cryptographicKey['revoked'], + 'parent_type' => $cryptographicKey['parent_type'], + 'parent_id' => $parent_id, + 'type' => $cryptographicKey['type'] ] ); @@ -399,7 +399,7 @@ class CryptographicKeysTable extends AppTable $cryptographicKey['parent_type'], $parent_id ); - $this->deleteAll(['CryptographicKey.id' => $toRemove]); + $this->deleteAll(['CryptographicKeys.id' => $toRemove]); $this->loadLog()->createLogEntry($user, 'updateCryptoKeys', $cryptographicKey['parent_type'], $cryptographicKey['parent_id'], $message); } diff --git a/src/Model/Table/EventBlocklistsTable.php b/src/Model/Table/EventBlocklistsTable.php index 3da0b0245..384d90e50 100644 --- a/src/Model/Table/EventBlocklistsTable.php +++ b/src/Model/Table/EventBlocklistsTable.php @@ -54,13 +54,17 @@ class EventBlocklistsTable extends AppTable */ public function removeBlockedEvents(array &$eventArray) { + if (empty($eventArray)) { + return; + } + // When event array contains a lot events, it is more efficient to fetch all blocked events - $conditions = (count($eventArray) > 10000) ? [] : ['EventBlocklist.event_uuid' => array_column($eventArray, 'uuid')]; + $conditions = (count($eventArray) > 10000) ? [] : ['event_uuid IN' => array_column($eventArray, 'uuid')]; $blocklistHits = $this->find( 'column', [ - 'conditions' => $conditions, - 'fields' => ['EventBlocklist.event_uuid'], + 'conditions' => $conditions, + 'fields' => ['event_uuid'], ] ); if (empty($blocklistHits)) { diff --git a/src/Model/Table/EventsTable.php b/src/Model/Table/EventsTable.php index 8f09cbf18..40556d26b 100644 --- a/src/Model/Table/EventsTable.php +++ b/src/Model/Table/EventsTable.php @@ -2,6 +2,7 @@ namespace App\Model\Table; +use App\Lib\Tools\ServerSyncTool; use App\Model\Table\AppTable; use ArrayObject; use Cake\Core\Configure; @@ -18,6 +19,34 @@ class EventsTable extends AppTable parent::initialize($config); $this->addBehavior('AuditLog'); + $this->belongsTo( + 'User', + [ + 'className' => 'Users', + 'foreignKey' => 'user_id' + ] + ); + $this->belongsTo( + 'ThreatLevel', + [ + 'className' => 'ThreatLevels', + 'foreignKey' => 'threat_level_id' + ] + ); + $this->belongsTo( + 'Org', + [ + 'className' => 'Organisations', + 'foreignKey' => 'org_id' + ] + ); + $this->belongsTo( + 'Orgc', + [ + 'className' => 'Organisations', + 'foreignKey' => 'orgc_id' + ] + ); $this->belongsTo( 'SharingGroup', [ @@ -25,6 +54,7 @@ class EventsTable extends AppTable 'foreignKey' => 'sharing_group_id' ] ); + $this->hasMany( 'Attributes', [ @@ -32,6 +62,52 @@ class EventsTable extends AppTable 'propertyName' => 'Attribute' ] ); + $this->hasMany( + 'ShadowAttributes', + [ + 'dependent' => true, + 'propertyName' => 'ShadowAttribute' + ] + ); + $this->hasMany( + 'Objects', + [ + 'dependent' => true, + 'propertyName' => 'Object', + ] + ); + $this->hasMany( + 'EventTags', + [ + 'dependent' => true, + 'propertyName' => 'EventTag', + ] + ); + $this->hasMany( + 'Sightings', + [ + 'dependent' => true, + 'propertyName' => 'Sighting', + ] + ); + $this->hasMany( + 'EventReports', + [ + 'dependent' => true, + 'propertyName' => 'EventReport', + ] + ); + $this->hasMany( + 'CryptographicKeys', + [ + 'dependent' => true, + 'propertyName' => 'CryptographicKey', + 'foreignKey' => 'parent_id', + 'conditions' => [ + 'parent_type' => 'Events' + ], + ] + ); $this->setDisplayField('title'); } @@ -124,6 +200,8 @@ class EventsTable extends AppTable public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = []) { // TODO: [3.x-MIGRATION] implement when events controller is migrated see #9391 + + // THIS IS A PLACEHOLDER ! $data['Event']['user_id'] = $user['id']; if ($fromPull) { $data['Event']['org_id'] = $org_id; @@ -143,6 +221,46 @@ class EventsTable extends AppTable public function _edit(array &$data, array $user, $id = null, $jobId = null, $passAlong = null, $force = false, $fast_update = false) { // TODO: [3.x-MIGRATION] implement when events controller is migrated see #9391 + + // THIS IS A PLACEHOLDER ! return true; } + + public function fetchEvent($user, $options = [], $useCache = false) + { + // TODO: [3.x-MIGRATION] implement when events controller is migrated see #9391 + + // THIS IS A PLACEHOLDER ! + if (isset($options['event_uuid'])) { + return $this->find( + 'all', + [ + 'conditions' => [ + 'uuid' => $options['event_uuid'] + ] + ] + )->disableHydration()->toArray(); + } + + return []; + } + + /** + * @param array $event + * @param array $server + * @param ServerSyncTool $serverSync + * @return false|string + * @throws HttpSocketJsonException + * @throws JsonException + * @throws Exception + */ + public function uploadEventToServer(array $event, array $server, ServerSyncTool $serverSync) + { + // TODO: [3.x-MIGRATION] implement when events controller is migrated see #9391 + // THIS IS A PLACEHOLDER ! + + $serverSync->pushEvent($event)->getJson(); + + return 'Success'; + } } diff --git a/src/Model/Table/FeedsTable.php b/src/Model/Table/FeedsTable.php index e8bf8f69d..682afba19 100644 --- a/src/Model/Table/FeedsTable.php +++ b/src/Model/Table/FeedsTable.php @@ -193,7 +193,7 @@ class FeedsTable extends AppTable * Gets the event UUIDs from the feed by ID * Returns an array with the UUIDs of events that are new or that need updating. * - * @param array $feed + * @param Feed $feed * @param HttpClient|null $HttpSocket * @return array * @throws Exception @@ -228,12 +228,12 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param HttpClient|null $HttpSocket Null can be for local feed * @return Generator * @throws Exception */ - public function getCache(array $feed, HttpClient $HttpSocket = null) + public function getCache(Feed $feed, HttpClient $HttpSocket = null) { $uri = $feed['url'] . '/hashes.csv'; $data = $this->feedGetUri($feed, $uri, $HttpSocket); @@ -288,13 +288,13 @@ class FeedsTable extends AppTable /** * Get remote manifest for feed with etag checking. - * @param array $feed + * @param Feed $feed * @param HttpClient $HttpSocket * @return array * @throws HttpException * @throws JsonException */ - private function getRemoteManifest(array $feed, HttpClient $HttpSocket) + private function getRemoteManifest(Feed $feed, HttpClient $HttpSocket) { $feedCache = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['id'] . '_manifest.cache.gz'; $feedCacheEtag = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['id'] . '_manifest.etag'; @@ -350,12 +350,12 @@ class FeedsTable extends AppTable /** * Load remote file with cache support and etag checking. - * @param array $feed + * @param Feed $feed * @param HttpClient $HttpSocket * @return string * @throws HttpException */ - private function getFreetextFeedRemote(array $feed, HttpClient $HttpSocket) + private function getFreetextFeedRemote(Feed $feed, HttpClient $HttpSocket) { $feedCache = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['id'] . '.cache.gz'; $feedCacheEtag = Feed::CACHE_DIR . 'misp_feed_' . (int)$feed['id'] . '.etag'; @@ -403,7 +403,7 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param HttpClient|null $HttpSocket Null can be for local feed * @param string $type * @return array|bool @@ -728,14 +728,14 @@ class FeedsTable extends AppTable /** * @param array $actions - * @param array $feed + * @param Feed $feed * @param HttpClient|null $HttpSocket * @param array $user * @param int|false $jobId * @return array * @throws Exception */ - private function downloadFromFeed(array $actions, array $feed, HttpClient $HttpSocket = null, array $user, $jobId = false) + private function downloadFromFeed(array $actions, Feed $feed, HttpClient $HttpSocket = null, array $user, $jobId = false) { $total = count($actions['add']) + count($actions['edit']); $currentItem = 0; @@ -1012,11 +1012,11 @@ class FeedsTable extends AppTable /** * @param array $event - * @param array $feed + * @param Feed $feed * @param array $filterRules * @return array|string */ - private function __prepareEvent($event, array $feed, $filterRules) + private function __prepareEvent($event, Feed $feed, $filterRules) { if (isset($event['response'])) { $event = $event['response']; @@ -1150,7 +1150,7 @@ class FeedsTable extends AppTable /** * @param HttpClient|null $HttpSocket - * @param array $feed + * @param Feed $feed * @param string $uuid * @param array $user * @param array|bool $filterRules @@ -1236,7 +1236,7 @@ class FeedsTable extends AppTable if ($feed['source_format'] === 'misp') { $this->jobProgress($jobId, 'Fetching event manifest.'); try { - $actions = $this->getNewEventUuids($feed->toArray(), $HttpSocket); + $actions = $this->getNewEventUuids($feed, $HttpSocket); } catch (Exception $e) { $this->logException("Could not get new event uuids for feed $feedId.", $e); $this->jobProgress($jobId, 'Could not fetch event manifest. See error log for more details.'); @@ -1249,12 +1249,12 @@ class FeedsTable extends AppTable $total = count($actions['add']) + count($actions['edit']); $this->jobProgress($jobId, __("Fetching %s events.", $total)); - $result = $this->downloadFromFeed($actions, $feed->toArray(), $HttpSocket, $user->toArray(), $jobId); + $result = $this->downloadFromFeed($actions, $feed, $HttpSocket, $user->toArray(), $jobId); $this->__cleanupFile($feed, '/manifest.json'); } else { $this->jobProgress($jobId, 'Fetching data.'); try { - $temp = $this->getFreetextFeed($feed->toArray(), $HttpSocket, $feed['source_format']); + $temp = $this->getFreetextFeed($feed, $HttpSocket, $feed['source_format']); } catch (Exception $e) { $this->logException("Could not get freetext feed $feedId", $e); $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.'); @@ -1511,7 +1511,7 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param Redis $redis * @param int|false $jobId * @return bool @@ -1535,13 +1535,13 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param Redis $redis * @param HttpClient|null $HttpSocket * @param int|false $jobId * @return bool */ - private function __cacheFreetextFeed(array $feed, $redis, HttpClient $HttpSocket = null, $jobId = false) + private function __cacheFreetextFeed(Feed $feed, $redis, HttpClient $HttpSocket = null, $jobId = false) { $feedId = $feed['id']; @@ -2092,7 +2092,7 @@ class FeedsTable extends AppTable /** * Download and parse event from feed. * - * @param array $feed + * @param Feed $feed * @param string $eventUuid * @param HttpClient|null $HttpSocket Null can be for local feed * @return array @@ -2115,7 +2115,7 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param string $uri * @param HttpClient|null $HttpSocket Null can be for local feed * @return string @@ -2136,14 +2136,14 @@ class FeedsTable extends AppTable } /** - * @param array $feed + * @param Feed $feed * @param string $uri * @param HttpClient $HttpSocket * @param string|null $etag * @return false|HttpClientResponse * @throws HttpException */ - private function feedGetUriRemote(array $feed, $uri, HttpClient $HttpSocket, $etag = null) + private function feedGetUriRemote(Feed $feed, $uri, HttpClient $HttpSocket, $etag = null) { $request = $this->__createFeedRequest($feed['headers']); if ($etag) { diff --git a/src/Model/Table/GalaxyClusterRelationsTable.php b/src/Model/Table/GalaxyClusterRelationsTable.php index f07e31424..67492d091 100644 --- a/src/Model/Table/GalaxyClusterRelationsTable.php +++ b/src/Model/Table/GalaxyClusterRelationsTable.php @@ -4,6 +4,10 @@ namespace App\Model\Table; use App\Model\Entity\Distribution; use App\Model\Table\AppTable; +use ArrayObject; +use Cake\Collection\CollectionInterface; +use Cake\Event\EventInterface; +use Cake\ORM\Query; use Cake\Utility\Hash; use Cake\Validation\Validator; @@ -88,17 +92,24 @@ class GalaxyClusterRelationsTable extends AppTable ); } - public function afterFind($results, $primary = false) + public function beforeFind(EventInterface $event, Query $query, ArrayObject $options) { - foreach ($results as $k => $result) { - if (isset($result['TargetCluster']) && key_exists('id', $result['TargetCluster']) && is_null($result['TargetCluster']['id'])) { - $results[$k]['TargetCluster'] = []; - } - if (isset($result['GalaxyClusterRelation']['distribution']) && $result['GalaxyClusterRelation']['distribution'] != 4) { - unset($results[$k]['SharingGroup']); - } - } - return $results; + $query->formatResults( + function (CollectionInterface $results) { + return $results->map( + function ($row) { + if (isset($row['TargetCluster']) && key_exists('id', $row['TargetCluster']) && is_null($row['TargetCluster']['id'])) { + $row['TargetCluster'] = []; + } + if (isset($row['GalaxyClusterRelation']['distribution']) && $row['GalaxyClusterRelation']['distribution'] != 4) { + unset($row['SharingGroup']); + } + return $row; + } + ); + }, + $query::APPEND + ); } public function buildConditions($user, $clusterConditions = true) diff --git a/src/Model/Table/GalaxyClustersTable.php b/src/Model/Table/GalaxyClustersTable.php index 245d1e0ea..f44918f15 100644 --- a/src/Model/Table/GalaxyClustersTable.php +++ b/src/Model/Table/GalaxyClustersTable.php @@ -5,6 +5,7 @@ namespace App\Model\Table; use App\Http\Exception\HttpSocketHttpException; use App\Lib\Tools\BackgroundJobsTool; use App\Lib\Tools\HttpTool; +use App\Lib\Tools\LogExtendedTrait; use App\Lib\Tools\ServerSyncTool; use App\Lib\Tools\TmpFileTool; use App\Model\Entity\Distribution; @@ -37,6 +38,8 @@ use Exception; */ class GalaxyClustersTable extends AppTable { + use LogExtendedTrait; + private $__assetCache = []; private $__clusterCache = []; private $deletedClusterUUID; diff --git a/src/Model/Table/JobsTable.php b/src/Model/Table/JobsTable.php index 3ad304a75..a06d82f1d 100644 --- a/src/Model/Table/JobsTable.php +++ b/src/Model/Table/JobsTable.php @@ -3,6 +3,7 @@ namespace App\Model\Table; use App\Lib\Tools\BackgroundJobsTool; +use App\Lib\Tools\LogExtendedTrait; use App\Model\Entity\Job; use App\Model\Table\AppTable; use ArrayObject; @@ -12,6 +13,8 @@ use Exception; class JobsTable extends AppTable { + use LogExtendedTrait; + public function initialize(array $config): void { parent::initialize($config); diff --git a/src/Model/Table/LogsTable.php b/src/Model/Table/LogsTable.php index 5844ca683..629ce2bf4 100644 --- a/src/Model/Table/LogsTable.php +++ b/src/Model/Table/LogsTable.php @@ -1348,4 +1348,20 @@ class LogsTable extends AppTable } return $this->elasticSearchClient; } + + /** + * @param $data + * @param $options + * @return array|bool|mixed + */ + public function saveOrFailSilently($data, $options = []) + { + try { + $entity = $this->newEntity($data, $options); + return $this->save($entity, $options); + } catch (Exception $e) { + $this->logException('Could not save log to database', $e); + return false; + } + } } diff --git a/src/Model/Table/MispObjectsTable.php b/src/Model/Table/MispObjectsTable.php new file mode 100644 index 000000000..ada22c72c --- /dev/null +++ b/src/Model/Table/MispObjectsTable.php @@ -0,0 +1,14 @@ +setTable('objects'); + } +} diff --git a/src/Model/Table/OrgBlocklistsTable.php b/src/Model/Table/OrgBlocklistsTable.php new file mode 100644 index 000000000..90ae0053d --- /dev/null +++ b/src/Model/Table/OrgBlocklistsTable.php @@ -0,0 +1,190 @@ +addBehavior('AuditLog'); + } + + public $blocklistFields = ['org_uuid', 'comment', 'org_name']; + + public $blocklistTarget = 'org'; + + private $blockedCache = []; + + public function validationDefault(Validator $validator): Validator + { + $validator + ->requirePresence('org_uuid') + ->notEmptyString('org_uuid') + ->uuid('org_uuid'); + + return $validator; + } + + public function buildRules(RulesChecker $rules): RulesChecker + { + $rules->add($rules->isUnique(['org_uuid'])); + return $rules; + } + + public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + if (empty($entity->id)) { + $entity->created = date('Y-m-d H:i:s'); + } + return true; + } + + public function afterDelete(EventInterface $event, EntityInterface $entity, ArrayObject $options) + { + parent::afterDelete(); + if (!empty($entity['org_uuid'])) { + $this->cleanupBlockedCount($entity['org_uuid']); + } + } + + public function beforeFind(EventInterface $event, Query $query, ArrayObject $options) + { + $query->formatResults( + function (CollectionInterface $results) { + return $results->map( + function ($row) { + if (isset($row['org_uuid'])) { + $row['blocked_data'] = $this->getBlockedData($row['org_uuid']); + } + return $row; + } + ); + }, + $query::APPEND + ); + } + + /** + * @param array $eventArray + */ + public function removeBlockedEvents(array &$eventArray) + { + if (empty($eventArray)) { + return; + } + + // When event array contains a lot events, it is more efficient to fetch all blocked events + $blocklistHits = $this->find( + 'column', + [ + 'conditions' => ['org_uuid IN' => array_column($eventArray, 'orgc_uuid')], + 'fields' => ['org_uuid'], + ] + ); + if (empty($blocklistHits)) { + return; + } + $blocklistHits = array_flip($blocklistHits->toArray()); + foreach ($eventArray as $k => $event) { + if (isset($blocklistHits[$event['orgc_uuid']])) { + unset($eventArray[$k]); + } + } + } + + /** + * @param int|string $orgIdOrUuid Organisation ID or UUID + * @return bool + */ + public function isBlocked($orgIdOrUuid) + { + if (isset($this->blockedCache[$orgIdOrUuid])) { + return $this->blockedCache[$orgIdOrUuid]; + } + + if (is_numeric($orgIdOrUuid)) { + $orgUuid = $this->getUUIDFromID($orgIdOrUuid); + } else { + $orgUuid = $orgIdOrUuid; + } + + $isBlocked = $this->exists(['org_uuid' => $orgUuid]); + $this->blockedCache[$orgIdOrUuid] = $isBlocked; + + return $isBlocked; + } + + private function getUUIDFromID($orgID) + { + $OrganisationsTable = $this->fetchTable('Organisations'); + $orgUuid = $OrganisationsTable->get($orgID, [ + 'fields' => ['Organisation.uuid'], + ]); + if (empty($orgUuid)) { + return false; // org not found by ID, so it is not blocked + } + $orgUuid = $orgUuid['uuid']; + + return $orgUuid; + } + + public function saveEventBlocked($orgIdOrUUID) + { + if (is_numeric($orgIdOrUUID)) { + $orgcUUID = $this->getUUIDFromID($orgIdOrUUID); + } else { + $orgcUUID = $orgIdOrUUID; + } + $lastBlockTime = time(); + $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}"; + $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}"; + $redis = RedisTool::init(); + if ($redis !== false) { + $pipe = $redis->multi(Redis::PIPELINE) + ->incr($redisKeyBlockAmount) + ->set($redisKeyBlockLastTime, $lastBlockTime); + $pipe->exec(); + } + } + + private function cleanupBlockedCount($orgcUUID) + { + $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}"; + $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}"; + $redis = RedisTool::init(); + if ($redis !== false) { + $pipe = $redis->multi(Redis::PIPELINE) + ->del($redisKeyBlockAmount) + ->del($redisKeyBlockLastTime); + $pipe->exec(); + } + } + + public function getBlockedData($orgcUUID) + { + $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}"; + $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}"; + $blockData = [ + 'blocked_amount' => false, + 'blocked_last_time' => false, + ]; + $redis = RedisTool::init(); + if ($redis !== false) { + $blockData['blocked_amount'] = $redis->get($redisKeyBlockAmount); + $blockData['blocked_last_time'] = $redis->get($redisKeyBlockLastTime); + } + return $blockData; + } +} diff --git a/src/Model/Table/ServersTable.php b/src/Model/Table/ServersTable.php index c055f1757..11c2c1913 100644 --- a/src/Model/Table/ServersTable.php +++ b/src/Model/Table/ServersTable.php @@ -2,19 +2,2842 @@ namespace App\Model\Table; +use App\Http\Exception\HttpSocketHttpException; +use App\Http\Exception\HttpSocketJsonException; use App\Lib\Tools\BackgroundJobsTool; +use App\Lib\Tools\BetterSecurity; +use App\Lib\Tools\EncryptedValue; +use App\Lib\Tools\FileAccessTool; +use App\Lib\Tools\GitTool; +use App\Lib\Tools\GpgTool; +use App\Lib\Tools\HttpTool; +use App\Lib\Tools\JsonTool; +use App\Lib\Tools\LogExtendedTrait; use App\Lib\Tools\ProcessTool; +use App\Lib\Tools\RedisTool; +use App\Lib\Tools\ServerSyncTool; +use App\Model\Entity\Event; +use App\Model\Entity\Job; +use App\Model\Entity\SystemSetting; use App\Model\Table\AppTable; +use ArrayObject; +use Cake\Chronos\Chronos; use Cake\Core\Configure; +use Cake\Datasource\EntityInterface; +use Cake\Event\EventInterface; +use Cake\Http\Exception\MethodNotAllowedException; +use Cake\Http\Exception\NotFoundException; +use Cake\Utility\Hash; +use Cake\Validation\Validation; +use Cake\Validation\Validator; +use Closure; +use DirectoryIterator; use Exception; +use InvalidArgumentException; +use RegexIterator; +use SplFileInfo; class ServersTable extends AppTable { + use LogExtendedTrait; + + public const MYSQL_RECOMMENDED_SETTINGS = [ + 'innodb_buffer_pool_size' => [ + 'default' => '134217728', + 'recommended' => '2147483648', + 'explanation' => 'The InnoDB buffer pool is the memory area where caches table and index data reside. It is the most important MySQL setting, in a dedicated server it should be around 3/4 of all the available RAM. In a shared server it should be around 1/2 of the available RAM.', + ], + 'innodb_dedicated_server' => [ + 'default' => '0', + 'recommended' => '', + 'explanation' => 'Set to `1` if the database is running in a dedicated server. The database engine will examine the available memory and dynamically set `innodb_buffer_pool_size`, `innodb_log_file_size`, `innodb_log_files_in_group` and `innodb_flush_method`. It is particularly useful in cloud enviroments that can be auto-scaled.', + ], + 'innodb_log_file_size' => [ + 'default' => '100663296', + 'recommended' => '629145600', + 'explanation' => 'This parameter determines the fixed size for MySQLs redo logs. Tuning this value affects the crash recovery time and also overall system performance.', + ], + 'innodb_log_files_in_group' => [ + 'default' => '2', + 'recommended' => '2', + 'explanation' => 'Defines the number of log files in the log group.', + ], + 'innodb_change_buffering' => [ + 'default' => 'none', + 'recommended' => 'none', + 'explanation' => 'Whether InnoDB performs change buffering, an optimization that delays write operations to secondary indexes so that the I/O operations can be performed sequentially, enabling it causes extremely long shutdown times for upgrades.', + ], + 'innodb_io_capacity' => [ + 'default' => '200', + 'recommended' => '1000', + 'explanation' => 'Defines the number of I/O operations per second (IOPS) available to InnoDB background tasks, such as flushing pages from the buffer pool and merging data from the change buffer.', + ], + 'innodb_io_capacity_max' => [ + 'default' => '2000', + 'recommended' => '2000', + 'explanation' => 'If flushing activity falls behind, InnoDB can flush more aggressively, at a higher rate of I/O operations per second (IOPS) than defined by the `innodb_io_capacity variable`.', + ], + 'innodb_stats_persistent' => [ + 'default' => 'ON', + 'recommended' => 'ON', + 'explanation' => 'Specifies whether InnoDB index statistics are persisted to disk. Otherwise, statistics may be recalculated frequently which can lead to variations in query execution plans.', + ], + 'innodb_read_io_threads' => [ + 'default' => '4', + 'recommended' => '16', + 'explanation' => 'The number of I/O threads for read operations in InnoDB.', + ], + 'innodb_write_io_threads' => [ + 'default' => '4', + 'recommended' => '4', + 'explanation' => 'The number of I/O threads for write operations in InnoDB.', + ], + ]; + + public const VALID_EVENT_INDEX_FILTERS = ['searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute']; + + protected $serverSettings = []; + public function initialize(array $config): void { parent::initialize($config); $this->addBehavior('AuditLog'); $this->addBehavior('EncryptedFields', ['fields' => ['authkey']]); + $this->addBehavior( + 'JsonFields', + [ + 'fields' => [ + 'push_rules' => [ + 'default' => ["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []]] + ], + 'pull_rules' => [ + 'default' => ["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []], "type_attributes" => ["NOT" => []], "type_objects" => ["NOT" => []], "url_params" => ""] + ] + ], + ] + ); + + $this->belongsTo( + 'Organisations', + [ + 'className' => 'Organisations', + 'foreignKey' => 'org_id', + 'propertyName' => 'Organisation', + ] + ); + $this->belongsTo( + 'RemoteOrg', + [ + 'className' => 'Organisations', + 'foreignKey' => 'remote_org_id', + 'propertyName' => 'RemoteOrg', + ] + ); + $this->hasMany( + 'SharingGroupServers', + [ + 'foreignKey' => 'server_id', + 'dependent' => true, + ] + ); + $this->hasMany( + 'Users', + [ + 'className' => 'Users', + 'foreignKey' => 'server_id', + 'dependent' => true, + ] + ); + } + + public function validationDefault(Validator $validator): Validator + { + $validator + ->notEmptyString('name') + ->requirePresence(['name'], 'create') + ->add( + 'url', + [ + 'validateURL' => [ + 'rule' => function ($value) { + return $this->testURL($value); + } + ] + ] + ) + ->add( + 'authkey', + [ + 'validateAuthkey' => [ + 'rule' => function ($value) { + return $this->validateAuthkey($value); + } + ] + ] + ) + ->add( + 'org_id', + [ + 'validateOrgId' => [ + 'rule' => function ($value) { + return $this->valueIsID($value); + }, + 'allowEmpty' => false, + 'required' => true, + ] + ] + ) + ->boolean('push') + ->allowEmptyString('push') + ->boolean('pull') + ->allowEmptyString('pull') + ->boolean('push_sightings') + ->allowEmptyString('push_sightings') + ->integer('lastpushedid') + ->allowEmptyString('lastpushedid') + ->integer('lastpulledid') + ->allowEmptyString('lastpulledid'); + + return $validator; + } + + public function beforeSave(EventInterface $event, EntityInterface $server, ArrayObject $options) + { + if (!empty($server['url'])) { + $server['url'] = rtrim($server['url'], '/'); + } + if (empty($server['id'])) { + $max_prio = $this->find( + 'all', + [ + 'recursive' => -1, + 'order' => ['priority' => 'DESC'], + 'fields' => ['priority'] + ] + )->first(); + if (empty($max_prio)) { + $max_prio = 0; + } else { + $max_prio = $max_prio['priority']; + } + $server['priority'] = $max_prio + 1; + } + // Encrypt authkey if plain key provided and encryption is enabled + if (!empty($server['authkey']) && strlen($server['authkey']) === 40) { + $server['authkey'] = EncryptedValue::encryptIfEnabled($server['authkey']); + } + + try { + // Clean caches when remote server setting changed + $cacheKeys = [ + "misp:event_index:{$server->id}", + "misp:fetched_sightings:{$server->id}", + "misp:empty_events:{$server->id}", + ]; + RedisTool::unlink(RedisTool::init(), $cacheKeys); + } catch (Exception $e) { + // ignore + } + + return true; + } + + /** + * @param int|string $technique 'full', 'update', remote event ID or remote event UUID + * @param ServerSyncTool $serverSync + * @param bool $force + * @return array Event UUIDSs or IDs + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + private function __getEventIdListBasedOnPullTechnique($technique, ServerSyncTool $serverSync, $force = false) + { + if ("full" === $technique) { + // get a list of the event_ids on the server + $eventIds = $this->getEventIdsFromServer($serverSync, false, false, $force); + // reverse array of events, to first get the old ones, and then the new ones + return array_reverse($eventIds); + } elseif ("update" === $technique) { + $eventIds = $this->getEventIdsFromServer($serverSync, false, true, $force); + $EventsTable = $this->fetchTable('Events'); + $localEventUuids = $EventsTable->find( + 'column', + [ + 'fields' => ['Event.uuid'], + ] + ); + return array_intersect($eventIds, $localEventUuids); + } elseif (is_numeric($technique)) { + return [intval($technique)]; + } elseif (Validation::uuid($technique)) { + return [$technique]; + } + throw new InvalidArgumentException("Invalid pull technique `$technique`."); + } + + /** + * @param array $event + * @param array $server + * @param array $user + * @param array $pullRules + * @return bool Return true if event was emptied by pull rules + */ + private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules) + { + $pullRulesEmptiedEvent = false; + // we have an Event array + // The event came from a pull, so it should be locked. + $event['Event']['locked'] = true; + if (!isset($event['Event']['distribution'])) { // version 1 + $event['Event']['distribution'] = '1'; + } + // Distribution + if (empty(Configure::read('MISP.host_org_id')) || !$server['internal'] || Configure::read('MISP.host_org_id') != $server['org_id']) { + switch ($event['Event']['distribution']) { + case 1: + // if community only, downgrade to org only after pull + $event['Event']['distribution'] = '0'; + break; + case 2: + // if connected communities downgrade to community only + $event['Event']['distribution'] = '1'; + break; + } + // We remove local tags obtained via pull + if (isset($event['Event']['Tag'])) { + foreach ($event['Event']['Tag'] as $key => $a) { + if ($a['local']) { + unset($event['Event']['Tag'][$key]); + } + } + } + + $filterOnTypeEnabled = !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')); + $attributeTypeFilteringEnabled = $filterOnTypeEnabled && !empty($pullRules['type_attributes']['NOT']); + + if (isset($event['Event']['Attribute'])) { + $originalCount = count($event['Event']['Attribute']); + foreach ($event['Event']['Attribute'] as $key => $attribute) { + if ($attributeTypeFilteringEnabled && in_array($attribute['type'], $pullRules['type_attributes']['NOT'], true)) { + unset($event['Event']['Attribute'][$key]); + continue; + } + switch ($attribute['distribution']) { + case '1': + $event['Event']['Attribute'][$key]['distribution'] = '0'; + break; + case '2': + $event['Event']['Attribute'][$key]['distribution'] = '1'; + break; + } + // We remove local tags obtained via pull + if (isset($attribute['Tag'])) { + foreach ($attribute['Tag'] as $k => $v) { + if ($v['local']) { + unset($event['Event']['Attribute'][$key]['Tag'][$k]); + } + } + } + } + if ($attributeTypeFilteringEnabled && $originalCount > 0 && empty($event['Event']['Attribute'])) { + $pullRulesEmptiedEvent = true; + } + } + + if (isset($event['Event']['Object'])) { + $originalObjectCount = count($event['Event']['Object']); + foreach ($event['Event']['Object'] as $i => $object) { + if ( + $filterOnTypeEnabled && + !empty($pullRules['type_objects']['NOT']) && + in_array($object['template_uuid'], $pullRules['type_objects']['NOT'], true) + ) { + unset($event['Event']['Object'][$i]); + continue; + } + switch ($object['distribution']) { + case '1': + $event['Event']['Object'][$i]['distribution'] = '0'; + break; + case '2': + $event['Event']['Object'][$i]['distribution'] = '1'; + break; + } + if (isset($object['Attribute'])) { + $originalAttributeCount = count($object['Attribute']); + foreach ($object['Attribute'] as $j => $a) { + if ($attributeTypeFilteringEnabled && in_array($a['type'], $pullRules['type_attributes']['NOT'], true)) { + unset($event['Event']['Object'][$i]['Attribute'][$j]); + continue; + } + switch ($a['distribution']) { + case '1': + $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0'; + break; + case '2': + $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '1'; + break; + } + // We remove local tags obtained via pull + if (isset($a['Tag'])) { + foreach ($a['Tag'] as $k => $v) { + if ($v['local']) { + unset($event['Event']['Object'][$i]['Attribute'][$j]['Tag'][$k]); + } + } + } + } + if ($attributeTypeFilteringEnabled && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) { + unset($event['Event']['Object'][$i]); // Object is empty, get rid of it + } + } + } + if ($filterOnTypeEnabled && $originalObjectCount > 0 && empty($event['Event']['Object'])) { + $pullRulesEmptiedEvent = true; + } + } + if (isset($event['Event']['EventReport'])) { + foreach ($event['Event']['EventReport'] as $key => $r) { + switch ($r['distribution']) { + case '1': + $event['Event']['EventReport'][$key]['distribution'] = '0'; + break; + case '2': + $event['Event']['EventReport'][$key]['distribution'] = '1'; + break; + } + } + } + } + + // Distribution, set reporter of the event, being the admin that initiated the pull + $event['Event']['user_id'] = $user['id']; + + return $pullRulesEmptiedEvent; + } + + /** + * @param array $event + * @return bool True if event is not empty + */ + private function __checkIfEventSaveAble(array $event) + { + if (!empty($event['Event']['Attribute'])) { + foreach ($event['Event']['Attribute'] as $attribute) { + if (empty($attribute['deleted'])) { + return true; + } + } + } + if (!empty($event['Event']['Object'])) { + foreach ($event['Event']['Object'] as $object) { + if (!empty($object['deleted'])) { + continue; + } + if (!empty($object['Attribute'])) { + foreach ($object['Attribute'] as $attribute) { + if (empty($attribute['deleted'])) { + return true; + } + } + } + } + } + if (!empty($event['Event']['EventReport'])) { + foreach ($event['Event']['EventReport'] as $report) { + if (empty($report['deleted'])) { + return true; + } + } + } + return false; + } + + /** + * @param array $event + * @param int|string $eventId + * @param array $successes + * @param array $fails + * @param EventsTable $EventsTable + * @param array $server + * @param array $user + * @param int $jobId + * @param bool $force + * @param Response $response + * @return false|void + * @throws Exception + */ + private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, EventsTable $EventsTable, $server, $user, $jobId, $force, $response) + { + $force = $force ?? false; + + // check if the event already exist (using the uuid) + $existingEvent = $EventsTable->find( + 'all', + [ + 'conditions' => ['uuid' => $event['Event']['uuid']], + 'recursive' => -1, + 'fields' => ['id', 'locked', 'protected'], + 'contain' => ['CryptographicKeys'] + ] + )->first(); + $passAlong = $server['id']; + if (!$existingEvent) { + // add data for newly imported events + if (isset($event['Event']['protected']) && $event['Event']['protected']) { + if (!$EventsTable->CryptographicKeys->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) { + $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); + return false; + } + } + $result = $EventsTable->_add($event, true, $user, $server['org_id'], $passAlong, true, $jobId); + if ($result) { + $successes[] = $eventId; + if ($this->pubToZmq('event')) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->event_save(['Event' => $eventId, 'Server' => $server['id']], 'add_from_connected_server'); + } + } else { + $fails[$eventId] = __('Failed (partially?) because of validation errors: ') . json_encode($EventsTable->validationErrors); + } + } else { + if (!$existingEvent['Event']['locked'] && !$server['internal']) { + $fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.'); + } else { + if ($existingEvent['Event']['protected']) { + if (!$EventsTable->CryptographicKeys->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) { + $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); + } + } + $result = $EventsTable->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force); + if ($result === true) { + $successes[] = $eventId; + if ($this->pubToZmq('event')) { + $pubSubTool = $this->getPubSubTool(); + $pubSubTool->event_save(['Event' => $eventId, 'Server' => $server['id']], 'edit_from_connected_server'); + } + } elseif (isset($result['error'])) { + $fails[$eventId] = $result['error']; + } else { + $fails[$eventId] = json_encode($result); + } + } + } + } + + /** + * @param int|string $eventId Event ID or UUID + * @param array $successes + * @param array $fails + * @param EventsTable $EventsTable + * @param ServerSyncTool $serverSync + * @param array $user + * @param int $jobId + * @param bool $force + * @return bool + */ + private function __pullEvent($eventId, array &$successes, array &$fails, EventsTable $EventsTable, ServerSyncTool $serverSync, $user, $jobId, $force = false) + { + $params = [ + 'deleted' => [0, 1], + 'excludeGalaxy' => 1, + 'includeEventCorrelations' => 0, // we don't need remote correlations + 'includeFeedCorrelations' => 0, + 'includeWarninglistHits' => 0, // we don't need remote warninglist hits + ]; + if (empty($serverSync->server()['internal'])) { + $params['excludeLocalTags'] = 1; + } + try { + $response = $serverSync->fetchEvent($eventId, $params); + $event = $response->getJson(); + } catch (Exception $e) { + $this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e); + $fails[$eventId] = __('failed downloading the event'); + return false; + } + + $pullRulesEmptiedEvent = $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules()); + + if (!$this->__checkIfEventSaveAble($event)) { + if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure + $fails[$eventId] = __('Empty event detected.'); + $this->addEmptyEvent($serverSync->serverId(), $event); + } + return false; + } + $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $EventsTable, $serverSync->server(), $user, $jobId, $force, $response); + return true; + } + + /** + * @param array $user + * @param string $technique + * @param array $server + * @param int|false $jobId + * @param bool $force + * @return array|string + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + */ + public function pull(array $user, $technique, array $server, $jobId = false, $force = false) + { + if ($jobId) { + Configure::write('CurrentUserId', $user['id']); + $JobsTable = $this->fetchTable('Jobs'); + $email = "Scheduled job"; + } else { + $email = $user['email']; + } + + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + try { + $server['version'] = $serverSync->info()['version']; + } catch (Exception $e) { + $this->logException("Could not get remote server `{$server['name']}` version.", $e); + if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) { + $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'); + } else { + $message = $e->getMessage(); + } + $title = 'Failed pull from ' . $server['url'] . ' initiated by ' . $email; + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['id'], $title, $message); + return $message; + } + + $pulledClusters = 0; + if (!empty($server['pull_galaxy_clusters'])) { + $GalaxyClustersTable = $this->fetchTable('GalaxyClusters'); + if ($jobId) { + $JobsTable->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.')); + } + $pulledClusters = $GalaxyClustersTable->pullGalaxyClusters($user, $serverSync, $technique); + if ($technique === 'pull_relevant_clusters') { + if ($jobId) { + $JobsTable->saveStatus($jobId, true, 'Pulling complete.'); + } + return [[], [], 0, 0, $pulledClusters]; + } + if ($jobId) { + $JobsTable->saveProgress($jobId, 'Pulling events.', 10); + } + } + + try { + $eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force); + } catch (Exception $e) { + $this->logException("Could not fetch event IDs from server `{$server['name']}`.", $e); + if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) { + $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'); + } else { + $message = $e->getMessage(); + } + $title = 'Failed pull from ' . $server['url'] . ' initiated by ' . $email; + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['id'], $title, $message); + return $message; + } + + /** @var EventsTable $EventTable */ + $EventsTable = $this->fetchTable('Events'); + $successes = []; + $fails = []; + // now process the $eventIds to pull each of the events sequentially + if (!empty($eventIds)) { + // download each event + if ($jobId) { + $JobsTable->saveProgress($jobId, __n('Pulling {0} event.', 'Pulling {1} events.', count($eventIds), count($eventIds))); + } + foreach ($eventIds as $k => $eventId) { + $this->__pullEvent($eventId, $successes, $fails, $EventsTable, $serverSync, $user, $jobId, $force); + if ($jobId && $k % 10 === 0) { + $JobsTable->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds))); + } + } + foreach ($fails as $eventid => $message) { + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message); + } + } + if ($jobId) { + $JobsTable->saveProgress($jobId, 'Pulling proposals.', 50); + } + $pulledProposals = $pulledSightings = 0; + if ($technique === 'full' || $technique === 'update') { + $pulledProposals = $EventsTable->ShadowAttributes->pullProposals($user, $serverSync); + + if ($jobId) { + $JobsTable->saveProgress($jobId, 'Pulling sightings.', 75); + } + $pulledSightings = $EventsTable->Sightings->pullSightings($user, $serverSync); + } + if ($jobId) { + $JobsTable->saveStatus($jobId, true, 'Pull completed.'); + } + + $change = sprintf( + '%s events, %s proposals, %s sightings and %s galaxy clusters pulled or updated. %s events failed or didn\'t need an update.', + count($successes), + $pulledProposals, + $pulledSightings, + $pulledClusters, + count($fails) + ); + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['id'], 'Pull from ' . $server['url'] . ' initiated by ' . $email, $change); + return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters]; + } + + public function filterRuleToParameter($filter_rules) + { + $final = []; + if (empty($filter_rules)) { + return $final; + } + $url_params = []; + foreach ($filter_rules as $field => $rules) { + $temp = []; + if ($field === 'url_params') { + $url_params = empty($rules) ? [] : $this->jsonDecode($rules); + } else { + foreach ($rules as $operator => $elements) { + foreach ($elements as $k => $element) { + if ($operator === 'NOT') { + $element = '!' . $element; + } + if (!empty($element)) { + $temp[] = $element; + } + } + } + if (!empty($temp)) { + $final[substr($field, 0, strlen($field) - 1)] = $temp; + } + } + } + if (!empty($url_params)) { + $final = array_merge_recursive($final, $url_params); + } + return $final; + } + + /** + * fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions + * + * @param ServerSyncTool $serverSync + * @param array $conditions + * @return array The list of clusters + * @throws JsonException|HttpSocketHttpException|HttpSocketJsonException + */ + private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions = []) + { + $filterRules = [ + 'published' => 1, + 'minimal' => 1, + 'custom' => 1, + ]; + $filterRules = array_merge($filterRules, $conditions); + $clusterArray = $serverSync->galaxyClusterSearch($filterRules)->getJson(); + if (isset($clusterArray['response'])) { + $clusterArray = $clusterArray['response']; + } + return $clusterArray; + } + + /** + * Get a list of cluster IDs that are present on the remote server and returns clusters that should be pulled + * + * @param ServerSyncTool $serverSync + * @param bool $onlyUpdateLocalCluster If set to true, only cluster present locally will be returned + * @param array $eligibleClusters Array of cluster present locally that could potentially be updated. Linked to $onlyUpdateLocalCluster + * @param array $conditions Conditions to be sent to the remote server while fetching accessible clusters IDs + * @return array List of cluster UUIDs to be pulled + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + */ + public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster = true, array $eligibleClusters = [], array $conditions = []) + { + $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO); + + if ($onlyUpdateLocalCluster && empty($eligibleClusters)) { + return []; // no clusters for update + } + + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions = $conditions); + if (empty($clusterArray)) { + return []; // empty remote clusters + } + + /** @var GalaxyClusterBlocklistsTable $GalaxyClusterBlocklistsTable */ + $GalaxyClusterBlocklistsTable = $this->fetchTable('GalaxyClusterBlocklists'); + + if (!$onlyUpdateLocalCluster) { + /** @var GalaxyClustersTable $GalaxyClustersTable */ + $GalaxyClustersTable = $this->fetchTable('GalaxyClusters'); + // Do not fetch clusters with the same or newer version that already exists on local instance + $eligibleClusters = $GalaxyClustersTable->find( + 'list', + [ + 'conditions' => ['GalaxyCluster.uuid' => array_column(array_column($clusterArray, 'GalaxyCluster'), 'uuid')], + 'fields' => ['GalaxyCluster.uuid', 'GalaxyCluster.version'], + ] + ); + } + + $clustersForPull = []; + foreach ($clusterArray as $cluster) { + $clusterUuid = $cluster['GalaxyCluster']['uuid']; + + if ($GalaxyClusterBlocklistsTable->checkIfBlocked($clusterUuid)) { + continue; // skip blocked clusters + } + + if (isset($eligibleClusters[$clusterUuid])) { + $localVersion = $eligibleClusters[$clusterUuid]; + if ($localVersion < $cluster['GalaxyCluster']['version']) { + $clustersForPull[] = $clusterUuid; + } + } elseif (!$onlyUpdateLocalCluster) { + $clustersForPull[] = $clusterUuid; + } + } + return $clustersForPull; + } + + /** + * Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed. + * @param ServerSyncTool $serverSync + * @param array $localClusters + * @param array $conditions + * @return array + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + */ + private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters = [], array $conditions = []) + { + $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO); + $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions = $conditions); + $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version'); + if (!empty($localClusters)) { + foreach ($localClusters as $k => $localCluster) { + if (isset($keyedClusterArray[$localCluster['GalaxyCluster']['uuid']])) { + $remoteVersion = $keyedClusterArray[$localCluster['GalaxyCluster']['uuid']]; + if ($localCluster['GalaxyCluster']['version'] <= $remoteVersion) { + unset($localClusters[$k]); + } + } + } + } + return $localClusters; + } + + /** + * @param ServerSyncTool $serverSync + * @param bool $ignoreFilterRules Ignore defined server pull rules + * @return array + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + * @throws RedisException + */ + public function getEventIndexFromServer(ServerSyncTool $serverSync, $ignoreFilterRules = false) + { + if (!$ignoreFilterRules) { + $filterRules = $this->filterRuleToParameter($serverSync->server()['pull_rules']); + if (!empty($filterRules['org']) && !$serverSync->isSupported(ServerSyncTool::FEATURE_ORG_RULE)) { + $filterRules['org'] = implode('|', $filterRules['org']); + } + } else { + $filterRules = []; + } + $filterRules['minimal'] = 1; + $filterRules['published'] = 1; + + // Fetch event index from cache if exists and is not modified + $redis = RedisTool::init(); + $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}"); + if ($indexFromCache) { + list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache)); + } else { + $etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data + } + + $response = $serverSync->eventIndex($filterRules, $etag); + + if ($response->getStatusCode() === 304 && $indexFromCache) { + return $eventIndex; + } + + $eventIndex = $response->getJson(); + + // correct $eventArray if just one event, probably this response returns old MISP + if (isset($eventIndex['id'])) { + $eventIndex = [$eventIndex]; + } + + // Save to cache for 24 hours if ETag provided + $etag = null; + if (count($response->getHeader('etag'))) { + $etag = $response->getHeader('etag')[0]; + } + + if ($etag) { + $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex])); + $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data); + } elseif ($indexFromCache) { + RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}"); + } + + return $eventIndex; + } + + /** + * @param array $events + * @return void + */ + private function removeOlderEvents(array &$events) + { + if (empty($events)) { + return; + } + + $conditions = (count($events) > 10000) ? [] : ['uuid IN' => array_column($events, 'uuid')]; + $EventsTable = $this->fetchTable('Events'); + $localEvents = $EventsTable->find( + 'all', + [ + 'recursive' => -1, + 'conditions' => $conditions, + 'fields' => ['Events.uuid', 'Events.timestamp', 'Events.locked'], + ] + )->toArray(); + $localEvents = array_column($localEvents, null, 'uuid'); + foreach ($events as $k => $event) { + $uuid = $event['uuid']; + if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) { + unset($events[$k]); + } + } + } + + /** + * @param int $serverId + * @param array $event + * @return void + * @throws RedisException + */ + private function addEmptyEvent($serverId, array $event) + { + $emptyEventKey = "{$event['Event']['uuid']}:{$event['Event']['timestamp']}"; + $redis = RedisTool::init(); + $redis->sAdd("misp:empty_events:$serverId", $emptyEventKey); + $redis->expire("misp:empty_events:$serverId", 24 * 3600); + } + + /** + * Remove from $events array events, that was already fetched before and was empty. + * @param int $serverId + * @param array $events + * @return void + */ + private function removeEmptyEvents($serverId, array &$events) + { + try { + $emptyEvents = RedisTool::init()->sMembers("misp:empty_events:$serverId"); + } catch (Exception $e) { + return; + } + if (empty($emptyEvents)) { + return; + } + $emptyEvents = array_flip($emptyEvents); + foreach ($events as $k => $event) { + if (isset($emptyEvents["{$event['uuid']}:{$event['timestamp']}"])) { + unset($events[$k]); + } + } + } + + /** + * Get an array of event UUIDs that are present on the remote server. + * + * @param ServerSyncTool $serverSync + * @param bool $all + * @param bool $ignoreFilterRules Ignore defined server pull rules + * @param bool $force If true, returns all events regardless their update timestamp + * @return array Array of event UUIDs. + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws InvalidArgumentException + */ + private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $force = false) + { + $eventArray = $this->getEventIndexFromServer($serverSync, $ignoreFilterRules); + + if ($all) { + return array_column($eventArray, 'uuid'); + } + + if (Configure::read('MISP.enableEventBlocklisting') !== false) { + $EventBlocklistsTable = $this->fetchTable('EventBlocklists'); + $EventBlocklistsTable->removeBlockedEvents($eventArray); + } + + if (Configure::read('MISP.enableOrgBlocklisting') !== false) { + $OrgBlocklistsTable = $this->fetchTable('OrgBlocklists'); + $OrgBlocklistsTable->removeBlockedEvents($eventArray); + } + + foreach ($eventArray as $k => $event) { + if (1 != $event['published']) { + unset($eventArray[$k]); // do not keep non-published events + } + } + if (!$force) { + $this->removeOlderEvents($eventArray); + $this->removeEmptyEvents($serverSync->serverId(), $eventArray); + } + return array_column($eventArray, 'uuid'); + } + + public function serverEventsOverlap() + { + $servers = $this->find( + 'all', + [ + 'conditions' => ['Server.pull' => 1], + 'order' => ['Server.id ASC'], + 'recursive' => -1, + ] + )->toArray(); + + if (count($servers) < 2) { + return [$servers, []]; + } + + $serverUuids = []; + foreach ($servers as &$server) { + try { + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + $uuids = array_column($this->getEventIndexFromServer($serverSync, true), 'uuid'); + $serverUuids[$server['id']] = array_flip($uuids); + $server['events_count'] = count($uuids); + } catch (Exception $e) { + $this->logException("Could not get event UUIDs for server {$server['id']}", $e); + } + } + unset($server); + + $compared = []; + foreach ($servers as $server) { + if (!isset($serverUuids[$server['id']])) { + continue; + } + + foreach ($servers as $server2) { + if ($server['id'] == $server2['id']) { + continue; + } + if (!isset($serverUuids[$server2['id']])) { + continue; + } + + $intersect = count(array_intersect_key($serverUuids[$server['id']], $serverUuids[$server2['id']])); + $percentage = round(100 * $intersect / $server['events_count']); + $compared[$server['id']][$server2['id']] = [ + 'percentage' => $percentage, + 'events' => $intersect, + ]; + } + } + return [$servers, $compared]; + } + + /** + * @param int $id Server ID + * @param string|int $technique Can be 'full', 'incremental' or event ID + * @param int|false $jobId + * @param HttpSocket $HttpSocket + * @param array $user + * @return array|bool + * @throws Exception + */ + public function push($id, $technique, $jobId, $HttpSocket, array $user) + { + $jobId = $jobId ?? false; + + $technique = $technique ?? 'full'; + + if ($jobId) { + $JobsTable = $this->fetchTable('Jobs'); + } + $server = $this->get($id); + if (!$server) { + throw new NotFoundException('Server not found'); + } + $serverSync = new ServerSyncTool($server->toArray(), $this->setupSyncRequest($server->toArray())); + + $EventsTable = $this->fetchTable('Events'); + $url = $server['url']; + $push = $this->checkVersionCompatibility($server->toArray(), $user, $serverSync); + if (is_array($push) && !$push['canPush'] && !$push['canSight']) { + $push = 'Remote instance is outdated or no permission to push.'; + } + if (!is_array($push)) { + $message = __('Push to server {0} failed. Reason: {1}', $id, $push); + $LogsTable = $this->fetchTable('Logs'); + $LogsTable->saveOrFailSilently( + [ + 'org' => $user['Organisation']['name'], + 'model' => 'Server', + 'model_id' => $id, + 'email' => $user['email'], + 'action' => 'error', + 'user_id' => $user['id'], + 'title' => 'Failed: Push to ' . $url . ' initiated by ' . $user['email'], + 'change' => $message + ] + ); + if ($jobId) { + $JobsTable->saveStatus($jobId, false, $message); + } + return $push; + } + + // sync events if user is capable and server is configured for push + if ($push['canPush'] && $server['push']) { + $successes = []; + if ("full" == $technique) { + $eventid_conditions_key = 'Events.id >'; + $eventid_conditions_value = 0; + } elseif ("incremental" == $technique) { + $eventid_conditions_key = 'Events.id >'; + $eventid_conditions_value = $server['lastpushedid']; + } elseif (intval($technique) !== 0) { + $eventid_conditions_key = 'Events.id'; + $eventid_conditions_value = intval($technique); + } else { + throw new InvalidArgumentException("Technique parameter must be 'full', 'incremental' or event ID."); + } + + // sync custom galaxy clusters if user is capable + if ($push['canEditGalaxyCluster'] && $server['push_galaxy_clusters'] && "full" == $technique) { + $clustersSuccesses = $this->syncGalaxyClusters($serverSync, $server->toArray(), $user, $technique = 'full'); + } else { + $clustersSuccesses = []; + } + $successes = array_merge($successes, $clustersSuccesses); + + $sgs = $EventsTable->SharingGroup->find( + 'all', + [ + 'recursive' => -1, + 'contain' => ['Organisations', 'SharingGroupOrgs' => ['Organisations'], 'SharingGroupServers'] + ] + ); + $sgIds = []; + foreach ($sgs as $sg) { + if ($EventsTable->SharingGroup->checkIfServerInSG($sg, $server)) { + $sgIds[] = $sg['id']; + } + } + if (empty($sgIds)) { + $sgIds = [-1]; + } + $eventReportQuery = 'EXISTS (SELECT id, deleted FROM event_reports WHERE event_reports.event_id = Events.id and event_reports.deleted = 0)'; + $findParams = [ + 'conditions' => [ + $eventid_conditions_key => $eventid_conditions_value, + 'Events.published' => 1, + 'OR' => [ + [ + ['Events.attribute_count >' => 0], + [$eventReportQuery] + ], + [ + 'AND' => [ + ['Events.distribution >' => 0], + ['Events.distribution <' => 4], + ], + ], + [ + 'AND' => [ + 'Events.distribution' => 4, + 'Events.sharing_group_id IN' => $sgIds + ], + ] + ] + ], // array of conditions + 'recursive' => -1, //int + 'contain' => ['EventTags' => ['fields' => ['EventTags.tag_id', 'EventTags.event_id']]], + 'fields' => ['Events.id', 'Events.timestamp', 'Events.sighting_timestamp', 'Events.uuid', 'Events.orgc_id'], // array of field names + ]; + $eventIds = $EventsTable->find('all', $findParams)->toArray(); + $eventUUIDsFiltered = $this->getEventIdsForPush($server->toArray(), $serverSync, $eventIds); + if (!empty($eventUUIDsFiltered)) { + $eventCount = count($eventUUIDsFiltered); + // now process the $eventIds to push each of the events sequentially + $fails = []; + foreach ($eventUUIDsFiltered as $k => $eventUuid) { + $params = []; + if (!empty($server['push_rules'])) { + if (!empty($server['push_rules']['tags']['NOT'])) { + $params['blockedAttributeTags'] = $server['push_rules']['tags']['NOT']; + } + } + $params = array_merge( + $params, + [ + 'event_uuid' => $eventUuid, + 'includeAttachments' => true, + 'includeAllTags' => true, + 'deleted' => [0, 1], + 'excludeGalaxy' => 1 + ] + ); + if (empty($server['push_sightings'])) { + $params['noSightings'] = 1; + } + $event = $EventsTable->fetchEvent($user, $params); + $event = $event[0]; + $event['locked'] = 1; + + // Check if remote server supports galaxy cluster push, is set to push and if event will be pushed to + // server + $pushGalaxyClustersForEvent = $push['canEditGalaxyCluster'] && + $server['push_galaxy_clusters'] && + "full" !== $technique && + $EventsTable->shouldBePushedToServer($event, $server->toArray()); + + if ($pushGalaxyClustersForEvent) { + $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique = $event['id'], $event = $event); + } + + $result = $EventsTable->uploadEventToServer($event, $server->toArray(), $serverSync); + if ('Success' === $result) { + $successes[] = $event['id']; + } else { + $fails[$event['id']] = $result; + } + if ($jobId && $k % 10 == 0) { + $JobsTable->saveProgress($jobId, null, 100 * $k / $eventCount); + } + } + if (count($fails) > 0) { + // there are fails, take the lowest fail + $lastpushedid = min(array_keys($fails)); + } else { + // no fails, take the highest success + $lastpushedid = max($successes); + } + // increment lastid based on the highest ID seen + // Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields. + $server['lastpushedid'] = $lastpushedid; + $this->save($server); + } + $this->syncProposals($HttpSocket, $server->toArray(), null, null, $EventsTable); + } + + if ($push['canPush'] || $push['canSight']) { + $SightingsTable = $this->fetchTable('Sightings'); + $sightingSuccesses = $SightingsTable->pushSightings($user, $serverSync); + } else { + $sightingSuccesses = []; + } + + if (!isset($successes)) { + $successes = $sightingSuccesses; + } else { + $successes = array_merge($successes, $sightingSuccesses); + } + if (!isset($fails)) { + $fails = []; + } + + $LogsTable = $this->fetchTable('Logs'); + $LogsTable->saveOrFailSilently( + [ + 'org' => $user['Organisation']['name'], + 'model' => 'Server', + 'model_id' => $id, + 'email' => $user['email'], + 'action' => 'push', + 'user_id' => $user['id'], + 'title' => 'Push to ' . $url . ' initiated by ' . $user['email'], + 'change' => count($successes) . ' events pushed or updated. ' . count($fails) . ' events failed or didn\'t need an update.' + ] + ); + if ($jobId) { + $JobsTable->saveStatus($jobId, true, __('Push to server {0} complete.', $id)); + } else { + return [$successes, $fails]; + } + return true; + } + + /** + * @param array $server + * @param ServerSyncTool $serverSync + * @param array $events + * @return array|false + */ + private function getEventIdsForPush(array $server, ServerSyncTool $serverSync, array $events) + { + $request = []; + foreach ($events as $event) { + if (empty($this->eventFilterPushableServers($event, [$server]))) { + continue; + } + $request[] = [ + 'Event' => [ + 'uuid' => $event['uuid'], + 'timestamp' => $event['timestamp'], + ] + + ]; + } + + if (empty($request)) { + return []; + } + + try { + return $serverSync->filterEventIdsForPush($request)->getJson(); + } catch (Exception $e) { + $this->logException("Could not filter events for push when pushing to server {$serverSync->serverId()}", $e); + return false; + } + } + + /** + * syncGalaxyClusters Push eligible clusters depending on the provided technique + * + * @param ServerSyncTool $serverSync + * @param array $server + * @param array $user + * @param string|int $technique Either the 'full' string or the event id + * @param array|bool $event + * @return array List of successfully pushed clusters + */ + public function syncGalaxyClusters(ServerSyncTool $serverSync, array $server, array $user, $technique = 'full', $event = false) + { + if (!$server['push_galaxy_clusters']) { + return []; // pushing clusters is not enabled + } + + $this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO); + + $GalaxyClustersTable = $this->fetchTable('GalaxyClusters'); + + if ($technique === 'full') { + $clusters = $GalaxyClustersTable->getElligibleClustersToPush($user, $conditions = [], $full = true); + } else { + if ($event === false) { + throw new InvalidArgumentException('The event from which the cluster should be taken must be provided.'); + } + $tagNames = $this->User->Event->extractAllTagNames($event); + if (empty($tagNames)) { + return []; + } + // Filter out tag names that are not in custom galaxy cluster format + $customGalaxyClusterTags = array_filter( + $tagNames, + function ($tagName) { + return $this->User->Event->EventTag->Tag->isCustomGalaxyClusterTag($tagName); + } + ); + if (empty($customGalaxyClusterTags)) { + return []; + } + $clusters = $GalaxyClustersTable->getElligibleClustersToPush($user, $conditions = ['GalaxyCluster.tag_name' => $customGalaxyClusterTags], $full = true); + } + if (empty($clusters)) { + return []; // no local clusters eligible for push + } + $localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid'); + try { + $clustersToPush = $this->getElligibleClusterIdsFromServerForPush($serverSync, $localClusters = $clusters, $conditions = ['uuid' => $localClusterUUIDs]); + } catch (Exception $e) { + $this->logException("Could not get eligible cluster IDs from server #{$server['id']} for push.", $e); + return []; + } + $successes = []; + foreach ($clustersToPush as $cluster) { + $result = $GalaxyClustersTable->uploadClusterToServer($cluster, $server, $serverSync, $user); + if ($result === 'Success') { + $successes[] = __('GalaxyCluster {0}', $cluster['GalaxyCluster']['uuid']); + } + } + return $successes; + } + + public function syncProposals($HttpSocket, array $server, $sa_id, $event_id, $EventsTable) + { + $sa_id = $sa_id ?? null; + $event_id = $event_id ?? null; + + $ShadowAttributesTable = $this->fetchTable('ShadowAttributes'); + + $HttpSocket = new HttpTool(); + $HttpSocket->configFromServer($server); + + if ($sa_id == null) { + if ($event_id == null) { + // event_id is null when we are doing a push + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + try { + $ids = $this->getEventIdsFromServer($serverSync, true, true); + } catch (Exception $e) { + $this->logException("Could not fetch event IDs from server {$server['name']}", $e); + return false; + } + $conditions = ['uuid IN' => $ids]; + } else { + $conditions = ['id' => $event_id]; + // event_id is not null when we are doing a publish + } + + if (empty($ids)) { + return true; + } + + $events = $EventsTable->find( + 'all', + [ + 'conditions' => $conditions, + 'recursive' => 1, + 'contain' => 'ShadowAttributes', + 'fields' => ['uuid'] + ] + )->toArray(); + + $fails = 0; + $success = 0; + $error_message = ""; + foreach ($events as $k => &$event) { + if (!empty($event['ShadowAttribute'])) { + foreach ($event['ShadowAttribute'] as &$sa) { + $sa['data'] = $ShadowAttributesTable->base64EncodeAttachment($sa); + unset($sa['id']); + unset($sa['value1']); + unset($sa['value2']); + } + + $data = json_encode($event['ShadowAttribute']); + $request = $this->setupSyncRequest($server); + $uri = $server['url'] . '/events/pushProposals/' . $event['Event']['uuid']; + $response = $HttpSocket->post($uri, $data, $request); + if ($response->getStatusCode() === 200) { + $result = $response->getJson(); + if ($result['success']) { + $success += intval($result['counter']); + } else { + $fails++; + if ($error_message == "") { + $result['message']; + } else { + $error_message .= " --- " . $result['message']; + } + } + } else { + $fails++; + } + } + } + } else { + // connect to checkuuid($uuid) + $request = $this->setupSyncRequest($server); + $uri = $server['url'] . '/events/checkuuid/' . $sa_id; + $response = $HttpSocket->get($uri, '', $request); + if ($response->getStatusCode() !== 200) { + return false; + } + } + return true; + } + + /** + * @return array + */ + public function getCurrentServerSettings() + { + $serverSettings = $this->serverSettings; + $moduleTypes = ['Enrichment', 'Import', 'Export', 'Action', 'Cortex']; + return $this->readModuleSettings($serverSettings, $moduleTypes); + } + + /** + * @param array $serverSettings + * @param array $moduleTypes + * @return array + */ + private function readModuleSettings(array $serverSettings, array $moduleTypes) + { + $ModulesTable = $this->fetchTable('Modules'); + foreach ($moduleTypes as $moduleType) { + if (Configure::read('Plugin.' . $moduleType . '_services_enable')) { + $results = $ModulesTable->getModuleSettings($moduleType); + foreach ($results as $module => $data) { + foreach ($data as $result) { + $setting = ['level' => 1, 'errorMessage' => '']; + if ($result['type'] === 'boolean') { + $setting['test'] = 'testBool'; + $setting['type'] = 'boolean'; + $setting['description'] = __('Enable or disable the {0} module.', $module); + if (!empty($result['description'])) { + $setting['description'] = sprintf( + "[%s%s%s] %s", + '', + $setting['description'], + '', + $result['description'] + ); + } + $setting['value'] = false; + } elseif ($result['type'] === 'orgs') { + $setting['description'] = __('Restrict the {0} module to the given organisation.', $module); + $setting['value'] = 0; + $setting['test'] = 'testLocalOrg'; + $setting['type'] = 'numeric'; + $setting['optionsSource'] = function () { + return $this->loadLocalOrganisations(); + }; + } else { + $setting['test'] = isset($result['test']) ? $result['test'] : 'testForEmpty'; + $setting['type'] = isset($result['type']) ? $result['type'] : 'string'; + $setting['description'] = isset($result['description']) ? $result['description'] : __('Set this required module specific setting.'); + $setting['value'] = isset($result['value']) ? $result['value'] : ''; + } + $serverSettings['Plugin'][$moduleType . '_' . $module . '_' . $result['name']] = $setting; + } + } + } + if (Configure::read('Plugin.Workflow_enable')) { + $WorkflowsTable = $this->fetchTable('Workflows'); + $triggerModules = $WorkflowsTable->getModulesByType('trigger'); + foreach ($triggerModules as $triggerModule) { + $setting = [ + 'level' => 1, + 'description' => __('Enable/disable the `{0}` trigger', $triggerModule['id']), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ]; + $serverSettings['Plugin']['Workflow_triggers_' . $triggerModule['id']] = $setting; + } + } + } + return $serverSettings; + } + + private function __serverSettingsRead($serverSettings, $currentSettings) + { + foreach ($serverSettings as $branchKey => &$branchValue) { + if (isset($branchValue['branch'])) { + foreach ($branchValue as $leafKey => &$leafValue) { + if ($leafKey !== 'branch' && $leafValue['level'] == 3 && !isset($currentSettings[$branchKey][$leafKey])) { + continue; + } + $setting = null; + if (isset($currentSettings[$branchKey][$leafKey])) { + $setting = $currentSettings[$branchKey][$leafKey]; + } + if ($leafKey !== 'branch') { + $leafValue = $this->__evaluateLeaf($leafValue, $leafKey, $setting); + if ($branchKey == 'Plugin') { + $pluginData = explode('_', $leafKey); + $leafValue['subGroup'] = $pluginData[0]; + } + if (strpos($branchKey, 'Secur') === 0) { + $leafValue['tab'] = 'Security'; + } else { + $leafValue['tab'] = $branchKey; + } + $finalSettingsUnsorted[$branchKey . '.' . $leafKey] = $leafValue; + } + } + } else { + $setting = null; + if (isset($currentSettings[$branchKey])) { + $setting = $currentSettings[$branchKey]; + } + $branchValue = $this->__evaluateLeaf($branchValue, $branchKey, $setting); + $branchValue['tab'] = 'misc'; + $finalSettingsUnsorted[$branchKey] = $branchValue; + } + } + return $finalSettingsUnsorted; + } + + private function __sortFinalSettings($finalSettingsUnsorted) + { + $finalSettings = []; + for ($i = 0; $i < 4; $i++) { + foreach ($finalSettingsUnsorted as $k => $s) { + $s['setting'] = $k; + if ($s['level'] == $i) { + $finalSettings[] = $s; + } + } + } + return $finalSettings; + } + + public function serverSettingsRead($unsorted = false) + { + $settingTabMergeRules = [ + 'GnuPG' => 'Encryption', + 'SMIME' => 'Encryption', + 'misc' => 'Security', + 'Security' => 'Security', + 'Session' => 'Security', + 'LinOTPAuth' => 'Security', + 'SimpleBackgroundJobs' => 'SimpleBackgroundJobs' + ]; + + $serverSettings = $this->getCurrentServerSettings(); + $currentSettings = Configure::read(); + $finalSettingsUnsorted = $this->__serverSettingsRead($serverSettings, $currentSettings); + foreach ($finalSettingsUnsorted as $key => $temp) { + if (isset($settingTabMergeRules[$temp['tab']])) { + $finalSettingsUnsorted[$key]['tab'] = $settingTabMergeRules[$temp['tab']]; + } + } + if ($unsorted) { + return $finalSettingsUnsorted; + } + return $this->__sortFinalSettings($finalSettingsUnsorted); + } + + public function serverSettingReadSingle($settingObject, $settingName, $leafKey) + { + $setting = Configure::read($settingName); + $result = $this->__evaluateLeaf($settingObject, $leafKey, $setting); + $result['setting'] = $settingName; + return $result; + } + + /** + * @param array $leafValue + * @param string $leafKey + * @param mixed $setting + * @return array + */ + private function __evaluateLeaf(array $leafValue, $leafKey, $setting) + { + if (isset($setting)) { + if ($setting instanceof EncryptedValue) { + try { + $setting = $setting->decrypt(); + } catch (Exception $e) { + $leafValue['errorMessage'] = 'Could not decrypt.'; + return $leafValue; + } + } + if (!empty($leafValue['test'])) { + if ($leafValue['test'] instanceof Closure) { + $result = $leafValue['test']($setting); + } else { + $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']); + } + if ($result !== true) { + $leafValue['error'] = 1; + if ($result !== false) { + $leafValue['errorMessage'] = $result; + } + } + } + if (isset($leafValue['optionsSource'])) { + $leafValue['options'] = $leafValue['optionsSource'](); + } + if (!isset($leafValue['error']) && isset($leafValue['options']) && !isset($leafValue['options'][$setting])) { + $leafValue['error'] = 1; + $validValues = implode(', ', array_keys($leafValue['options'])); + $leafValue['errorMessage'] = __('Invalid setting `{0}`, valid values are: {1}', $setting, $validValues); + } + + if ($setting !== '') { + $leafValue['value'] = $setting; + } + } else { + if ($leafKey !== 'branch' && (!isset($leafValue['null']) || !$leafValue['null'])) { + $leafValue['error'] = 1; + $leafValue['errorMessage'] = __('Value not set.'); + } + } + return $leafValue; + } + + public function loadAvailableLanguages() + { + $dirs = glob(APP . 'Locale/*', GLOB_ONLYDIR); + $languages = ['eng' => 'eng']; + foreach ($dirs as $dir) { + $dir = str_replace(APP . 'Locale' . DS, '', $dir); + $languages[$dir] = $dir; + } + return $languages; + } + + public function testLanguage($value) + { + $languages = $this->loadAvailableLanguages(); + if (!isset($languages[$value])) { + return __('Invalid language.'); + } + return true; + } + + public function loadTagCollections() + { + $TagCollectionsTable = $this->fetchTable('TagCollections'); + $user = ['Role' => ['perm_site_admin' => 1]]; + $tagCollections = $TagCollectionsTable->fetchTagCollection($user); + $options = [0 => 'None']; + foreach ($tagCollections as $tagCollection) { + $options[intval($tagCollection['TagCollection']['id'])] = $tagCollection['TagCollection']['name']; + } + return $options; + } + + private function loadLocalOrganisations($strict = false) + { + static $localOrgs; + + if ($localOrgs === null) { + $localOrgs = $this->Organisation->find( + 'list', + [ + 'conditions' => ['local' => 1], + 'recursive' => -1, + 'fields' => ['Organisation.id', 'Organisation.name'] + ] + ); + } + + if (!$strict) { + return array_replace([0 => __('No organisation selected.')], $localOrgs); + } + + return $localOrgs; + } + + public function testTagCollections($value) + { + $tag_collections = $this->loadTagCollections(); + if (!isset($tag_collections[intval($value)])) { + return __('Invalid tag_collection.'); + } + return true; + } + + public function testForNumeric($value) + { + if (!is_numeric($value)) { + return __('This setting has to be a number.'); + } + return true; + } + + public function testForPositiveInteger($value) + { + if ((is_int($value) && $value >= 0) || ctype_digit($value)) { + return true; + } + return __('The value has to be a whole number greater or equal 0.'); + } + + public function testForCookieTimeout($value) + { + $numeric = $this->testForNumeric($value); + if ($numeric !== true) { + return $numeric; + } + if ($value < Configure::read('Session.timeout') && $value !== 0) { + return __('The cookie timeout is currently lower than the session timeout. This will invalidate the cookie before the session expires.'); + } + return true; + } + + public function testUuid($value) + { + if (empty($value) || !preg_match('/^\{?[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}\}?$/', $value)) { + return 'Invalid UUID.'; + } + return true; + } + + public function testForSessionDefaults($value) + { + if (empty($value) || !in_array($value, ['php', 'database', 'cake', 'cache'])) { + return 'Please choose a valid session handler. Recommended values: php or database. Alternate options are cake (cakephp file based sessions) and cache.'; + } else { + return true; + } + } + + public function testForCorrelationEngine($value) + { + $options = Configure::read('MISP.correlation_engine.options'); + if (!empty($value) && !in_array($value, array_keys($options))) { + return __('Please select a valid option from the list of available engines: ', implode(', ', array_keys($options))); + } else { + return true; + } + } + + public function testLocalOrg($value) + { + if ($value == 0) { + return true; // `No organisation selected` option + } + + return $this->testLocalOrgStrict($value); + } + + public function testLocalOrgStrict($value) + { + if ($value == 0) { + return 'No organisation selected'; + } + $local_orgs = $this->loadLocalOrganisations(true); + if (in_array($value, array_keys($local_orgs))) { + return true; + } + return 'Invalid organisation'; + } + + public function testForEmpty($value) + { + $value = trim($value); + if ($value === '') { + return 'Value not set.'; + } + return true; + } + + public function testForPath($value) + { + if ($value === '') { + return true; + } + if (preg_match('@^\/?(([a-z0-9_.]+[a-z0-9_.\-.\:]*[a-z0-9_.\-.\:]|[a-z0-9_.])+\/?)+$@i', $value)) { + return true; + } + return 'Invalid characters in the path.'; + } + + public function beforeHookBinExec($setting, $value) + { + return $this->testForBinExec($value); + } + + public function testForBinExec($value) + { + if (substr($value, 0, 7) === "phar://") { + return 'Phar protocol not allowed.'; + } + if ($value === '') { + return true; + } + if (is_executable($value)) { + $finfo = finfo_open(FILEINFO_MIME_TYPE); + $type = finfo_file($finfo, $value); + finfo_close($finfo); + if ($type === "application/x-executable" || $type === "application/x-pie-executable" || $type === "application/x-sharedlib") { + return true; + } else { + return 'Binary file not executable. It is of type: ' . $type; + } + } else { + return 'Binary file not executable.'; + } + } + + public function testForWritableDir($value) + { + if (substr($value, 0, 7) === "phar://") { + return 'Phar protocol not allowed.'; + } + if (substr($value, 0, 5) === "s3://") { + return true; + } + if (!is_dir($value)) { + return 'Not a valid directory.'; + } + if (!is_writeable($value)) { + return 'Not a writable directory.'; + } + return true; + } + + public function testDebug($value) + { + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + if ($this->testForNumeric($value) !== true) { + return 'This setting has to be a number between 0 and 2, with 0 disabling debug mode.'; + } + if ($value === 0) { + return true; + } + return 'This setting has to be set to 0 on production systems. Ignore this warning if this is not the case.'; + } + + public function testDebugAdmin($value) + { + if ($this->testBool($value) !== true) { + return 'This setting has to be either true or false.'; + } + if (!$value) { + return true; + } + return 'Enabling debug is not recommended. Turn this on temporarily if you need to see a stack trace to debug an issue, but make sure this is not left on.'; + } + + public function testDate($date) + { + if ($this->testForEmpty($date) !== true) { + return $this->testForEmpty($date); + } + if (!strtotime($date)) { + return 'The date that you have entered is invalid. Expected: yyyy-mm-dd'; + } + return true; + } + + public function getHost() + { + if (function_exists('apache_request_headers')) { + $headers = apache_request_headers(); + } else { + $headers = $_SERVER; + } + + if (array_key_exists('X-Forwarded-Host', $headers)) { + $host = $headers['X-Forwarded-Host']; + } else { + $host = $_SERVER['HTTP_HOST']; + } + return $host; + } + + public function getProto() + { + if (function_exists('apache_request_headers')) { + $headers = apache_request_headers(); + } else { + $headers = $_SERVER; + } + + if (array_key_exists('X-Forwarded-Proto', $headers)) { + $proto = $headers['X-Forwarded-Proto']; + } else { + $proto = ((!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') || $_SERVER['SERVER_PORT'] == 443) === true ? 'HTTPS' : 'HTTP'; + } + return $proto; + } + + public function validateURL($check) + { + $check = array_values($check); + $check = $check[0]; + return $this->testURL($check); + } + + public function testBaseURL($value) + { + // only run this check via the GUI, via the CLI it won't work + if (php_sapi_name() == 'cli') { + if (!empty($value) && !preg_match('/^http(s)?:\/\//i', $value)) { + return 'Invalid baseurl, please make sure that the protocol is set.'; + } + return true; + } + if (empty($value)) { + return true; + } + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + $regex = "/^(?https?):\/\/(?([\w,\-,\.]+))(?::(?[0-9]+))?(?\/[a-z0-9_\-\.]+)?$/i"; + if ( + !preg_match($regex, $value, $matches) || + strtolower($matches['proto']) != strtolower($this->getProto()) || + ( + strtolower($matches['host']) != strtolower($this->getHost()) && + strtolower($matches['host']) . ':' . $matches['port'] != strtolower($this->getHost()) + ) + ) { + return 'Invalid baseurl, it has to be in the "https://FQDN" format.'; + } + return true; + } + + public function testURL($value) + { + // only run this check via the GUI, via the CLI it won't work + if (!empty($value) && !preg_match('/^http(s)?:\/\//i', $value)) { + return 'Invalid baseurl, please make sure that the protocol is set.'; + } + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + return true; + } + + public function testDisableEmail($value) + { + if (isset($value) && $value) { + return 'E-mailing is blocked.'; + } + return true; + } + + public function testDisableCache($value) + { + if (isset($value) && $value) { + return 'Export caches are disabled.'; + } + return true; + } + + public function testLive($value) + { + if ($this->testBool($value) !== true) { + return $this->testBool($value); + } + if (!$value) { + return 'MISP disabled.'; + } + return true; + } + + public function testBool($value, $errorMessage = false) + { + if ($value !== true && $value !== false) { + if ($errorMessage) { + return $errorMessage; + } + return __('Value is not a boolean, make sure that you convert \'true\' to true for example.'); + } + return true; + } + + public function testBoolTrue($value, $errorMessage = false) + { + if ($this->testBool($value, $errorMessage) !== true) { + return $this->testBool($value, $errorMessage); + } + if ($value === false) { + if ($errorMessage) { + return $errorMessage; + } + return 'It is highly recommended that this setting is enabled. Make sure you understand the impact of having this setting turned off.'; + } else { + return true; + } + } + + public function testBoolFalse($value, $errorMessage = false) + { + if ($this->testBool($value, $errorMessage) !== true) { + return $this->testBool($value, $errorMessage); + } + if ($value !== false) { + if ($errorMessage) { + return $errorMessage; + } + return 'It is highly recommended that this setting is disabled. Make sure you understand the impact of having this setting turned on.'; + } else { + return true; + } + } + + public function testParanoidSkipDb($value) + { + if (!empty(Configure::read('MISP.log_paranoid')) && empty($value)) { + return 'Perhaps consider skipping the database when using paranoid mode. A great number of entries will be added to your log database otherwise that will lead to performance degradation.'; + } + return true; + } + + public function testSalt($value) + { + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + if (strlen($value) < 32) { + return 'The salt has to be an at least 32 byte long string.'; + } + if ($value == "Rooraenietu8Eeyo__testForFile($value, APP . 'files' . DS . 'terms'); + } + + public function testForCABundle($value) + { + $file = new SplFileInfo($value); + if (!$file->isFile()) { + return __('Invalid file path or file not accessible.'); + } + if ($file->getExtension() !== 'pem') { + return __('File has to be in .pem format.'); + } + return true; + } + + public function testForStyleFile($value) + { + if (empty($value)) { + return true; + } + return $this->__testForFile($value, APP . 'webroot' . DS . 'css'); + } + + public function testForCustomImage($value) + { + return $this->__testForFile($value, APP . 'webroot' . DS . 'img' . DS . 'custom'); + } + + public function testPasswordLength($value) + { + $numeric = $this->testForNumeric($value); + if ($numeric !== true) { + return $numeric; + } + if ($value < 0) { + return 'Length cannot be negative, set a positive integer or 0 (to choose the default option).'; + } + return true; + } + + public function testForPortNumber($value) + { + $numeric = $this->testForNumeric($value); + if ($numeric !== true) { + return $numeric; + } + if ($value < 21 || $value > 65535) { + return 'Make sure that you pick a valid port number.'; + } + return true; + } + + public function testForZMQPortNumber($value) + { + $numeric = $this->testForNumeric($value); + if ($numeric !== true) { + return $numeric; + } + if ($value < 49152 || $value > 65535) { + return 'It is recommended that you pick a port number in the dynamic range (49152-65535). However, if you have a valid reason to use a different port, ignore this message.'; + } + return true; + } + + public function testPasswordRegex($value) + { + if (!empty($value) && @preg_match($value, 'test') === false) { + return 'Invalid regex.'; + } + return true; + } + + public function testPasswordResetText($value) + { + if (strpos($value, '$password') === false || strpos($value, '$username') === false || strpos($value, '$misp') === false) { + return 'The text served to the users must include the following replacement strings: "$username", "$password", "$misp"'; + } + return true; + } + + public function testForGPGBinary($value) + { + if (empty($value)) { + $value = $this->serverSettings['GnuPG']['binary']['value']; + } + if (file_exists($value)) { + return true; + } + return 'Could not find the GnuPG executable at the defined location.'; + } + + public function testForRPZDuration($value) + { + if (($this->testForNumeric($value) !== true && preg_match('/^[0-9]*[mhdw]$/i', $value)) || $value >= 0) { + return true; + } else { + return 'Negative seconds found. The following formats are accepted: seconds (positive integer), or duration (positive integer) followed by a letter denoting scale (such as m, h, d, w for minutes, hours, days, weeks)'; + } + } + + public function testForRPZBehaviour($value) + { + $numeric = $this->testForNumeric($value); + if ($numeric !== true) { + return $numeric; + } + if ($value < 0 || $value > 5) { + return 'Invalid setting, valid range is 0-5 (0 = DROP, 1 = NXDOMAIN, 2 = NODATA, 3 = walled garden, 4 = PASSTHRU, 5 = TCP-only.'; + } + return true; + } + + public function sightingsBeforeHook($setting, $value) + { + if ($value == true) { + $this->updateDatabase('addSightings'); + } + return true; + } + + public function email_otpBeforeHook($setting, $value) + { + if ($value && !empty(Configure::read('MISP.disable_emailing'))) { + return __('Emailing is currently disabled. Enabling OTP without e-mailing being configured would lock all users out.'); + } + return true; + } + + public function otpBeforeHook($setting, $value) + { + if ($value && (!class_exists('\OTPHP\TOTP') || !class_exists('\BaconQrCode\Writer'))) { + return __('The TOTP and QR code generation libraries are not installed. Enabling OTP without those libraries installed would lock all users out.'); + } + if ($value && Configure::read('LinOTPAuth.enabled')) { + return __('The TOTP and LinOTPAuth should not be used at the same time.'); + } + return true; + } + + public function testForRPZSerial($value) + { + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + if (!preg_match('/^((\$date(\d*)|\$time|\d*))$/', $value)) { + return 'Invalid format.'; + } + return true; + } + + public function testForRPZNS($value) + { + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + if (!preg_match('/^\w+(\.\w+)*(\.?) \w+(\.\w+)*$/', $value)) { + return 'Invalid format.'; + } + return true; + } + + public function zmqAfterHook($setting, $value) + { + // If we are trying to change the enable setting to false, we don't need to test anything, just kill the server and return true. + if ($setting === 'Plugin.ZeroMQ_enable') { + if ($value == false || $value == 0) { + $this->getPubSubTool()->killService(); + return true; + } + } elseif (!Configure::read('Plugin.ZeroMQ_enable')) { + // If we are changing any other ZeroMQ settings but the feature is disabled, don't reload the service + return true; + } + $this->getPubSubTool()->reloadServer(); + return true; + } + + public function disableCacheAfterHook($setting, $value) + { + if ($value) { + // delete all cache files + foreach (Event::exportTypes() as $type => $settings) { + $files = new DirectoryIterator(APP . 'tmp/cached_exports/' . $type); + // No caches created for this type of export, move on + if ($files == null) { + continue; + } + foreach ($files as $file) { + if ($file->getExtension() === $settings['extension']) { + unlink($file->getPathname()); + } + } + } + } + return true; + } + + public function correlationAfterHook($setting, $value) + { + if (!Configure::read('BackgroundJobs.enabled')) { + $AttributesTable = $this->fetchTable('Attributes'); + if ($value) { + $AttributesTable->purgeCorrelations(); + } else { + $AttributesTable->generateCorrelation(); + } + } else { + if ($value) { + $jobType = 'jobPurgeCorrelation'; + $jobTypeText = 'purge correlations'; + } else { + $jobType = 'jobGenerateCorrelation'; + $jobTypeText = 'generate correlation'; + } + + /** @var Job $job */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + 'SYSTEM', + Job::WORKER_PRIO, + $jobTypeText, + 'All attributes', + 'Job created.' + ); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + $jobType, + $jobId + ], + true, + $jobId + ); + } + return true; + } + + public function ipLogBeforeHook($setting, $value) + { + if ($setting == 'MISP.log_client_ip') { + if ($value == true) { + $this->updateDatabase('addIPLogging'); + } + } + return true; + } + + public function customAuthBeforeHook($setting, $value) + { + if (!empty($value)) { + $this->updateDatabase('addCustomAuth'); + } + $this->cleanCacheFiles(); + return true; + } + + // never come here directly, always go through a secondary check like testForTermsFile in order to also pass along the expected file path + private function __testForFile($value, $path) + { + if ($this->testForEmpty($value) !== true) { + return $this->testForEmpty($value); + } + if (!$this->checkFilename($value)) { + return 'Invalid filename.'; + } + $file = $path . DS . $value; + if (!file_exists($file)) { + return 'Could not find the specified file. Make sure that it is uploaded into the following directory: ' . $path; + } + return true; + } + + private function __serverSettingNormaliseValue($data, $value) + { + if (!empty($data['type'])) { + if ($data['type'] === 'boolean') { + $value = (bool)$value; + } elseif ($data['type'] === 'numeric') { + $value = (int)$value; + } + } + return $value; + } + + /** + * @param string $settingName + * @return array|false False if setting doesn't exists + */ + public function getSettingData($settingName, $withOptions = true) + { + // This is just hack to reset opcache, so for next request cache will be reloaded. + $this->opcacheResetConfig(); + + if (strpos($settingName, 'Plugin.Enrichment') !== false || strpos($settingName, 'Plugin.Import') !== false || strpos($settingName, 'Plugin.Export') !== false || strpos($settingName, 'Plugin.Cortex') !== false || strpos($settingName, 'Plugin.Action') !== false || strpos($settingName, 'Plugin.Workflow') !== false) { + $serverSettings = $this->getCurrentServerSettings(); + } else { + $serverSettings = $this->serverSettings; + } + + $setting = $serverSettings; + $parts = explode('.', $settingName); + foreach ($parts as $part) { + if (isset($setting[$part])) { + $setting = $setting[$part]; + } else { + return false; + } + } + + if (isset($setting['level'])) { + $setting['name'] = $settingName; + if ($withOptions && isset($setting['optionsSource'])) { + $setting['options'] = $setting['optionsSource'](); + } + } + + return $setting; + } + + /** + * @param array|string $user + * @param array $setting + * @param mixed $value + * @param bool $forceSave + * @return mixed|string|true|null + * @throws Exception + */ + public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false) + { + if (isset($setting['beforeHook'])) { + $beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value); + if ($beforeResult !== true) { + $change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.'; + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); + return $beforeResult; + } + } + if ($value !== null) { + $value = trim($value); + if ($setting['type'] === 'boolean') { + $value = (bool)$value; + } else if ($setting['type'] === 'numeric') { + $value = (int)$value; + } + if (isset($setting['test'])) { + if ($setting['test'] instanceof Closure) { + $testResult = $setting['test']($value); + } else { + $testResult = $this->{$setting['test']}($value); + } + } else { + $testResult = true; # No test defined for this setting: cannot fail + } + } else if (isset($setting['null']) && $setting['null']) { + $testResult = true; + } else { + $testResult = __('Value could not be null.'); + } + + if (!$forceSave && $testResult !== true) { + if ($testResult === false) { + $errorMessage = $setting['errorMessage']; + } else { + $errorMessage = $testResult; + } + return $errorMessage; + } + $oldValue = Configure::read($setting['name']); + $fileOnly = isset($setting['cli_only']) && $setting['cli_only']; + $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value, $fileOnly); + if ($settingSaveResult) { + if (SystemSetting::isSensitive($setting['name'])) { + $change = [$setting['name'] => ['*****', '*****']]; + } else { + $change = [$setting['name'] => [$oldValue, $value]]; + } + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting changed', $change); + + // execute after hook + if (isset($setting['afterHook'])) { + if ($setting['afterHook'] instanceof Closure) { + $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue); + } else { + $afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue); + } + if ($afterResult !== true) { + $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); + return $afterResult; + } + } + return true; + } + return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.'); + } + + /** + * @param string $setting + * @param mixed $value + * @param bool $fileOnly If true, always store value in config file even when `MISP.system_setting_db` is enabled + * @return bool + * @throws Exception + */ + public function serverSettingsSaveValue($setting, $value, $fileOnly = false) + { + if (!$fileOnly && Configure::read('MISP.system_setting_db')) { + /** @var SystemSetting $systemSetting */ + $SystemSettingsTable = $this->fetchTable('SystemSettings'); + return $SystemSettingsTable->setSetting($setting, $value); + } + + $configFilePath = APP . 'Config' . DS . 'config.php'; + if (!is_writable($configFilePath)) { + return false; // config file is not writeable + } + + // validate if current config.php is intact: + $current = FileAccessTool::readFromFile($configFilePath); + if (strlen(trim($current)) < 20) { + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Tried to modify server settings but current config is broken.'); + return false; + } + $safeConfigChanges = empty(Configure::read('MISP.server_settings_skip_backup_rotate')); + if ($safeConfigChanges) { + $backupFilePath = APP . 'Config' . DS . 'config.backup.php'; + // Create current config file backup + if (!copy($configFilePath, $backupFilePath)) { + throw new Exception("Could not create config backup `$backupFilePath`."); + } + } + + $settingObject = $this->getSettingData($setting, false); + if ($settingObject) { + $value = $this->__serverSettingNormaliseValue($settingObject, $value); + } + + /** @var array $config */ + require $configFilePath; + if (!isset($config)) { + throw new Exception("Could not load config file `$configFilePath`."); + } + $config = Hash::insert($config, $setting, $value); + + $settingsToSave = [ + 'debug', 'MISP', 'GnuPG', 'SMIME', 'Proxy', 'SecureAuth', + 'Security', 'Session', 'site_admin_debug', 'Plugin', 'CertAuth', + 'ApacheShibbAuth', 'ApacheSecureAuth', 'OidcAuth', 'AadAuth', + 'SimpleBackgroundJobs', 'LinOTPAuth' + ]; + $settingsArray = []; + foreach ($settingsToSave as $setting) { + if (Hash::check($config, $setting)) { + $settingsArray[$setting] = Hash::get($config, $setting); + } + } + $settingsString = var_export($settingsArray, true); + $settingsString = 'logException('Could not create temp config file.', $e); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Could not create temp config file.'); + return false; + } + if (!rename($tmpFile, $configFilePath)) { + FileAccessTool::deleteFile($tmpFile); + throw new Exception("Could not rename `$tmpFile` to config file `$configFilePath`."); + } + $this->opcacheResetConfig(); + chmod($configFilePath, octdec($previous_file_perm)); + $config_saved = FileAccessTool::readFromFile($configFilePath); + // if the saved config file is empty, restore the backup. + if (strlen($config_saved) < 20) { + rename($backupFilePath, $configFilePath); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', 0, 'Error: Something went wrong saving the config file, reverted to backup file.'); + return false; + } else { + FileAccessTool::deleteFile($backupFilePath); + } + } else { + FileAccessTool::writeToFile($configFilePath, $settingsString); + $this->opcacheResetConfig(); + } + return true; + } + + public function getFileRules() + { + return [ + 'orgs' => [ + 'name' => __('Organisation logos'), + 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), + 'expected' => [], + 'valid_format' => __('48x48 pixel .png files'), + 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', + 'regex' => '.*\.(png|PNG)$', + 'regex_error' => __('Filename must be in the following format: *.png'), + 'files' => [], + ], + 'img' => [ + 'name' => __('Additional image files'), + 'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'), + 'expected' => [ + 'MISP.footer_logo' => Configure::read('MISP.footer_logo'), + 'MISP.home_logo' => Configure::read('MISP.home_logo'), + 'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'), + 'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'), + ], + 'valid_format' => __('PNG or SVG file'), + 'path' => APP . 'webroot' . DS . 'img' . DS . 'custom', + 'regex' => '.*\.(png|svg)$', + 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), + 'files' => [], + ], + ]; + } + + public function grabFiles() + { + $validItems = $this->getFileRules(); + foreach ($validItems as $k => $item) { + $dir = new DirectoryIterator($item['path']); + + $files = new RegexIterator($dir, $item['regex']); + foreach ($files as $file) { + $f = new SplFileInfo($item['path'] . DS . $file); + $validItems[$k]['files'][] = [ + 'filename' => $file, + 'filesize' => $f->getSize(), + 'read' => $f->isReadable(), + 'write' => $f->isWritable(), + 'execute' => $f->isExecutable(), + ]; + } + } + return $validItems; + } + + /** + * @param array $server + * @param bool $withPostTest + * @return array + * @throws JsonException + */ + public function runConnectionTest(array $server, $withPostTest = true) + { + try { + $clientCertificate = HttpTool::getServerClientCertificateInfo($server); + if ($clientCertificate) { + $clientCertificate['valid_from'] = $clientCertificate['valid_from'] ? $clientCertificate['valid_from']->format('c') : __('Not defined'); + $clientCertificate['valid_to'] = $clientCertificate['valid_to'] ? $clientCertificate['valid_to']->format('c') : __('Not defined'); + $clientCertificate['public_key_size'] = $clientCertificate['public_key_size'] ?: __('Unknown'); + $clientCertificate['public_key_type'] = $clientCertificate['public_key_type'] ?: __('Unknown'); + } + } catch (Exception $e) { + $clientCertificate = ['error' => $e->getMessage()]; + } + + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + + try { + $info = $serverSync->info(); + $response = [ + 'status' => 1, + 'info' => $info, + 'client_certificate' => $clientCertificate, + ]; + + $connectionMeta = $serverSync->connectionMetaData(); + if (isset($connectionMeta['crypto']['protocol'])) { + $response['tls_version'] = $connectionMeta['crypto']['protocol']; + } + if (isset($connectionMeta['crypto']['cipher_name'])) { + $response['tls_cipher'] = $connectionMeta['crypto']['cipher_name']; + } + + if ($withPostTest) { + $response['post'] = $serverSync->isSupported(ServerSyncTool::FEATURE_POST_TEST) ? $this->runPOSTtest($serverSync) : null; + } + + return $response; + } catch (HttpSocketHttpException $e) { + $response = $e->getResponse(); + if ($e->getCode() === 403) { + return ['status' => 4, 'client_certificate' => $clientCertificate]; + } else if ($e->getCode() === 405) { + try { + $responseText = $e->getResponse()->getJson()['message']; + if ($responseText === 'Your user account is expecting a password change, please log in via the web interface and change it before proceeding.') { + return ['status' => 5, 'client_certificate' => $clientCertificate]; + } elseif ($responseText === 'You have not accepted the terms of use yet, please log in via the web interface and accept them.') { + return ['status' => 6, 'client_certificate' => $clientCertificate]; + } + } catch (Exception $e) { + // pass + } + } + } catch (HttpSocketJsonException $e) { + $response = $e->getResponse(); + } catch (Exception $e) { + $logTitle = 'Error: Connection test failed. Reason: ' . $e->getMessage(); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['id'], $logTitle); + return ['status' => 2, 'client_certificate' => $clientCertificate]; + } + + $logTitle = 'Error: Connection test failed. Returned data is in the change field.'; + $this->loadLog()->createLogEntry( + 'SYSTEM', + 'error', + 'Server', + $server['id'], + $logTitle, + [ + 'response' => ['', $response->getStringBody()], + 'response-code' => ['', $response->getStatusCode()], + ] + ); + return ['status' => 3, 'client_certificate' => $clientCertificate]; + } + + /** + * @param ServerSyncTool $serverSync + * @return array + * @throws Exception + */ + private function runPOSTtest(ServerSyncTool $serverSync) + { + $testFile = file_get_contents(APP . '../tests/Files/test_payload.txt'); + if (!$testFile) { + throw new Exception("Could not load payload for POST test."); + } + + try { + $response = $serverSync->postTest($testFile); + $contentEncoding = $response->getHeader('Content-Encoding'); + $rawBody = $response->body; + $response = $response->getJson(); + } catch (Exception $e) { + $this->logException("Invalid response for remote server {$serverSync->server()['name']} POST test.", $e); + $title = 'Error: POST connection test failed. Reason: ' . $e->getMessage(); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $serverSync->serverId(), $title); + return ['status' => 8]; + } + if (!isset($response['body']['testString']) || $response['body']['testString'] !== $testFile) { + if (!empty($response['body']['testString'])) { + $responseString = $response['body']['testString']; + } else if (!empty($rawBody)) { + $responseString = $rawBody; + } else { + $responseString = __('Response was empty.'); + } + + $title = 'Error: POST connection test failed due to the message body not containing the expected data. Response: ' . PHP_EOL . PHP_EOL . $responseString; + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $serverSync->serverId(), $title); + return ['status' => 9, 'content-encoding' => $contentEncoding]; + } + $headers = ['Accept', 'Content-type']; + foreach ($headers as $header) { + if (!isset($response['headers'][$header]) || $response['headers'][$header] !== 'application/json') { + $responseHeader = isset($response['headers'][$header]) ? $response['headers'][$header] : 'Header was not set.'; + $title = 'Error: POST connection test failed due to a header ' . $header . ' not matching the expected value. Expected: "application/json", received "' . $responseHeader . '"'; + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $serverSync->serverId(), $title); + return ['status' => 10, 'content-encoding' => $contentEncoding]; + } + } + return ['status' => 1, 'content-encoding' => $contentEncoding]; + } + + /** + * @param array $server + * @param array $user + * @param ServerSyncTool|null $serverSync + * @return array|string + * @throws JsonException + */ + public function checkVersionCompatibility(array $server, $user = [], ServerSyncTool $serverSync = null) + { + // for event publishing when we don't have a user. + if (empty($user)) { + $user = 'SYSTEM'; + } + + $serverSync = $serverSync ? $serverSync : new ServerSyncTool($server, $this->setupSyncRequest($server)); + + try { + $remoteVersion = $serverSync->info(); + } catch (Exception $e) { + $this->logException("Connection to the server {$server['id']} has failed", $e); + + if ($e instanceof HttpSocketHttpException) { + $title = 'Error: Connection to the server has failed. Returned response code: ' . $e->getCode(); + } else { + $title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage(); + } + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['id'], $title); + return $title; + } + + $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; + $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; + $canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false; + $remoteVersionString = $remoteVersion['version']; + $remoteVersion = explode('.', $remoteVersion['version']); + if (!isset($remoteVersion[0])) { + $message = __('Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.'); + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['id'], $message); + return $message; + } + $localVersion = $this->checkMISPVersion(); + $protectedMode = version_compare($remoteVersionString, '2.4.156') >= 0; + $response = false; + $success = false; + $issueLevel = "warning"; + if ($localVersion['major'] > $remoteVersion[0]) { + $response = "Sync to Server ('{$server['id']}') aborted. The remote instance's MISP version is behind by a major version."; + } + if ($response === false && $localVersion['major'] < $remoteVersion[0]) { + $response = "Sync to Server ('{$server['id']}') aborted. The remote instance is at least a full major version ahead - make sure you update your MISP instance!"; + } + if ($response === false && $localVersion['minor'] > $remoteVersion[1]) { + $response = "Sync to Server ('{$server['id']}') aborted. The remote instance's MISP version is behind by a minor version."; + } + if ($response === false && $localVersion['minor'] < $remoteVersion[1]) { + $response = "Sync to Server ('{$server['id']}') aborted. The remote instance is at least a full minor version ahead - make sure you update your MISP instance!"; + } + + // if we haven't set a message yet, we're good to go. We are only behind by a hotfix version + if ($response === false) { + $success = true; + } else { + $issueLevel = "error"; + } + if ($response === false && $localVersion['hotfix'] > $remoteVersion[2]) { + $response = "Sync to Server ('{$server['id']}') initiated, but the remote instance is a few hotfixes behind."; + } + if ($response === false && $localVersion['hotfix'] < $remoteVersion[2]) { + $response = "Sync to Server ('{$server['id']}') initiated, but the remote instance is a few hotfixes ahead. Make sure you keep your instance up to date!"; + } + if (empty($response) && $remoteVersion[2] < 111) { + $response = "Sync to Server ('{$server['id']}') initiated, but version 2.4.111 is required in order to be able to pull proposals from the remote side."; + } + + if ($response !== false) { + $this->loadLog()->createLogEntry($user, $issueLevel, 'Server', $server['id'], ucfirst($issueLevel) . ': ' . $response); + } + return [ + 'success' => $success, + 'response' => $response, + 'canPush' => $canPush, + 'canSight' => $canSight, + 'canEditGalaxyCluster' => $canEditGalaxyCluster, + 'version' => $remoteVersion, + 'protectedMode' => $protectedMode, + ]; } public function captureServer($server, $user) @@ -40,27 +2863,864 @@ class ServersTable extends AppTable return $existingServer['id']; } - public function fetchServer($id) + public function dbSpaceUsage() { - if (empty($id)) { + $inMb = function ($value) { + return round($value / 1024 / 1024, 2) . " MB"; + }; + + $result = []; + if ($this->isMysql()) { + $sql = sprintf( + 'select TABLE_NAME, DATA_LENGTH, INDEX_LENGTH, DATA_FREE from information_schema.tables where table_schema = %s group by TABLE_NAME, DATA_LENGTH, INDEX_LENGTH, DATA_FREE;', + "'" . $this->getDataSource()->config['database'] . "'" + ); + $sqlResult = $this->query($sql); + + foreach ($sqlResult as $temp) { + $result[$temp['tables']['TABLE_NAME']] = [ + 'table' => $temp['tables']['TABLE_NAME'], + 'used' => $inMb($temp['tables']['DATA_LENGTH'] + $temp['tables']['INDEX_LENGTH']), + 'reclaimable' => $inMb($temp['tables']['DATA_FREE']), + 'data_in_bytes' => (int) $temp['tables']['DATA_LENGTH'], + 'index_in_bytes' => (int) $temp['tables']['INDEX_LENGTH'], + 'reclaimable_in_bytes' => (int) $temp['tables']['DATA_FREE'], + ]; + } + } else { + $sql = sprintf( + 'select TABLE_NAME as table, pg_total_relation_size(%s||%s||TABLE_NAME) as used from information_schema.tables where table_schema = %s group by TABLE_NAME;', + "'" . $this->getDataSource()->config['database'] . "'", + "'.'", + "'" . $this->getDataSource()->config['database'] . "'" + ); + $sqlResult = $this->query($sql); + foreach ($sqlResult as $temp) { + foreach ($temp[0] as $k => $v) { + if ($k == "table") { + continue; + } + $temp[0][$k] = $inMb($v); + } + $temp[0]['reclaimable'] = '0 MB'; + $result[] = $temp[0]; + } + } + return $result; + } + + public function redisInfo() + { + $output = [ + 'extensionVersion' => phpversion('redis'), + 'connection' => false, + ]; + + try { + $redis = RedisTool::init(); + $output['connection'] = true; + $output = array_merge($output, $redis->info()); + } catch (Exception $e) { + $output['connection_error'] = $e->getMessage(); + } + + return $output; + } + + public function dbSchemaDiagnostic() + { + $AdminSettingsTable = $this->fetchTable('AdminSettings'); + $actualDbVersion = $AdminSettingsTable->getSetting('db_version'); + $dataSource = $this->getDataSource()->config['datasource']; + $schemaDiagnostic = [ + 'dataSource' => $dataSource, + 'actual_db_version' => $actualDbVersion, + 'checked_table_column' => [], + 'diagnostic' => [], + 'diagnostic_index' => [], + 'expected_db_version' => '?', + 'error' => '', + 'update_locked' => $this->isUpdateLocked(), + 'remaining_lock_time' => $this->getLockRemainingTime(), + 'update_fail_number_reached' => $this->UpdateFailNumberReached(), + 'indexes' => [] + ]; + if ($this->isMysql()) { + $dbActualSchema = $this->getActualDBSchema(); + $dbExpectedSchema = $this->getExpectedDBSchema(); + if ($dbExpectedSchema !== false) { + $db_schema_comparison = $this->compareDBSchema($dbActualSchema['schema'], $dbExpectedSchema['schema']); + $db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes'], $dbExpectedSchema); + $schemaDiagnostic['checked_table_column'] = $dbActualSchema['column']; + $schemaDiagnostic['diagnostic'] = $db_schema_comparison; + $schemaDiagnostic['diagnostic_index'] = $db_indexes_comparison; + $schemaDiagnostic['expected_db_version'] = $dbExpectedSchema['db_version']; + foreach ($dbActualSchema['schema'] as $tableName => $tableMetas) { + foreach ($tableMetas as $tableMeta) { + $schemaDiagnostic['columnPerTable'][$tableName][] = $tableMeta['column_name']; + } + } + $schemaDiagnostic['indexes'] = $dbActualSchema['indexes']; + } else { + $schemaDiagnostic['error'] = 'Diagnostic not available as the expected schema file could not be loaded'; + } + } else { + $schemaDiagnostic['error'] = sprintf('Diagnostic not available for DataSource `%s`', $dataSource); + } + if (!empty($schemaDiagnostic['diagnostic'])) { + foreach ($schemaDiagnostic['diagnostic'] as $table => &$fields) { + foreach ($fields as &$field) { + $field = $this->__attachRecoveryQuery($field, $table); + } + } + } + return $schemaDiagnostic; + } + + /* + * Get RDBMS configuration values + */ + public function dbConfiguration(): array + { + if ($this->isMysql()) { + $configuration = []; + + $dbVariables = $this->query("SHOW VARIABLES;"); + $settings = array_keys(self::MYSQL_RECOMMENDED_SETTINGS); + + foreach ($dbVariables as $dbVariable) { + // different rdbms have different casing + if (isset($dbVariable['SESSION_VARIABLES'])) { + $dbVariable = $dbVariable['SESSION_VARIABLES']; + } elseif (isset($dbVariable['session_variables'])) { + $dbVariable = $dbVariable['session_variables']; + } else { + continue; + } + + if (in_array($dbVariable['Variable_name'], $settings)) { + $configuration[] = [ + 'name' => $dbVariable['Variable_name'], + 'value' => $dbVariable['Value'], + 'default' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['default'], + 'recommended' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['recommended'], + 'explanation' => self::MYSQL_RECOMMENDED_SETTINGS[$dbVariable['Variable_name']]['explanation'], + ]; + } + } + + return $configuration; + } else { + return []; + } + } + + /* + * Work in progress, still needs DEFAULT in the schema for it to work correctly + * Currently only works for missing_column and column_different + * Only currently supported field types are: int, tinyint, varchar, text + */ + private function __attachRecoveryQuery($field, $table) + { + if (isset($field['error_type'])) { + $length = false; + if (in_array($field['error_type'], ['missing_column', 'column_different'])) { + preg_match('/([a-z]+)(?:\((?[0-9,]+)\))?\s*([a-z]+)?/i', $field['expected']['column_type'], $displayWidthMatches); + if (isset($displayWidthMatches['dw'])) { + $length = $displayWidthMatches[2]; + } elseif ($field['expected']['data_type'] === 'int') { + $length = 11; + } elseif ($field['expected']['data_type'] === 'tinyint') { + $length = 1; + } elseif ($field['expected']['data_type'] === 'varchar') { + $length = $field['expected']['character_maximum_length']; + } elseif ($field['expected']['data_type'] === 'text') { + $length = null; + } + } + if ($length !== false) { + switch ($field['error_type']) { + case 'missing_column': + $field['sql'] = sprintf( + 'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s %s %s;', + $table, + $field['column_name'], + $field['expected']['data_type'], + $length !== null ? sprintf('(%d)', $length) : '', + strpos($field['expected']['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', + isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '', + $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', + empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'], + empty($field['expected']['extra']) ? '' : $field['expected']['extra'] + ); + break; + case 'column_different': + $field['sql'] = sprintf( + 'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s %s %s;', + $table, + $field['column_name'], + $field['expected']['data_type'], + $length !== null ? sprintf('(%d)', $length) : '', + strpos($field['expected']['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', + isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '', + $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', + empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'], + empty($field['expected']['extra']) ? '' : $field['expected']['extra'] + ); + break; + } + } elseif ($field['error_type'] == 'missing_table') { + $allFields = []; + foreach ($field['expected_table'] as $expectedField) { + $length = false; + if ($expectedField['data_type'] === 'int') { + $length = 11; + } elseif ($expectedField['data_type'] === 'tinyint') { + $length = 1; + } elseif ($expectedField['data_type'] === 'varchar') { + $length = $expectedField['character_maximum_length']; + } elseif ($expectedField['data_type'] === 'text') { + $length = null; + } + $fieldSql = sprintf( + '`%s` %s%s %s %s %s %s %s', + $expectedField['column_name'], + $expectedField['data_type'], + $length !== null ? sprintf('(%d)', $length) : '', + strpos($expectedField['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', + isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '', + $expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', + empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name'], + empty($field['expected']['extra']) ? '' : $field['expected']['extra'] + ); + $allFields[] = $fieldSql; + } + $field['sql'] = __("% The command below is a suggestion and might be incorrect. Please ask if you are not sure what you are doing.") . "

" . sprintf( + "CREATE TABLE IF NOT EXISTS `%s` ( %s ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;", + $table, + implode(', ', $allFields) + ); + } + } + return $field; + } + + public function getExpectedDBSchema() + { + try { + return FileAccessTool::readJsonFromFile(ROOT . DS . 'db_schema.json'); + } catch (Exception $e) { return false; } - $conditions = ['Servers.id' => $id]; - if (!is_numeric($id)) { - $conditions = ['OR' => [ - 'LOWER(Servers.name)' => strtolower($id), - 'LOWER(Servers.url)' => strtolower($id) - ] + } + + // TODO: Use CakePHP 3.X's Schema System + /* + $db = ConnectionManager::get('default'); + + // Create a schema collection. + $collection = $db->schemaCollection(); + + // Get the table names + $tables = $collection->listTables(); + + // Get a single table (instance of Schema\TableSchema) + $tableSchema = $collection->describe('posts'); + + */ + public function getActualDBSchema( + $tableColumnNames = [ + 'column_name', + 'is_nullable', + 'data_type', + 'character_maximum_length', + 'numeric_precision', + // 'datetime_precision', -- Only available on MySQL 5.6+ + 'collation_name', + 'column_type', + 'column_default', + 'extra', + ] + ) { + $dbActualSchema = []; + $dbActualIndexes = []; + if ($this->isMysql()) { + $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s ORDER BY TABLE_NAME;', "'" . $this->getDataSource()->config['database'] . "'"); + $sqlResult = $this->query($sqlGetTable); + $tables = Hash::extract($sqlResult, '{n}.tables.TABLE_NAME'); + foreach ($tables as $table) { + $sqlSchema = sprintf( + "SELECT %s + FROM information_schema.columns + WHERE table_schema = '%s' AND TABLE_NAME = '%s'", + implode(',', $tableColumnNames), + $this->getDataSource()->config['database'], + $table + ); + $sqlResult = $this->query($sqlSchema)->toArray(); + $sqlResult = array_column($sqlResult, 'columns'); + foreach ($sqlResult as $column_schema) { + $column_schema = array_change_key_case($column_schema, CASE_LOWER); + $dbActualSchema[$table][] = $column_schema; + } + $dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table); + } + } else { + return ['Database/Postgres' => ['description' => __('Can\'t check database schema for Postgres database type')]]; + } + return ['schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes]; + } + + private function compareDBSchema($dbActualSchema, $dbExpectedSchema) + { + // Column that should be ignored while performing the comparison + $allowedlistFields = [ + 'users' => ['external_auth_required', 'external_auth_key'], + ]; + $nonCriticalColumnElements = ['collation_name']; + $dbDiff = []; + // perform schema comparison for tables + foreach ($dbExpectedSchema as $tableName => $columns) { + if (!array_key_exists($tableName, $dbActualSchema)) { + $dbDiff[$tableName][] = [ + 'description' => __('Table `{0}` does not exist', $tableName), + 'error_type' => 'missing_table', + 'expected_table' => $columns, + 'column_name' => $tableName, + 'is_critical' => true + ]; + } else { + // perform schema comparison for table's columns + $expectedColumnKeys = []; + $keyedExpectedColumn = []; + foreach ($columns as $column) { + $expectedColumnKeys[] = $column['column_name']; + $keyedExpectedColumn[$column['column_name']] = $column; + } + $existingColumnKeys = []; + $keyedActualColumn = []; + foreach ($dbActualSchema[$tableName] as $column) { + $existingColumnKeys[] = $column['column_name']; + $keyedActualColumn[$column['column_name']] = $column; + } + + $additionalKeysInActualSchema = array_diff($existingColumnKeys, $expectedColumnKeys); + foreach ($additionalKeysInActualSchema as $additionalKeys) { + if (isset($allowedlistFields[$tableName]) && in_array($additionalKeys, $allowedlistFields[$tableName])) { + continue; // column is allowedlisted + } + $dbDiff[$tableName][] = [ + 'description' => __('Column `{0}` exists but should not', $additionalKeys), + 'error_type' => 'additional_column', + 'column_name' => $additionalKeys, + 'is_critical' => false + ]; + } + foreach ($keyedExpectedColumn as $columnName => $column) { + if (isset($allowedlistFields[$tableName]) && in_array($columnName, $allowedlistFields[$tableName])) { + continue; // column is allowedlisted + } + if (isset($keyedActualColumn[$columnName])) { + $colDiff = array_diff_assoc($column, $keyedActualColumn[$columnName]); + if (count($colDiff) > 0) { + $colElementDiffs = array_keys(array_diff_assoc($column, $keyedActualColumn[$columnName])); + $isCritical = false; + foreach ($colElementDiffs as $colElementDiff) { + if (!in_array($colElementDiff, $nonCriticalColumnElements)) { + if ($colElementDiff == 'column_default') { + $expectedValue = $column['column_default']; + $actualValue = $keyedActualColumn[$columnName]['column_default']; + if (preg_match(sprintf('@(\'|")+%s(\1)+@', $expectedValue), $actualValue) || (empty($expectedValue) && $actualValue === 'NULL')) { // some version of mysql quote the default value + continue; + } else { + $isCritical = true; + break; + } + } else { + $isCritical = true; + break; + } + } + } + $dbDiff[$tableName][] = [ + 'description' => __('Column `{0}` is different', $columnName), + 'column_name' => $column['column_name'], + 'error_type' => 'column_different', + 'actual' => $keyedActualColumn[$columnName], + 'expected' => $column, + 'is_critical' => $isCritical + ]; + } + } else { + $dbDiff[$tableName][] = [ + 'description' => __('Column `{0}` does not exist but should', $columnName), + 'column_name' => $columnName, + 'error_type' => 'missing_column', + 'actual' => [], + 'expected' => $column, + 'is_critical' => true + ]; + } + } + } + } + foreach (array_diff(array_keys($dbActualSchema), array_keys($dbExpectedSchema)) as $additionalTable) { + $dbDiff[$additionalTable][] = [ + 'description' => __('Table `{0}` is an additional table', $additionalTable), + 'column_name' => $additionalTable, + 'error_type' => 'additional_table', + 'is_critical' => false ]; } - $server = $this->find( - 'all', - [ - 'conditions' => $conditions, - 'recursive' => -1 - ] - )->disableHydration()->first(); - return (empty($server)) ? false : $server; + return $dbDiff; + } + + /** + * Returns `true` if given column for given table contains just unique values. + * + * @param string $tableName + * @param string $columnName + * @return bool + */ + private function checkIfColumnContainsJustUniqueValues($tableName, $columnName) + { + $db = $this->getDataSource(); + $duplicates = $this->query( + sprintf( + 'SELECT %s, COUNT(*) c FROM %s GROUP BY %s HAVING c > 1;', + $db->name($columnName), + $db->name($tableName), + $db->name($columnName) + ) + ); + return empty($duplicates); + } + + private function generateSqlDropIndexQuery($tableName, $columnName) + { + return sprintf( + 'DROP INDEX `%s` ON %s;', + $columnName, + $tableName + ); + } + + private function generateSqlIndexQuery(array $dbExpectedSchema, $tableName, $columnName, $shouldBeUnique = false, $defaultIndexKeylength = 255) + { + $columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], "{n}[column_name=$columnName]"); + if (empty($columnData)) { + throw new Exception("Index in db_schema.json is defined for `$tableName.$columnName`, but this column is not defined."); + } + + $columnData = $columnData[0]; + if ($columnData['data_type'] === 'varchar') { + $keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength); + } elseif ($columnData['data_type'] === 'text') { + $keyLength = sprintf('(%s)', $defaultIndexKeylength); + } else { + $keyLength = ''; + } + return sprintf( + 'CREATE%s INDEX `%s` ON `%s` (`%s`%s);', + $shouldBeUnique ? ' UNIQUE' : '', + $columnName, + $tableName, + $columnName, + $keyLength + ); + } + + /** + * @throws Exception + */ + private function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema) + { + $indexDiff = []; + foreach ($expectedIndex as $tableName => $indexes) { + if (!array_key_exists($tableName, $actualIndex)) { + continue; // If table does not exist, it is covered by the schema diagnostic + } + $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes + foreach ($tableIndexDiff as $columnDiff) { + $shouldBeUnique = $indexes[$columnDiff]; + + $message = __('Column `{0}` should be indexed', $columnDiff); + $indexDiff[$tableName][$columnDiff] = [ + 'message' => $message, + 'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique), + ]; + } + $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes + foreach ($tableIndexDiff as $columnDiff) { + $message = __('Column `{0}` is indexed but should not', $columnDiff); + $indexDiff[$tableName][$columnDiff] = [ + 'message' => $message, + 'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff), + ]; + } + foreach ($indexes as $column => $unique) { + if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) { + if ($actualIndex[$tableName][$column]) { + $sql = $this->generateSqlDropIndexQuery($tableName, $column); + $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false); + + $message = __('Column `{0}` has unique index, but should be non unique', $column); + $indexDiff[$tableName][$column] = [ + 'message' => $message, + 'sql' => $sql, + ]; + } else { + $sql = $this->generateSqlDropIndexQuery($tableName, $column); + $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true); + + $message = __('Column `{0}` should be unique index', $column); + $indexDiff[$tableName][$column] = [ + 'message' => $message, + 'sql' => $sql, + ]; + } + } + } + } + return $indexDiff; + } + + /** + * Returns indexes for given schema and table in array, where key is column name and value is `true` if + * index is index is unique, `false` otherwise. + * + * @param string $database + * @param string $table + * @return array + */ + private function getDatabaseIndexes($database, $table) + { + $sqlTableIndex = sprintf( + "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER BY COLUMN_NAME;", + $database, + $table + ); + $sqlTableIndexResult = $this->query($sqlTableIndex); + $output = []; + foreach ($sqlTableIndexResult as $index) { + $output[$index['statistics']['COLUMN_NAME']] = $index['statistics']['NON_UNIQUE'] == 0; + } + return $output; + } + + public function writeableDirsDiagnostics(&$diagnostic_errors) + { + $writeableDirs = [ + '/tmp' => 0, + APP . 'tmp' => 0, + APP . 'files' => 0, + APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0, + APP . 'tmp' . DS . 'csv_all' => 0, + APP . 'tmp' . DS . 'csv_sig' => 0, + APP . 'tmp' . DS . 'md5' => 0, + APP . 'tmp' . DS . 'sha1' => 0, + APP . 'tmp' . DS . 'snort' => 0, + APP . 'tmp' . DS . 'suricata' => 0, + APP . 'tmp' . DS . 'text' => 0, + APP . 'tmp' . DS . 'xml' => 0, + APP . 'tmp' . DS . 'files' => 0, + APP . 'tmp' . DS . 'logs' => 0, + APP . 'tmp' . DS . 'bro' => 0, + ]; + + $attachmentDir = Configure::read('MISP.attachments_dir'); + if ($attachmentDir && !isset($writeableDirs[$attachmentDir])) { + $writeableDirs[$attachmentDir] = 0; + } + + $tmpDir = Configure::read('MISP.tmpdir'); + if ($tmpDir && !isset($writeableDirs[$tmpDir])) { + $writeableDirs[$tmpDir] = 0; + } + + foreach ($writeableDirs as $path => &$error) { + if (!file_exists($path)) { + // Try to create directory if not exists + if (!mkdir($path, 0700, true)) { + $error = 1; + } + } + if (!is_writable($path)) { + $error = 2; + } + if ($error !== 0) { + $diagnostic_errors++; + } + } + return $writeableDirs; + } + + public function writeableFilesDiagnostics(&$diagnostic_errors) + { + $writeableFiles = [ + APP . 'Config' . DS . 'config.php' => 0, + ROOT . DS . '.git' . DS . 'ORIG_HEAD' => 0, + ]; + foreach ($writeableFiles as $path => &$error) { + if (!file_exists($path)) { + $error = 1; + continue; + } + if (!is_writeable($path)) { + $error = 2; + $diagnostic_errors++; + } + } + return $writeableFiles; + } + + public function readableFilesDiagnostics(&$diagnostic_errors) + { + $readableFiles = [ + APP . 'files' . DS . 'scripts' . DS . 'stixtest.py' => 0 + ]; + foreach ($readableFiles as $path => &$error) { + if (!is_readable($path)) { + $error = 1; + continue; + } + } + return $readableFiles; + } + + public function yaraDiagnostics(&$diagnostic_errors) + { + $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'yaratest.py'; + try { + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); + $scriptResult = JsonTool::decode($scriptResult); + } catch (Exception $exception) { + $this->logException('Failed to run yara diagnostics.', $exception); + return [ + 'operational' => 0, + 'plyara' => 0, + 'test_run' => false + ]; + } + return ['operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara'], 'test_run' => true]; + } + + public function stixDiagnostics(&$diagnostic_errors) + { + $expected = ['stix' => '>1.2.0.11', 'cybox' => '>2.1.0.21', 'mixbox' => '>1.0.5', 'maec' => '>4.1.0.17', 'stix2' => '>3.0.0', 'pymisp' => '>2.4.120']; + // check if the STIX and Cybox libraries are working using the test script stixtest.py + $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'; + try { + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); + } catch (Exception $exception) { + $this->logException('Failed to run STIX diagnostics.', $exception); + return [ + 'operational' => 0, + 'invalid_version' => false, + 'test_run' => false + ]; + } + + try { + $scriptResult = JsonTool::decode($scriptResult); + } catch (Exception $e) { + $this->logException('Invalid JSON returned from stixtest', $e); + return [ + 'operational' => -1, + 'invalid_version' => false, + 'stix' => ['expected' => $expected['stix']], + 'cybox' => ['expected' => $expected['cybox']], + 'mixbox' => ['expected' => $expected['mixbox']], + 'maec' => ['expected' => $expected['maec']], + 'stix2' => ['expected' => $expected['stix2']], + 'pymisp' => ['expected' => $expected['pymisp']] + ]; + } + $scriptResult['operational'] = $scriptResult['success']; + if ($scriptResult['operational'] == 0) { + $diagnostic_errors++; + } + $result = [ + 'operational' => $scriptResult['operational'], + 'invalid_version' => false, + 'test_run' => true + ]; + foreach ($expected as $package => $expectedVersion) { + $result[$package]['version'] = $scriptResult[$package]; + $result[$package]['expected'] = $expectedVersion; + if ($expectedVersion[0] === '>') { + $result[$package]['status'] = version_compare($result[$package]['version'], trim($expectedVersion, '>')) >= 0 ? 1 : 0; + } else { + $result[$package]['status'] = $result[$package]['version'] === $expectedVersion ? 1 : 0; + } + if ($result[$package]['status'] == 0) { + $diagnostic_errors++; + $result['invalid_version'] = true; + } + } + return $result; + } + + /** + * @param int $diagnostic_errors + * @return array + */ + public function gpgDiagnostics(&$diagnostic_errors) + { + $output = ['status' => 0, 'version' => null]; + if (Configure::read('GnuPG.email') && Configure::read('GnuPG.homedir')) { + try { + $gpg = GpgTool::initializeGpg(); + } catch (Exception $e) { + $this->logException("Error during initializing GPG.", $e, LOG_NOTICE); + $output['status'] = 2; + } + if ($output['status'] === 0) { + try { + $output['version'] = $gpg->getVersion(); + } catch (Exception $e) { + // ignore + } + + try { + $gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password')); + } catch (Exception $e) { + $this->logException("Error during adding GPG signing key.", $e, LOG_NOTICE); + $output['status'] = 3; + } + } + if ($output['status'] === 0) { + try { + $gpg->sign('test', \Crypt_GPG::SIGN_MODE_CLEAR); + } catch (Exception $e) { + $this->logException("Error during GPG signing.", $e, LOG_NOTICE); + $output['status'] = 4; + } + } + } else { + $output['status'] = 1; + } + if ($output['status'] !== 0) { + $diagnostic_errors++; + } + return $output; + } + + public function zmqDiagnostics(&$diagnostic_errors) + { + if (!Configure::read('Plugin.ZeroMQ_enable')) { + return 1; + } + $pubSubTool = $this->getPubSubTool(); + try { + $isInstalled = $pubSubTool->checkIfPythonLibInstalled(); + } catch (Exception $e) { + $this->logException('ZMQ is not properly installed.', $e, LOG_NOTICE); + $diagnostic_errors++; + return 2; + } + + if (!$isInstalled) { + $diagnostic_errors++; + return 2; + } + if ($pubSubTool->checkIfRunning()) { + return 0; + } + $diagnostic_errors++; + return 3; + } + + public function moduleDiagnostics(&$diagnostic_errors, $type = 'Enrichment') + { + $ModulesTable = $this->fetchTable('Modules'); + $diagnostic_errors++; + if (Configure::read('Plugin.' . $type . '_services_enable')) { + try { + $result = $ModulesTable->getModules($type, true); + } catch (Exception $e) { + return $e->getMessage(); + } + if (empty($result)) { + return 2; + } + $diagnostic_errors--; + return 0; + } + return 1; + } + + public function proxyDiagnostics(&$diagnostic_errors) + { + $proxyStatus = 0; + $proxy = Configure::read('Proxy'); + if (!empty($proxy['host'])) { + try { + $HttpSocket = $this->setupHttpSocket(null); + $proxyResponse = $HttpSocket->get('https://www.github.com/'); + } catch (Exception $e) { + $proxyStatus = 2; + } + if (empty($proxyResponse) || $proxyResponse->code > 399) { + $proxyStatus = 2; + } + } else { + $proxyStatus = 1; + } + if ($proxyStatus > 1) { + $diagnostic_errors++; + } + return $proxyStatus; + } + + public function sessionDiagnostics(&$diagnostic_errors = 0) + { + $sessionCount = null; + $sessionHandler = null; + + switch (Configure::read('Session.defaults')) { + case 'php': + $sessionHandler = 'php_' . ini_get('session.save_handler'); + switch ($sessionHandler) { + case 'php_files': + $diagnostic_errors++; + $errorCode = 2; + break; + case 'php_redis': + $errorCode = 0; + break; + default: + $diagnostic_errors++; + $errorCode = 8; + break; + } + break; + case 'database': + $sessionHandler = 'database'; + $sql = 'SELECT COUNT(id) AS session_count FROM cake_sessions WHERE expires < ' . time() . ';'; + $sqlResult = $this->query($sql); + if (isset($sqlResult[0][0])) { + $sessionCount = $sqlResult[0][0]['session_count']; + $errorCode = 0; + } else { + $errorCode = 9; + } + if ($sessionCount > 1000) { + $diagnostic_errors++; + $errorCode = 1; + } + break; + default: + $diagnostic_errors++; + $errorCode = 8; + break; + } + + return [ + 'handler' => $sessionHandler, + 'expired_count' => $sessionCount, + 'error_code' => $errorCode + ]; } /** @@ -145,4 +3805,1275 @@ class ServersTable extends AppTable return $worker_array; } + + public function backgroundJobsDiagnostics(&$diagnostic_errors) + { + $backgroundJobsStatus = $this->getBackgroundJobsTool()->getStatus(); + + if ($backgroundJobsStatus > 0) { + $diagnostic_errors++; + } + return $backgroundJobsStatus; + } + + public function retrieveCurrentSettings($branch, $subString) + { + $settings = []; + foreach ($this->serverSettings[$branch] as $settingName => $setting) { + if (strpos($settingName, $subString) !== false) { + $settings[$settingName] = $setting['value']; + if (Configure::read('Plugin.' . $settingName)) { + $settings[$settingName] = Configure::read('Plugin.' . $settingName); + } + if (isset($setting['options'])) { + $settings[$settingName] = $setting['options'][$settings[$settingName]]; + } + } + } + return $settings; + } + + /** + * Return PHP setting in basic unit (bytes). + * @param string $setting + * @return string|int|null + */ + public function getIniSetting($setting) + { + $value = ini_get($setting); + if ($value === '') { + return null; + } + + switch ($setting) { + case 'memory_limit': + case 'upload_max_filesize': + case 'post_max_size': + return (int)preg_replace_callback( + '/(-?\d+)(.?)/', + function ($m) { + return $m[1] * pow(1024, strpos('BKMG', $m[2])); + }, + strtoupper($value) + ); + case 'max_execution_time': + return (int)$value; + default: + return $value; + } + } + + public function killWorker($pid, $user) + { + if (!is_numeric($pid)) { + throw new MethodNotAllowedException('Non numeric PID found!'); + } + $workers = $this->getBackgroundJobsTool()->getWorkers(); + foreach ($workers as $worker) { + if ($worker['pid'] == $pid) { + if (substr_count(trim(shell_exec('ps -p ' . $pid)), PHP_EOL) > 0 ? true : false) { + shell_exec('kill ' . $pid . ' > /dev/null 2>&1 &'); + $this->__logRemoveWorker($user, $pid, $worker['queue'], false); + } else { + $this->__logRemoveWorker($user, $pid, $worker['queue'], true); + } + break; + } + } + } + + public function killAllWorkers($user = false, $force = false) + { + $workers = $this->getBackgroundJobsTool()->getWorkers(); + $killed = []; + foreach ($workers as $pid => $worker) { + if (!is_numeric($pid)) { + continue; + } + if (substr_count(trim(shell_exec('ps -p ' . $pid)), PHP_EOL) > 0) { + shell_exec('kill ' . ($force ? ' -9 ' : '') . $pid . ' > /dev/null 2>&1 &'); + $this->__logRemoveWorker($user, $pid, $worker['queue'], false); + } else { + $this->__logRemoveWorker($user, $pid, $worker['queue'], true); + } + } + return $killed; + } + + public function workerRemoveDead($user = false) + { + $workers = $this->getBackgroundJobsTool()->getWorkers(); + $killed = []; + foreach ($workers as $pid => $worker) { + if (!is_numeric($pid)) { + throw new MethodNotAllowedException('Non numeric PID found!'); + } + $pidTest = file_exists('/proc/' . addslashes($pid)); + if (!$pidTest) { + $this->__logRemoveWorker($user, $pid, $worker['queue'], true); + if (empty($killed[$worker['queue']])) { + $killed[$worker['queue']] = 1; + } else { + $killed[$worker['queue']] += 1; + } + } + } + return $killed; + } + + private function __logRemoveWorker($user, $pid, $queue, $dead = false) + { + $LogsTable = $this->fetchTable('Logs'); + if (empty($user)) { + $user = [ + 'id' => 0, + 'Organisation' => [ + 'name' => 'SYSTEM' + ], + 'email' => 'SYSTEM' + ]; + } + $type = $dead ? 'dead' : 'kill'; + $text = [ + 'dead' => [ + 'action' => 'remove_dead_workers', + 'title' => __('Removing a dead worker.'), + 'change' => sprintf(__('Removing dead worker data. Worker was of type {0} with pid {1}'), $queue, $pid) + ], + 'kill' => [ + 'action' => 'stop_worker', + 'title' => __('Stopping a worker.'), + 'change' => sprintf(__('Stopping a worker. Worker was of type {0} with pid {1}'), $queue, $pid) + ] + ]; + $LogsTable->saveOrFailSilently( + [ + 'org' => $user['Organisation']['name'], + 'model' => 'User', + 'model_id' => $user['id'], + 'email' => $user['email'], + 'action' => $text[$type]['action'], + 'user_id' => $user['id'], + 'title' => $text[$type]['title'], + 'change' => $text[$type]['change'] + ] + ); + } + + /** + * Returns an array with the events + * @param array $server + * @param $user - not used + * @param array $passedArgs + * @return array + * @throws Exception + */ + public function previewIndex(array $server, $user, array $passedArgs) + { + $validArgs = array_merge(['sort', 'direction', 'page', 'limit'], $this->validEventIndexFilters); + $urlParams = ''; + foreach ($validArgs as $v) { + if (isset($passedArgs[$v])) { + $urlParams .= '/' . $v . ':' . $passedArgs[$v]; + } + } + + $relativeUri = '/events/index' . $urlParams; + $response = $this->serverGetRequest($server, $relativeUri); + $events = $response->getJson(); + $totalCount = $response->getHeader('X-Result-Count') ?: 0; + + foreach ($events as $k => $event) { + if (!isset($event['Orgc'])) { + $event['Orgc']['name'] = $event['orgc']; + } + if (!isset($event['Org'])) { + $event['Org']['name'] = $event['org']; + } + if (!isset($event['EventTag'])) { + $event['EventTag'] = []; + } + $events[$k] = ['Event' => $event]; + } + + return [$events, $totalCount]; + } + + /** + * Returns an array with the event. + * @param array $server + * @param int $eventId + * @return array + * @throws Exception + */ + public function previewEvent(array $server, $eventId) + { + $relativeUri = '/events/' . $eventId; + $response = $this->serverGetRequest($server, $relativeUri); + $event = $response->getJson(); + + if (!isset($event['Event']['Orgc'])) { + $event['Event']['Orgc']['name'] = $event['Event']['orgc']; + } + if (isset($event['Event']['Orgc'][0])) { + $event['Event']['Orgc'] = $event['Event']['Orgc'][0]; + } + if (!isset($event['Event']['Org'])) { + $event['Event']['Org']['name'] = $event['Event']['org']; + } + if (isset($event['Event']['Org'][0])) { + $event['Event']['Org'] = $event['Event']['Org'][0]; + } + if (!isset($event['Event']['EventTag'])) { + $event['Event']['EventTag'] = []; + } + + return $event; + } + + // Loops through all servers and checks which servers' push rules don't conflict with the given event. + // returns the server objects that would allow the event to be pushed + public function eventFilterPushableServers($event, $servers) + { + $eventTags = []; + $validServers = []; + foreach ($event['EventTag'] as $tag) { + $eventTags[] = $tag['tag_id']; + } + foreach ($servers as $server) { + if (!empty($server['push_rules']['tags']['OR'])) { + $intersection = array_intersect($server['push_rules']['tags']['OR'], $eventTags); + if (empty($intersection)) { + continue; + } + } + if (!empty($server['push_rules']['tags']['NOT'])) { + $intersection = array_intersect($server['push_rules']['tags']['NOT'], $eventTags); + if (!empty($intersection)) { + continue; + } + } + if (!empty($server['push_rules']['orgs']['OR'])) { + if (!in_array($event['orgc_id'], $server['push_rules']['orgs']['OR'])) { + continue; + } + } + if (!empty($server['push_rules']['orgs']['NOT'])) { + if (in_array($event['orgc_id'], $server['push_rules']['orgs']['NOT'])) { + continue; + } + } + $validServers[] = $server; + } + return $validServers; + } + + /** + * Check installed PHP extensions and their versions. + * @return array + * @throws JsonException + */ + public function extensionDiagnostics() + { + try { + $composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json'); + $extensions = []; + $dependencies = []; + foreach ($composer['require'] as $require => $foo) { + if (substr($require, 0, 4) === 'ext-') { + $extensions[substr($require, 4)] = true; + } else if (mb_strpos($require, '/') !== false) { // external dependencies have namespaces, so a / + $dependencies[$require] = true; + } + } + foreach ($composer['suggest'] as $suggest => $reason) { + if (substr($suggest, 0, 4) === 'ext-') { + $extensions[substr($suggest, 4)] = $reason; + } else if (mb_strpos($suggest, '/') !== false) { // external dependencies have namespaces, so a / + $dependencies[$suggest] = $reason; + } + } + } catch (Exception $e) { + $this->logException('Could not load extensions from composer.json', $e, LOG_NOTICE); + $extensions = ['redis' => '', 'gd' => '', 'ssdeep' => '', 'zip' => '', 'intl' => '']; // Default extensions + } + + // check PHP extensions + $results = ['cli' => false]; + foreach ($extensions as $extension => $reason) { + $results['extensions'][$extension] = [ + 'web_version' => phpversion($extension), + 'web_version_outdated' => false, + 'cli_version' => false, + 'cli_version_outdated' => false, + 'required' => $reason === true, + 'info' => $reason === true ? null : $reason, + ]; + } + if (is_readable(APP . DS . 'files' . DS . 'scripts' . DS . 'selftest.php')) { + try { + $execResult = ProcessTool::execute(['php', APP . DS . 'files' . DS . 'scripts' . DS . 'selftest.php', json_encode(array_keys($extensions))]); + } catch (Exception $e) { + // pass + } + if (!empty($execResult)) { + $execResult = JsonTool::decodeArray($execResult); + $results['cli']['phpversion'] = $execResult['phpversion']; + foreach ($execResult['extensions'] as $extension => $loaded) { + $results['extensions'][$extension]['cli_version'] = $loaded; + } + } + } + + // version check + $minimalVersions = [ + 'redis' => '2.2.8', // because of sAddArray method + ]; + foreach ($minimalVersions as $extension => $version) { + if (!isset($results['extensions'][$extension])) { + continue; + } + $results['extensions'][$extension]['required_version'] = $version; + foreach (['web', 'cli'] as $type) { + if ($results['extensions'][$extension][$type . '_version']) { + $outdated = version_compare($results['extensions'][$extension][$type . '_version'], $version, '<'); + $results['extensions'][$extension][$type . '_version_outdated'] = $outdated; + } + } + } + + // check PHP dependencies, installed in the Vendor directory, just check presence of the folder + if (class_exists('\Composer\InstalledVersions')) { + foreach ($dependencies as $dependency => $reason) { + try { + $version = \Composer\InstalledVersions::getVersion($dependency); + } catch (Exception $e) { + $version = false; + } + $results['dependencies'][$dependency] = [ + 'version' => $version, + 'version_outdated' => false, + 'required' => $reason === true, + 'info' => $reason === true ? null : $reason, + ]; + } + } + + return $results; + } + + public function databaseEncodingDiagnostics(&$diagnostic_errors) + { + if (!isset($this->getDataSource()->config['encoding']) || strtolower($this->getDataSource()->config['encoding']) != 'utf8') { + $diagnostic_errors++; + return false; + } + return true; + } + + /** + * @param string $newest + * @return array + * @throws JsonException + */ + private function checkVersion($newest) + { + $version_array = $this->checkMISPVersion(); + $current = implode('.', $version_array); + + $upToDate = version_compare($current, substr($newest, 1)); + if ($newest === null && (Configure::read('MISP.online_version_check') || !Configure::check('MISP.online_version_check'))) { + $upToDate = 'error'; + } elseif ($newest === null && (!Configure::read('MISP.online_version_check') && Configure::check('MISP.online_version_check'))) { + $upToDate = 'disabled'; + } elseif ($upToDate === 0) { + $upToDate = 'same'; + } else { + $upToDate = $upToDate === -1 ? 'older' : 'newer'; + } + return ['current' => 'v' . $current, 'newest' => $newest, 'upToDate' => $upToDate]; + } + + /** + * Fetch latest MISP version from GitHub + * @return array|false + * @throws JsonException + */ + private function checkRemoteVersion($HttpSocket) + { + try { + $tags = GitTool::getLatestTags($HttpSocket); + } catch (Exception $e) { + $this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE); + return false; + } + // find the latest version tag in the v[major].[minor].[hotfix] format + foreach ($tags as $tag) { + if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) { + return $this->checkVersion($tag['name']); + } + } + return false; + } + + /** + * @param bool $checkVersion + * @return array + * @throws JsonException + */ + public function getCurrentGitStatus($checkVersion = false) + { + $latestCommit = false; + + if (Configure::read('MISP.online_version_check') || !Configure::check('MISP.online_version_check')) { + $HttpSocket = $this->setupHttpSocket(null, null, 3); + try { + $latestCommit = GitTool::getLatestCommit($HttpSocket); + } catch (Exception $e) { + $this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE); + } + } + + $output = [ + 'commit' => $this->checkMIPSCommit(), + 'branch' => $this->getCurrentBranch(), + 'latestCommit' => $latestCommit, + ]; + if ($checkVersion) { + $output['version'] = $latestCommit ? $this->checkRemoteVersion($HttpSocket) : $this->checkVersion(null); + } + return $output; + } + + public function getCurrentBranch() + { + try { + return GitTool::currentBranch(); + } catch (Exception $e) { + $this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE); + return false; + } + } + + /** + * Check if MISP update is possible. + * @return bool + */ + public function isUpdatePossible() + { + return $this->getCurrentBranch() !== false && is_writable(APP); + } + + public function checkoutMain() + { + $mainBranch = '2.4'; + return exec('git checkout ' . $mainBranch); + } + + public function getSubmodulesGitStatus() + { + try { + $submodules = GitTool::submoduleStatus(); + } catch (Exception $e) { + $this->logException('Could not fetch git submodules status', $e, LOG_NOTICE); + return []; + } + $status = []; + foreach ($submodules as $submodule) { + if ($this->_isAcceptedSubmodule($submodule['name'])) { + $status[$submodule['name']] = $this->getSubmoduleGitStatus($submodule['name'], $submodule['commit']); + } + } + return $status; + } + + private function _isAcceptedSubmodule($submodule) + { + $accepted_submodules_names = [ + 'PyMISP', + 'app/files/misp-galaxy', + 'app/files/taxonomies', + 'app/files/misp-objects', + 'app/files/noticelists', + 'app/files/warninglists', + 'app/files/misp-decaying-models', + 'app/files/scripts/cti-python-stix2', + 'app/files/scripts/misp-opendata', + 'app/files/scripts/python-maec', + 'app/files/scripts/python-stix', + ]; + return in_array($submodule, $accepted_submodules_names, true); + } + + /** + * @param string $submoduleName + * @param string $superprojectSubmoduleCommitId + * @return array + * @throws Exception + */ + private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId) + { + $path = APP . '../' . $submoduleName; + $submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName); + $submoduleName = end($submoduleName); + + $submoduleCurrentCommitId = GitTool::currentCommit($path); + + $currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path); + if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) { + $remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path); + } else { + $remoteTimestamp = $currentTimestamp; + } + + $status = [ + 'moduleName' => $submoduleName, + 'current' => $submoduleCurrentCommitId, + 'currentTimestamp' => $currentTimestamp, + 'remote' => $superprojectSubmoduleCommitId, + 'remoteTimestamp' => $remoteTimestamp, + 'upToDate' => 'error', + 'isReadable' => is_readable($path) && is_readable($path . '/.git'), + ]; + + if (!empty($status['remote'])) { + if ($status['remote'] === $status['current']) { + $status['upToDate'] = 'same'; + } else if ($status['currentTimestamp'] < $status['remoteTimestamp']) { + $status['upToDate'] = 'older'; + } else { + $status['upToDate'] = 'younger'; + } + } + + if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { + $date1 = new Chronos("@{$status['remoteTimestamp']}"); + $date2 = new Chronos("@{$status['currentTimestamp']}"); + $status['timeDiff'] = $date1->diff($date2); + } else { + $status['upToDate'] = 'error'; + } + + return $status; + } + + public function updateSubmodule($user, $submodule_name = false) + { + $path = APP . '../'; + if ($submodule_name == false) { + $command = sprintf('cd %s; git submodule update --init --recursive 2>&1', $path); + exec($command, $output, $return_code); + $output = implode("\n", $output); + $res = ['status' => ($return_code == 0 ? true : false), 'output' => $output]; + if ($return_code == 0) { // update all DB + $res = array_merge($res, $this->updateDatabaseAfterPullRouter($submodule_name, $user)); + } + } else if ($this->_isAcceptedSubmodule($submodule_name)) { + $command = sprintf('cd %s; git submodule update --init --recursive -- %s 2>&1', $path, $submodule_name); + exec($command, $output, $return_code); + $output = implode("\n", $output); + $res = ['status' => ($return_code == 0 ? true : false), 'output' => $output]; + if ($return_code == 0) { // update DB if necessary + $res = array_merge($res, $this->updateDatabaseAfterPullRouter($submodule_name, $user)); + } + } else { + $res = ['status' => false, 'output' => __('Invalid submodule.'), 'job_sent' => false, 'sync_result' => __('unknown')]; + } + return $res; + } + + public function updateDatabaseAfterPullRouter($submodule_name, $user) + { + if (Configure::read('BackgroundJobs.enabled')) { + /** @var Job $job */ + $JobsTable = $this->fetchTable('Jobs'); + $jobId = $JobsTable->createJob( + $user, + Job::WORKER_PRIO, + 'update_after_pull', + __('Updating: ' . $submodule_name), + 'Update the database after PULLing the submodule(s).' + ); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'updateAfterPull', + $submodule_name, + $jobId, + $user['id'] + ], + true, + $jobId + ); + + return ['job_sent' => true, 'sync_result' => __('unknown')]; + } else { + $result = $this->updateAfterPull($submodule_name, $user['id']); + return ['job_sent' => false, 'sync_result' => $result]; + } + } + + public function updateAfterPull($submodule_name, $userId) + { + $user = $this->User->getAuthUser($userId); + $result = []; + if ($user['Role']['perm_site_admin']) { + $updateAll = empty($submodule_name); + if ($submodule_name == 'app/files/misp-galaxy' || $updateAll) { + $GalaxiesTable = $this->fetchTable('Galaxies'); + $result[] = ($GalaxiesTable->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `' . h($submodule_name) . '` failed.') . PHP_EOL; + } + if ($submodule_name == 'app/files/misp-objects' || $updateAll) { + $ObjectTemplatesTable = $this->fetchTable('ObjectTemplates'); + $result[] = ($ObjectTemplatesTable->update($user, false, false) ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `' . h($submodule_name) . '` failed.') . PHP_EOL; + } + if ($submodule_name == 'app/files/noticelists' || $updateAll) { + $NoticelistsTable = $this->fetchTable('Noticelists'); + $result[] = ($NoticelistsTable->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `' . h($submodule_name) . '` failed.') . PHP_EOL; + } + if ($submodule_name == 'app/files/taxonomies' || $updateAll) { + $TaxonomiesTable = $this->fetchTable('Taxonomies'); + $result[] = ($TaxonomiesTable->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `' . h($submodule_name) . '` failed.') . PHP_EOL; + } + if ($submodule_name == 'app/files/warninglists' || $updateAll) { + $WarninglistsTable = $this->fetchTable('Warninglists'); + $result[] = ($WarninglistsTable->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `' . h($submodule_name) . '` failed.') . PHP_EOL; + } + } + return implode('\n', $result); + } + + public function update(array $status, &$raw = [], array $settings = []) + { + $final = ''; + $workingDirectoryPrefix = 'cd $(git rev-parse --show-toplevel) && '; + $cleanup_commands = [ + // (>^-^)> [hacky] + $workingDirectoryPrefix . 'git checkout app/composer.json 2>&1' + ]; + foreach ($cleanup_commands as $cleanup_command) { + $final .= $cleanup_command . "\n\n"; + $returnCode = false; + exec($cleanup_command, $output, $returnCode); + $raw[] = [ + 'input' => $cleanup_command, + 'output' => $output, + 'status' => $returnCode, + ]; + $final .= implode("\n", $output) . "\n\n"; + } + if (!empty($settings['branch'])) { + $branchname = false; + preg_match('/^[a-z0-9\_]+/i', $settings['branch'], $branchname); + if (!empty($branchname)) { + $branchname = $branchname[0]; + $checkout_command = $workingDirectoryPrefix . 'git checkout ' . escapeshellarg($branchname) . ' 2>&1'; + exec($checkout_command, $output, $returnCode); + $raw[] = [ + 'input' => $checkout_command, + 'output' => $output, + 'status' => $returnCode, + ]; + $status = $this->getCurrentGitStatus(); + } + } + $command1 = $workingDirectoryPrefix . 'git pull origin ' . escapeshellarg($status['branch']) . ' 2>&1'; + $commandSync = $workingDirectoryPrefix . 'git submodule sync 2>&1'; + $command2 = $workingDirectoryPrefix . 'git submodule update --init --recursive 2>&1'; + $final .= $command1 . "\n\n"; + $returnCode = false; + exec($command1, $output, $returnCode); + $raw[] = [ + 'input' => $command1, + 'output' => $output, + 'status' => $returnCode, + ]; + $final .= implode("\n", $output) . "\n\n=================================\n\n"; + + $output = []; + $final .= $commandSync . "\n\n"; + $returnCode = false; + exec($commandSync, $output, $returnCode); + $raw[] = [ + 'input' => $commandSync, + 'output' => $output, + 'status' => $returnCode, + ]; + $final .= implode("\n", $output) . "\n\n=================================\n\n"; + + $output = []; + $final .= $command2 . "\n\n"; + $returnCode = false; + exec($command2, $output, $returnCode); + $raw[] = [ + 'input' => $command2, + 'output' => $output, + 'status' => $returnCode, + ]; + $final .= implode("\n", $output); + return $final; + } + + public function fetchServer($id) + { + if (empty($id)) { + return false; + } + $conditions = ['Servers.id' => $id]; + if (!is_numeric($id)) { + $conditions = [ + 'OR' => [ + 'LOWER(Servers.name)' => strtolower($id), + 'LOWER(Servers.url)' => strtolower($id) + ] + ]; + } + $server = $this->find( + 'all', + [ + 'conditions' => $conditions, + 'recursive' => -1 + ] + )->disableHydration()->first(); + return (empty($server)) ? false : $server; + } + + public function restartWorkers($user = false) + { + if (Configure::read('BackgroundJobs.enabled')) { + $this->workerRemoveDead($user); + $prepend = ''; + shell_exec($prepend . APP . 'Console' . DS . 'worker' . DS . 'start.sh > /dev/null 2>&1 &'); + } + return true; + } + + public function restartDeadWorkers($user = false) + { + if (Configure::read('BackgroundJobs.enabled')) { + $killed = $this->workerRemoveDead($user); + foreach ($killed as $queue => $count) { + for ($i = 0; $i < $count; $i++) { + $this->startWorker($queue); + } + } + } + return true; + } + + public function restartWorker($pid) + { + if (Configure::read('BackgroundJobs.Enabled')) { + $workers = $this->getBackgroundJobsTool()->getWorkers(); + $pid = intval($pid); + if (!isset($workers[$pid])) { + return __('Invalid worker.'); + } + $currentWorker = $workers[$pid]; + $this->killWorker($pid, false); + $this->startWorker($currentWorker['queue']); + return true; + } + return __('Background workers not enabled.'); + } + + public function startWorker($queue) + { + $validTypes = ['default', 'email', 'cache', 'prio', 'update']; + if (!in_array($queue, $validTypes)) { + return __('Invalid worker type.'); + } + + $this->getBackgroundJobsTool()->startWorker($queue); + + return true; + } + + public function cacheServerInitiator($user, $id = 'all', $jobId = false) + { + $redis = RedisTool::init(); + if ($redis === false) { + return 'Redis not reachable.'; + } + $params = [ + 'conditions' => ['caching_enabled' => 1], + ]; + if ($id !== 'all') { + $params['conditions']['id'] = $id; + } else { + $redis->del('misp:server_cache:combined'); + $redis->del($redis->keys('misp:server_cache:event_uuid_lookup:*')); + } + $servers = $this->find('all', $params); + if ($jobId) { + $JobsTable = $this->fetchTable('Jobs'); + $job = $JobsTable->get($jobId); + if (!$job) { + $jobId = false; + } + } + foreach ($servers as $k => $server) { + $this->__cacheInstance($server->toArray(), $redis, $jobId); + if ($jobId) { + $job->progress = 100 * $k / $servers->count(); + $job->message = 'Server ' . $server['id'] . ' cached.'; + $JobsTable->save($job); + } + } + return true; + } + + /** + * @param array $server + * @param Redis $redis + * @param int|false $jobId + * @return bool + * @throws JsonException + */ + private function __cacheInstance($server, $redis, $jobId = false) + { + $serverId = $server['id']; + $i = 0; + $chunk_size = 50000; + $redis->del('misp:server_cache:' . $serverId); + + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + while (true) { + $i++; + $rules = [ + 'returnFormat' => 'cache', + 'includeEventUuid' => 1, + 'page' => $i, + 'limit' => $chunk_size, + ]; + try { + $data = $serverSync->attributeSearch($rules)->getStringBody(); + } catch (Exception $e) { + $this->logException("Could not fetch cached attribute from server {$serverSync->serverId()}.", $e); + break; + } + + $data = trim($data); + if (empty($data)) { + break; + } + + $data = explode(PHP_EOL, $data); + $pipe = $redis->pipeline(); + foreach ($data as $entry) { + list($value, $uuid) = explode(',', $entry); + if (!empty($value)) { + $redis->sAdd('misp:server_cache:' . $serverId, $value); + $redis->sAdd('misp:server_cache:combined', $value); + $redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $serverId . '/' . $uuid); + } + } + $pipe->exec(); + if ($jobId) { + $JobsTable = $this->fetchTable('Jobs'); + $JobsTable->saveProgress($jobId, 'Server ' . $server['id'] . ': ' . ((($i - 1) * $chunk_size) + count($data)) . ' attributes cached.'); + } + } + $redis->set('misp:server_cache_timestamp:' . $serverId, time()); + return true; + } + + /** + * @param array $servers + * @return array + */ + public function attachServerCacheTimestamps(array $servers) + { + $redis = RedisTool::init(); + if ($redis === false) { + return $servers; + } + $redis->pipeline(); + foreach ($servers as $server) { + $redis->get('misp:server_cache_timestamp:' . $server['id']); + } + $results = $redis->exec(); + foreach ($servers as $k => $v) { + $servers[$k]['cache_timestamp'] = $results[$k]; + } + return $servers; + } + + public function updateJSON() + { + $results = []; + foreach (['Galaxies', 'Noticelists', 'Warninglists', 'Taxonomies', 'ObjectTemplates', 'ObjectRelationships'] as $target) { + $Table = $this->fetchTable($target); + $result = $Table->update(); + $results[$target] = $result === false ? false : true; + } + return $results; + } + + public function resetRemoteAuthKey($id) + { + $server = $this->get($id); + if (empty($server)) { + return __('Invalid server'); + } + $serverSync = new ServerSyncTool($server->toArray(), $this->setupSyncRequest($server->toArray())); + + try { + $response = $serverSync->resetAuthKey(); + } catch (Exception $e) { + $message = 'Could not reset the remote authentication key.'; + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $id, 'Error: ' . $message); + return $message; + } + if ($response->isOk()) { + try { + $response = $response->getJson(); + } catch (Exception $e) { + $message = 'Invalid response received from the remote instance.'; + $this->logException($message, $e); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $id, 'Error: ' . $message); + return $message; + } + if (!empty($response['message'])) { + $authkey = $response['message']; + } + if (substr($authkey, 0, 17) === 'Authkey updated: ') { + $authkey = substr($authkey, 17, 57); + } + $server['authkey'] = $authkey; + $this->save($server); + return true; + } else { + return __('Could not reset the remote authentication key.'); + } + } + + public function reprioritise($id = false, $direction = 'up') + { + $servers = $this->find( + 'all', + [ + 'recursive' => -1, + 'order' => ['Server.priority ASC', 'Server.id ASC'] + ] + ); + $success = true; + if ($id) { + foreach ($servers as $k => $server) { + if ($server['id'] && $server['id'] == $id) { + if ( + !($k === 0 && $direction === 'up') && + !(empty($servers[$k + 1]) && $direction === 'down') + ) { + $temp = $servers[$k]; + $destination = $direction === 'up' ? $k - 1 : $k + 1; + $servers[$k] = $servers[$destination]; + $servers[$destination] = $temp; + } else { + $success = false; + } + } + } + } + foreach ($servers as $k => $server) { + $server['priority'] = $k + 1; + $result = $this->save($server); + $success = $success && $result; + } + return $success; + } + + /** + * @param array $server + * @param string $relativeUri + * @return HttpSocketResponseExtended + * @throws Exception + */ + private function serverGetRequest(array $server, $relativeUri) + { + $HttpSocket = $this->setupHttpSocket($server); + $request = $this->setupSyncRequest($server); + + $uri = $server['url'] . $relativeUri; + $response = $HttpSocket->get($uri, [], $request); + if ($response->code == 404) { // intentional != + throw new NotFoundException(__("Fetching the '{0}' failed with HTTP error 404: Not Found", $uri)); + } else if ($response->code == 405) { // intentional != + $responseText = json_decode($response->body, true); + if ($responseText !== null) { + throw new Exception(__("Fetching the '{0}' failed with HTTP error {1}: {2}", $uri, $response->code, $responseText['message'])); + } + } + + if ($response->code != 200) { // intentional != + throw new Exception(__("Fetching the '{0}' failed with HTTP error {1}: {2}", $uri, $response->code, $response->reasonPhrase)); + } + + return $response; + } + + /** + * @param int $id + * @return array|null + * @throws JsonException + */ + public function getRemoteUser($id) + { + $server = $this->find( + 'all', + [ + 'conditions' => ['Server.id' => $id], + 'recursive' => -1 + ] + )->first(); + if (empty($server)) { + return null; // server not found + } + + $serverSync = new ServerSyncTool($server->toArray(), $this->setupSyncRequest($server)); + + try { + $response = $serverSync->userInfo(); + $user = $response->getJson(); + + $results = [ + __('User') => $user['User']['email'], + __('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'), + __('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'), + ]; + if ($response->getHeader('X-Auth-Key-Expiration')) { + $date = new Chronos($response->getHeader('X-Auth-Key-Expiration')); + $results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s'); + } + return $results; + } catch (HttpSocketHttpException $e) { + $this->logException('Could not fetch remote user account.', $e); + return ['error' => $e->getCode()]; + } catch (Exception $e) { + $this->logException('Could not fetch remote user account.', $e); + $message = __('Could not fetch remote user account.'); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $id, 'Error: ' . $message); + return ['error' => $message]; + } + } + + public function __isset($name) + { + if ($name === 'serverSettings' || $name === 'command_line_functions') { + return true; + } + return parent::__isset($name); + } + + /** + * @return int Number of orphans removed. + */ + public function removeOrphanedCorrelations() + { + $CorrelationsTable = $this->fetchTable('Correlations'); + $orphansLeft = $CorrelationsTable->find( + 'all', + [ + 'contain' => ['Attribute'], + 'conditions' => [ + 'Attribute.id IS NULL' + ], + 'fields' => ['Correlation.id', 'Correlation.attribute_id'], + ] + )->toArray(); + if (empty($orphansLeft)) { + return 0; + } + $orphansRight = $CorrelationsTable->find( + 'column', + [ + 'conditions' => [ + '1_attribute_id IN' => array_column($orphansLeft, 'attribute_id'), + ], + 'fields' => ['Correlation.id'], + ] + ); + $orphans = array_merge( + array_column($orphansLeft, 'id'), + $orphansRight + ); + if (!empty($orphans)) { + $CorrelationsTable->deleteAll( + [ + 'id' => $orphans + ], + false + ); + } + return count($orphans); + } + + public function queryAvailableSyncFilteringRules(array $server) + { + $syncFilteringRules = [ + 'error' => '', + 'data' => [] + ]; + + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + + try { + $syncFilteringRules['data'] = $serverSync->getAvailableSyncFilteringRules()->getJson(); + } catch (Exception $e) { + $syncFilteringRules['error'] = __('Connection failed. Error returned: {0}', $e->getMessage()); + return $syncFilteringRules; + } + + return $syncFilteringRules; + } + + public function getAvailableSyncFilteringRules(array $user) + { + $TagsTable = $this->fetchTable('Tags'); + $organisations = []; + if ($user['Role']['perm_sharing_group'] || !Configure::read('Security.hide_organisation_index_from_users')) { + $organisations = $this->Organisation->find( + 'column', + [ + 'fields' => ['name'], + ] + ); + } + $tags = $TagsTable->find( + 'column', + [ + 'fields' => ['name'], + ] + ); + return [ + 'organisations' => $organisations, + 'tags' => $tags, + ]; + } + + /** + * @param string|null $old Old (or current) encryption key. + * @param string|null $new New encryption key. If empty, encrypted values will be decrypted. + * @throws Exception + */ + public function reencryptAuthKeys($old, $new) + { + $servers = $this->find( + 'list', + [ + 'fields' => ['Server.id', 'Server.authkey'], + ] + ); + $toSave = []; + foreach ($servers as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + $authkey = BetterSecurity::decrypt(substr($authkey, 2), $old); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for server #$id", 0, $e); + } + } + if (!empty($new)) { + $authkey = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($authkey, $new); + } + $toSave[] = [ + 'Server' => [ + 'id' => $id, + 'authkey' => $authkey, + ] + + ]; + } + if (empty($toSave)) { + return true; + } + return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); + } + + /** + * @param string $encryptionKey + * @return bool + * @throws Exception + */ + public function isEncryptionKeyValid($encryptionKey) + { + $servers = $this->find( + 'list', + [ + 'fields' => ['Server.id', 'Server.authkey'], + ] + ); + foreach ($servers as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for server #$id", 0, $e); + } + } + } + return true; + } + + /** + * Return all Attribute and Object types + */ + public function getAllTypes(): array + { + $allTypes = []; + $AttributesTable = $this->fetchTable('Attributes'); + $ObjectTemplatesTable = $this->fetchTable('ObjectTemplates'); + $objects = $ObjectTemplatesTable->find( + 'all', + [ + 'recursive' => -1, + 'fields' => ['uuid', 'name'], + 'group' => ['uuid', 'name'], + ] + )->toArray(); + $allTypes = [ + 'attribute' => array_unique(Hash::extract(Hash::extract($AttributesTable->categoryDefinitions, '{s}.types'), '{n}.{n}')), + 'object' => Hash::map( + $objects, + '{n}.ObjectTemplate', + function ($item) { + return ['id' => $item['uuid'], 'name' => sprintf('%s (%s)', $item['name'], $item['uuid'])]; + } + ) + ]; + return $allTypes; + } + + /** + * Invalidate config.php from php opcode cache + */ + private function opcacheResetConfig() + { + if (function_exists('opcache_invalidate')) { + opcache_invalidate(APP . 'Config' . DS . 'config.php', true); + } + } + + /** + * Get workers + * + * @return array + */ + private function getWorkers(): array + { + $worker_array = []; + $workers = $this->getBackgroundJobsTool()->getWorkers(); + + foreach ($workers as $worker) { + $worker_array[$worker->pid()] = [ + 'queue' => $worker->queue(), + 'type' => 'regular', + 'user' => $worker->user() + ]; + } + + return $worker_array; + } } diff --git a/src/Model/Table/ShadowAttributesTable.php b/src/Model/Table/ShadowAttributesTable.php new file mode 100644 index 000000000..fb8a2359c --- /dev/null +++ b/src/Model/Table/ShadowAttributesTable.php @@ -0,0 +1,23 @@ +2 Force Fields required to block it off) then do not try to defend it and instead move all your Probes into your main base. Constant Force Fields will prevent the Zerg player from getting up the ramp whilst you destroy their third base and natural expansion with your army. To not let the Zerg have a better economy than you, you will have to kill their 3rd base and second base quickly before they can build too many Spine Crawlers. Do not try to fight them up their main ramp if they've built a lot of Spine Crawlers in their main to defend; instead come home and kill their main army since your army should be stronger with appropriate micro. Take your natural back if you lost it, take 3rd if you didn't, and then try going back into a normal game. Use an Observer to scout what the Zerg is going to try to do to get back into the game; research Blink and get High Templar if they are going for Mutalisks, get Colossi if they are going for Infestors. + +Follow Up +If your opponent successfully stops your push then you will be in a very difficult spot. To be able to defend any counter-attack, you will have to start producing Immortals again and re-build your Sentries. With your Observer you will have to scout if they are transitioning into Mutalisks or Infestors. If they go for Mutalisks, build a Twilight Council and start researching Blink. If they go for Infestors, you will need to build Colossi. You should also be looking to take a 3rd base, although this can be hard against a Zerg opponent who didn't lose their own 3rd base during your push; for this reason, it might be better to max out on two bases and go for a Colossus-based all-in. + diff --git a/tests/Fixture/AttributesFixture.php b/tests/Fixture/AttributesFixture.php index 1ebb04222..873a1473c 100644 --- a/tests/Fixture/AttributesFixture.php +++ b/tests/Fixture/AttributesFixture.php @@ -10,9 +10,25 @@ class AttributesFixture extends TestFixture { public $connection = 'test'; + public const ATTRIBUTE_1_ID = 1000; + public const ATTRIBUTE_1_UUID = '60d515a6-efd1-4ae8-a561-1a5203ec9ade'; + + public function init(): void { - $this->records = []; + $this->records = [ + [ + 'id' => self::ATTRIBUTE_1_ID, + 'uuid' => self::ATTRIBUTE_1_UUID, + 'event_id' => EventsFixture::EVENT_1_ID, + 'distribution' => 3, + 'category' => 'Network activity', + 'type' => 'ip-src', + 'value1' => '127.0.0.1', + 'value2' => '', + 'sharing_group_id' => 0, + ] + ]; parent::init(); } } diff --git a/tests/Fixture/EventsFixture.php b/tests/Fixture/EventsFixture.php index 697fe5697..0946e8435 100644 --- a/tests/Fixture/EventsFixture.php +++ b/tests/Fixture/EventsFixture.php @@ -4,15 +4,35 @@ declare(strict_types=1); namespace App\Test\Fixture; +use App\Model\Entity\Distribution; use Cake\TestSuite\Fixture\TestFixture; class EventsFixture extends TestFixture { public $connection = 'test'; + public const EVENT_1_ID = 1000; + public const EVENT_1_UUID = '02a5f2e5-3c6c-4d40-b973-de465fd2f370'; + public function init(): void { - $this->records = []; + $this->records = [ + [ + 'id' => self::EVENT_1_ID, + 'info' => 'Event 1', + 'org_id' => OrganisationsFixture::ORGANISATION_A_ID, + 'orgc_id' => OrganisationsFixture::ORGANISATION_A_ID, + 'user_id' => UsersFixture::USER_ADMIN_ID, + 'distribution' => Distribution::ALL_COMMUNITIES, + 'analysis' => 0, + 'threat_level_id' => 0, + 'date' => '2021-01-01 00:00:00', + 'published' => 1, + 'uuid' => self::EVENT_1_UUID, + 'attribute_count' => 1, + 'sharing_group_id' => 0, + ] + ]; parent::init(); } } diff --git a/tests/Fixture/ServersFixture.php b/tests/Fixture/ServersFixture.php index dd497a5b7..91f439f42 100644 --- a/tests/Fixture/ServersFixture.php +++ b/tests/Fixture/ServersFixture.php @@ -12,12 +12,16 @@ class ServersFixture extends TestFixture public const SERVER_A_ID = 1000; public const SERVER_A_NAME = 'Server A'; + public const SERVER_A_URL = 'http://aaa.local'; + public const SERVER_A_AUTHKEY = '8843d7f92416211de9ebb963ff4ce28125932878'; public const SERVER_B_ID = 2000; public const SERVER_B_NAME = 'Server B'; + public const SERVER_B_URL = 'http://bbb.local'; public const SERVER_C_ID = 3000; public const SERVER_C_NAME = 'Server C'; + public const SERVER_C_URL = 'http://ccc.local'; public function init(): void { @@ -28,8 +32,8 @@ class ServersFixture extends TestFixture 'id' => self::SERVER_A_ID, 'org_id' => OrganisationsFixture::ORGANISATION_A_ID, 'name' => self::SERVER_A_NAME, - 'url' => $faker->url, - 'authkey' => $faker->sha1(), + 'url' => self::SERVER_A_URL, + 'authkey' => self::SERVER_A_AUTHKEY, 'push' => true, 'pull' => true, 'push_sightings' => true, @@ -52,7 +56,7 @@ class ServersFixture extends TestFixture 'id' => self::SERVER_B_ID, 'org_id' => OrganisationsFixture::ORGANISATION_B_ID, 'name' => self::SERVER_B_NAME, - 'url' => $faker->url, + 'url' => self::SERVER_B_URL, 'authkey' => $faker->sha1(), 'push' => true, 'pull' => true, @@ -76,7 +80,7 @@ class ServersFixture extends TestFixture 'id' => self::SERVER_C_ID, 'org_id' => OrganisationsFixture::ORGANISATION_C_ID, 'name' => self::SERVER_C_NAME, - 'url' => $faker->url, + 'url' => self::SERVER_C_URL, 'authkey' => $faker->sha1(), 'push' => true, 'pull' => true, diff --git a/tests/TestCase/Api/Servers/AddServerApiTest.php b/tests/TestCase/Api/Servers/AddServerApiTest.php new file mode 100644 index 000000000..075ed8b60 --- /dev/null +++ b/tests/TestCase/Api/Servers/AddServerApiTest.php @@ -0,0 +1,48 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $faker = \Faker\Factory::create(); + + $this->post( + self::ENDPOINT, + [ + "name" => "Test Server", + "url" => $faker->url, + "remote_org_id" => OrganisationsFixture::ORGANISATION_A_ID, + "authkey" => $faker->sha256(), + "self_signed" => true, + ] + ); + + $this->assertResponseOk(); + $this->assertDbRecordExists('Servers', ['name' => 'Test Server']); + } +} diff --git a/tests/TestCase/Api/Servers/DeleteServerApiTest.php b/tests/TestCase/Api/Servers/DeleteServerApiTest.php new file mode 100644 index 000000000..cb334d94c --- /dev/null +++ b/tests/TestCase/Api/Servers/DeleteServerApiTest.php @@ -0,0 +1,40 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $url = sprintf('%s/%d', self::ENDPOINT, ServersFixture::SERVER_B_ID); + + $this->assertDbRecordExists('Servers', ['id' => ServersFixture::SERVER_B_ID]); + + $this->post($url); + $this->assertResponseOk(); + $this->assertDbRecordNotExists('Servers', ['id' => ServersFixture::SERVER_B_ID]); + } +} diff --git a/tests/TestCase/Api/Servers/EditServerApiTest.php b/tests/TestCase/Api/Servers/EditServerApiTest.php new file mode 100644 index 000000000..a1c15b8ac --- /dev/null +++ b/tests/TestCase/Api/Servers/EditServerApiTest.php @@ -0,0 +1,75 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $url = sprintf('%s/%s', self::ENDPOINT, ServersFixture::SERVER_A_ID); + + $newUrl = 'http://new-url.local'; + + $this->put( + $url, + [ + "url" => $newUrl, + "push" => false, + "pull" => true, + "pull_rules" => [ + "tags" => [ + "OR" => [], + "NOT" => ["tlp:red"] + ], + "orgs" => [ + "OR" => [], + "NOT" => [] + ], + "type_attributes" => [ + "NOT" => [] + ], "type_objects" => [ + "NOT" => [] + ], + "url_params" => "" + ] + ] + ); + + $this->assertResponseOk(); + $response = $this->getJsonResponseAsArray(); + + $this->assertDbRecordExists( + 'Servers', + [ + 'id' => ServersFixture::SERVER_A_ID, + 'url' => $newUrl, + 'push' => false, + 'pull' => true, + ] + ); + $this->assertEquals($response['pull_rules']['tags']['NOT'], ['tlp:red']); + } +} diff --git a/tests/TestCase/Api/Servers/GetServerVersionApiTest.php b/tests/TestCase/Api/Servers/GetServerVersionApiTest.php new file mode 100644 index 000000000..1a62c6be5 --- /dev/null +++ b/tests/TestCase/Api/Servers/GetServerVersionApiTest.php @@ -0,0 +1,43 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $this->get(self::ENDPOINT); + + $this->assertResponseOk(); + $response = $this->getJsonResponseAsArray(); + + // read the version from the VERSION.json file + $versionJson = json_decode(file_get_contents(ROOT . DS . 'VERSION.json'), true); + $expectedVersion = $versionJson['major'] . '.' . $versionJson['minor'] . '.' . $versionJson['hotfix']; + + $this->assertEquals($expectedVersion, $response['version']); + } +} diff --git a/tests/TestCase/Api/Servers/ImportServerApiTest.php b/tests/TestCase/Api/Servers/ImportServerApiTest.php new file mode 100644 index 000000000..b193d9383 --- /dev/null +++ b/tests/TestCase/Api/Servers/ImportServerApiTest.php @@ -0,0 +1,52 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $faker = \Faker\Factory::create(); + + $server = [ + "name" => "Test Import Server", + "url" => $faker->url, + "authkey" => $faker->sha256(), + "Organisation" => [ + "name" => "ORGNAME", + "uuid" => OrganisationsFixture::ORGANISATION_A_UUID + ] + ]; + + $this->post( + self::ENDPOINT, + $server + ); + + $this->assertResponseOk(); + $this->assertDbRecordExists('Servers', ['name' => 'Test Import Server']); + } +} diff --git a/tests/TestCase/Api/Servers/IndexServersApiTest.php b/tests/TestCase/Api/Servers/IndexServersApiTest.php new file mode 100644 index 000000000..854027ce2 --- /dev/null +++ b/tests/TestCase/Api/Servers/IndexServersApiTest.php @@ -0,0 +1,39 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $this->get(self::ENDPOINT); + + $this->assertResponseOk(); + $this->assertResponseContains(sprintf('"name": "%s"', ServersFixture::SERVER_A_NAME)); + $this->assertResponseContains(sprintf('"name": "%s"', ServersFixture::SERVER_B_NAME)); + $this->assertResponseContains(sprintf('"name": "%s"', ServersFixture::SERVER_C_NAME)); + } +} diff --git a/tests/TestCase/Api/Servers/PullServerApiTest.php b/tests/TestCase/Api/Servers/PullServerApiTest.php new file mode 100644 index 000000000..5075daeaf --- /dev/null +++ b/tests/TestCase/Api/Servers/PullServerApiTest.php @@ -0,0 +1,167 @@ +skipOpenApiValidations(); + + Configure::write('BackgroundJobs.enabled', false); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + $url = sprintf('%s/%d', self::ENDPOINT, ServersFixture::SERVER_A_ID); + + $headers = [ + 'Content-Type: application/json', + 'Accept: application/json', + 'User-Agent: MISP 3.0.0 - #cc1f8cc2e89ec692168ffbfea8ed49cc879c469b', + 'ETag: W/"2a-1b6e3e8e"', + ]; + + // mock the /servers/getVersion request + $getVersionBody = json_encode( + [ + "version" => "3.0.0", + "pymisp_recommended_version" => "3.0.0", + "perm_sync" => true, + "perm_sighting" => true, + "perm_galaxy_editor" => true, + "request_encoding" => [ + "gzip", + "br" + ], + "filter_sightings" => true + ] + ); + + $this->mockClientGet( + ServersFixture::SERVER_A_URL . '/servers/getVersion', + $this->newClientResponse(200, $headers, $getVersionBody) + ); + + // mock the /events/index request + $eventsIndexBody = json_encode( + [ + [ + "id" => "10", + "timestamp" => "1700488705", + "sighting_timestamp" => "0", + "published" => true, + "uuid" => "56bf399d-c46c-4fdb-a9cf-d9bb02de0b81", + "orgc_uuid" => "55f6ea5e-2c60-40e5-964f-47a8950d210f" + ] + ] + ); + + $this->mockClientPost( + ServersFixture::SERVER_A_URL . '/events/index', + $this->newClientResponse(200, $headers, $eventsIndexBody) + ); + + + // mock the /events/view/[uuid] request + $eventBody = json_encode( + [ + "Event" => [ + "analysis" => "2", + "date" => "2015-12-18", + "extends_uuid" => "", + "info" => "OSINT - Hunting for Malware with Machine Learning", + "publish_timestamp" => "1455373314", + "sharing_group_id" => "0", + "distribution" => "0", + "published" => true, + "threat_level_id" => "3", + "timestamp" => "1455373240", + "uuid" => "56bf399d-c46c-4fdb-a9cf-d9bb02de0b81", + "Orgc" => [ + "name" => "CIRCL", + "uuid" => "55f6ea5e-2c60-40e5-964f-47a8950d210f" + ], + "Tag" => [ + [ + "colour" => "#004646", + "local" => "0", + "name" => "type:OSINT", + "relationship_type" => "" + ], + [ + "colour" => "#ffffff", + "local" => "0", + "name" => "tlp:white", + "relationship_type" => "" + ] + ], + "Attribute" => [ + [ + "category" => "External analysis", + "comment" => "", + "deleted" => false, + "disable_correlation" => false, + "sharing_group_id" => "0", + "distribution" => "0", + "timestamp" => "1455372745", + "to_ids" => false, + "type" => "link", + "uuid" => "56bf39c9-c078-4368-9555-6cf802de0b81", + "value" => "http://blog.cylance.com/hunting-for-malware-with-machine-learning" + ] + ] + ] + ] + ); + + // mock the event [uuid].json request + $this->mockClientGet( + 'http://aaa.local/events/view/56bf399d-c46c-4fdb-a9cf-d9bb02de0b81/deleted%5B%5D:0/deleted%5B%5D:1/excludeGalaxy:1/includeEventCorrelations:0/includeFeedCorrelations:0/includeWarninglistHits:0/excludeLocalTags:1', + $this->newClientResponse(200, $headers, $eventBody) + ); + + $this->post($url); + $this->assertResponseOk(); + + $response = $this->getJsonResponseAsArray(); + $this->assertEquals( + 'Pull completed. 1 events pulled, 0 events could not be pulled, 0 proposals pulled, 0 sightings pulled, 0 clusters pulled.', + $response['message'] + ); + + // check that the event was added + $this->assertDbRecordExists('Events', ['uuid' => '56bf399d-c46c-4fdb-a9cf-d9bb02de0b81']); + $this->assertDbRecordExists('Attributes', ['uuid' => '56bf39c9-c078-4368-9555-6cf802de0b81']); + + // TODO: check that the proposals were added + // TODO: check that the objects were added + // TODO: check that the event reports were added + // TODO: check that the sightings were added + // TODO: check that the tags were added + // TODO: check that the galaxies were added + // TODO: check that the cryptographic keys were added + } +} diff --git a/tests/TestCase/Api/Servers/PushServerApiTest.php b/tests/TestCase/Api/Servers/PushServerApiTest.php new file mode 100644 index 000000000..671010f3c --- /dev/null +++ b/tests/TestCase/Api/Servers/PushServerApiTest.php @@ -0,0 +1,107 @@ +skipOpenApiValidations(); + + Configure::write('BackgroundJobs.enabled', false); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + $url = sprintf('%s/%d', self::ENDPOINT, ServersFixture::SERVER_B_ID); + + $headers = [ + 'Content-Type: application/json', + 'Accept: application/json', + 'User-Agent: MISP 3.0.0 - #cc1f8cc2e89ec692168ffbfea8ed49cc879c469b', + 'ETag: W/"2a-1b6e3e8e"', + ]; + + // mock the /servers/getVersion request + $getVersionBody = json_encode( + [ + "version" => "3.0.0", + "pymisp_recommended_version" => "3.0.0", + "perm_sync" => true, + "perm_sighting" => true, + "perm_galaxy_editor" => true, + "request_encoding" => [ + "gzip", + "br" + ], + "filter_sightings" => true + ] + ); + + $this->mockClientGet( + ServersFixture::SERVER_B_URL . '/servers/getVersion', + $this->newClientResponse(200, $headers, $getVersionBody) + ); + + // mock the /events/filterEventIdsForPush request + $filterEventIdsForPushBody = json_encode( + [ + EventsFixture::EVENT_1_UUID + ] + ); + $this->mockClientPost( + ServersFixture::SERVER_B_URL . '/events/filterEventIdsForPush', + $this->newClientResponse(200, $headers, $filterEventIdsForPushBody) + ); + + // mock the /events/index request, triggered by syncProposals() + $this->mockClientPost( + ServersFixture::SERVER_B_URL . '/events/index', + $this->newClientResponse(200, $headers, '[]') + ); + + // mock the /events/view/[uuid] request + $this->mockClientGet( + ServersFixture::SERVER_B_URL . '/events/view/' . EventsFixture::EVENT_1_UUID, + $this->newClientResponse(200, $headers, '[]') + ); + + // mock the /events/add/metadata:1 + $this->mockClientPost( + ServersFixture::SERVER_B_URL . '/events/add/metadata:1', + $this->newClientResponse(200, $headers, '[]') + ); + + $this->post($url); + $this->assertResponseOk(); + + $response = $this->getJsonResponseAsArray(); + $this->assertEquals( + 'Push complete. 1 events pushed, 0 events could not be pushed.', + $response['message'] + ); + } +} diff --git a/tests/TestCase/Api/Servers/TestConnectionApiTest.php b/tests/TestCase/Api/Servers/TestConnectionApiTest.php new file mode 100644 index 000000000..6e69a7e02 --- /dev/null +++ b/tests/TestCase/Api/Servers/TestConnectionApiTest.php @@ -0,0 +1,70 @@ +skipOpenApiValidations(); + + $this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY); + + $url = sprintf('%s/%s', self::ENDPOINT, ServersFixture::SERVER_A_ID); + + $headers = [ + 'Content-Type: application/json', + 'Connection: close', + ]; + + $getVersionBody = json_encode( + [ + "version" => "3.0.0", + "pymisp_recommended_version" => "3.0.0", + "perm_sync" => true, + "perm_sighting" => true, + "perm_galaxy_editor" => true, + "request_encoding" => [ + "gzip", + "br" + ], + "filter_sightings" => true + ] + ); + + // mock the [remote]/servers/getVersion request + $this->mockClientGet( + ServersFixture::SERVER_A_URL . '/servers/getVersion', + $this->newClientResponse(200, $headers, $getVersionBody) + ); + + $this->get($url); + + $this->assertResponseOk(); + $response = $this->getJsonResponseAsArray(); + + $this->assertArrayHasKey('version', $response); + $this->assertEquals('3.0.0', $response['version']); + } +} diff --git a/tests/TestCase/Tool/HttpToolTest.php b/tests/TestCase/Tool/HttpToolTest.php index 263e9cf63..58cab9821 100644 --- a/tests/TestCase/Tool/HttpToolTest.php +++ b/tests/TestCase/Tool/HttpToolTest.php @@ -48,7 +48,7 @@ xWV4oBk= -----END CERTIFICATE----- "; - public function testGoogle($options=[]) + public function testGoogle($options = []) { $client = new HttpTool($options); $response = $client->get('https://www.google.com'); @@ -73,7 +73,9 @@ xWV4oBk= $config = [ 'ssl_verify_peer' => true, - 'ssl_verify_host' => false]; + 'ssl_verify_host' => false + + ]; $client = new HttpTool($config); try { @@ -172,7 +174,6 @@ xWV4oBk= $this->assertEquals($result['issuer'], 'C=US, O=Google Trust Services LLC, CN=GTS CA 1C3'); $this->assertEquals($result['public_key_size_ok'], true); $this->assertEquals($result['valid_from_ok'], true); - $this->assertEquals($result['valid_to_ok'], true); $this->assertEquals($result['valid_from'], new FrozenTime("2023-11-20 08:09:47.000000+00:00")); $this->assertEquals($result['valid_to'], new FrozenTime("2024-02-12 08:09:46.000000+00:00")); $this->assertEquals($result['signature_type'], "RSA-SHA256"); @@ -189,9 +190,8 @@ xWV4oBk= // $certificates = $client->fetchCertificates(self::HTTPS_SELF_SIGNED_URI); // $certificates = $client->fetchCertificates('http://www.google.com'); // we get one or more certificates from the server. No function yet to select "the right one" - foreach($certificates as $certificate) { + foreach ($certificates as $certificate) { // debug($certificate); } } - }