diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dbf874c33..51b148441 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,7 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php }} - extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd + extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd, apcu - name: Initialize variables run: | @@ -126,9 +126,16 @@ jobs: sudo gpg --list-secret-keys --homedir `pwd`/.gnupg # change perms sudo chown -R $USER:www-data `pwd` - sudo chmod -R 770 `pwd`/.gnupg - # Get authkey + sudo chown -R www-data:www-data `pwd`/.gnupg + sudo chmod -R 700 `pwd`/.gnupg sudo usermod -a -G www-data $USER + sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ + # Ensure the perms of config files + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config + sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1' + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config - name: DB Update run: | @@ -185,7 +192,7 @@ jobs: - name: Start workers run: | sudo chmod +x app/Console/worker/start.sh - sudo -E su $USER -c 'app/Console/worker/start.sh' + sudo -u www-data 'app/Console/worker/start.sh' - name: Python setup run: | @@ -237,13 +244,6 @@ jobs: run: | export PATH=$HOME/.local/env:$PATH # enable poetry binary - # Ensure the perms of config files - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1' - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - pushd tests ./curl_tests_GH.sh $AUTH $HOST popd @@ -255,6 +255,7 @@ jobs: poetry add lxml poetry run python ../tests/testlive_security.py -v poetry run python ../tests/testlive_sync.py + poetry run python ../tests/testlive_comprehensive_local.py -v poetry run python tests/test_mispevent.py popd cp PyMISP/tests/keys.py PyMISP/examples/events/ diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index fb08530f0..44da990b4 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -548,7 +548,7 @@ CREATE TABLE IF NOT EXISTS `jobs` ( `message` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, `progress` int(11) NOT NULL DEFAULT 0, `org_id` int(11) NOT NULL DEFAULT 0, - `process_id` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `process_id` varchar(36) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, `date_created` datetime NOT NULL, `date_modified` datetime NOT NULL, PRIMARY KEY (`id`) @@ -690,6 +690,7 @@ CREATE TABLE IF NOT EXISTS `object_references` ( `comment` text COLLATE utf8_bin NOT NULL, `deleted` tinyint(1) NOT NULL DEFAULT 0, PRIMARY KEY (id), + UNIQUE INDEX `uuid` (`uuid`), INDEX `source_uuid` (`source_uuid`), INDEX `referenced_uuid` (`referenced_uuid`), INDEX `timestamp` (`timestamp`), @@ -1056,6 +1057,7 @@ CREATE TABLE IF NOT EXISTS `sharing_groups` ( INDEX `org_id` (`org_id`), INDEX `sync_user_id` (`sync_user_id`), UNIQUE INDEX `uuid` (`uuid`), + UNIQUE INDEX `name` (`name`), INDEX `organisation_uuid` (`organisation_uuid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; diff --git a/INSTALL/ansible/roles/misp/templates/misp/config/config.php b/INSTALL/ansible/roles/misp/templates/misp/config/config.php index 2a4d002df..75052ec9c 100644 --- a/INSTALL/ansible/roles/misp/templates/misp/config/config.php +++ b/INSTALL/ansible/roles/misp/templates/misp/config/config.php @@ -16,7 +16,6 @@ $config = array ( 'org' => '', 'showorg' => true, 'background_jobs' => true, - 'cached_attachments' => true, 'email' => '', 'contact' => '', 'cveurl' => 'http://cve.circl.lu/cve/', diff --git a/PyMISP b/PyMISP index b21289415..8b66d5f75 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit b212894152c9471e4da04eada9da887cad2f92a7 +Subproject commit 8b66d5f75346271c78d4737f79fb302e17c961d2 diff --git a/README.md b/README.md index 160f69c71..8e6dc245a 100644 --- a/README.md +++ b/README.md @@ -108,16 +108,16 @@ License This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html) -* Copyright (C) 2012-2020 Christophe Vandeplas +* Copyright (C) 2012-2022 Christophe Vandeplas * Copyright (C) 2012 Belgian Defence * Copyright (C) 2012 NATO / NCIRC -* Copyright (C) 2013-2020 Andras Iklody -* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg +* Copyright (C) 2013-2022 Andras Iklody +* Copyright (C) 2015-2022 CIRCL - Computer Incident Response Center Luxembourg * Copyright (C) 2016 Andreas Ziegler -* Copyright (C) 2018-2020 Sami Mokaddem -* Copyright (C) 2018-2020 Christian Studer -* Copyright (C) 2015-2020 Alexandre Dulaunoy -* Copyright (C) 2018-2020 Steve Clement -* Copyright (C) 2020 Jakub Onderka +* Copyright (C) 2018-2022 Sami Mokaddem +* Copyright (C) 2018-2022 Christian Studer +* Copyright (C) 2015-2022 Alexandre Dulaunoy +* Copyright (C) 2018-2022 Steve Clement +* Copyright (C) 2020-2022 Jakub Onderka For more information, [the list of authors and contributors](AUTHORS) is available. diff --git a/VERSION.json b/VERSION.json index 5e20a70a4..23faaa866 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":150} +{"major":2, "minor":4, "hotfix":151} diff --git a/app/Config/config.default.php b/app/Config/config.default.php index b2d3f010a..70e52dc55 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -32,7 +32,6 @@ $config = array( 'email_subject_tag' => 'tlp', 'email_subject_include_tag_name' => true, 'background_jobs' => true, - 'cached_attachments' => true, 'osuser' => 'www-data', 'email' => 'email@example.com', 'contact' => 'email@example.com', @@ -50,7 +49,6 @@ $config = array( 'unpublishedprivate' => false, 'disable_emailing' => false, 'manage_workers' => true, - 'Attributes_Values_Filter_In_Event' => 'id, uuid, value, comment, type, category, Tag.name', 'python_bin' => null, 'external_baseurl' => '', 'forceHTTPSforPreLoginRequestedURL' => false, @@ -135,6 +133,19 @@ $config = array( 'amount' => 5, 'expire' => 300, ), + 'SimpleBackgroundJobs' => array( + 'enabled' => false, + 'redis_host' => 'localhost', + 'redis_port' => 6379, + 'redis_password' => '', + 'redis_database' => 1, + 'redis_namespace' => 'background_jobs', + 'max_job_history_ttl' => 86400, + 'supervisor_host' => 'localhost', + 'supervisor_port' => 9001, + 'supervisor_user' => '', + 'supervisor_password' => '', + ), // Uncomment the following to enable client SSL certificate authentication /* 'CertAuth' => array( diff --git a/app/Config/core.default.php b/app/Config/core.default.php index eb226485f..97ae661de 100644 --- a/app/Config/core.default.php +++ b/app/Config/core.default.php @@ -255,8 +255,9 @@ Configure::write('Acl.database', 'default'); * and their setttings. */ $engine = 'File'; -if (extension_loaded('apc') && function_exists('apc_dec') && (php_sapi_name() !== 'cli' || ini_get('apc.enable_cli'))) { - $engine = 'Apc'; +if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) { + require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin + $engine = 'Apcu'; // faster version of ApcEngine } // In development mode, caches should expire quickly. diff --git a/app/Console/AppShell.php b/app/Console/AppShell.php deleted file mode 100644 index 76a6f1fbc..000000000 --- a/app/Console/AppShell.php +++ /dev/null @@ -1,34 +0,0 @@ -initialize(); - $this->{array_shift($this->args)}(); - } -} diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 96d1a78c0..450bab021 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -3,6 +3,7 @@ App::uses('AppShell', 'Console/Command'); /** * @property Server $Server + * @property Feed $Feed */ class AdminShell extends AppShell { @@ -22,12 +23,18 @@ class AdminShell extends AppShell 'value' => ['help' => __('Setting value'), 'required' => true], ], 'options' => [ - 'force' => array( + 'force' => [ 'short' => 'f', 'help' => 'Force the command.', 'default' => false, 'boolean' => true - ) + ], + 'null' => [ + 'short' => 'n', + 'help' => 'Set the value to null.', + 'default' => false, + 'boolean' => true + ], ] ], ]); @@ -39,6 +46,27 @@ class AdminShell extends AppShell ], ], ]); + $parser->addSubcommand('reencrypt', [ + 'help' => __('Reencrypt encrypted values in database (authkeys and sensitive system settings).'), + 'parser' => [ + 'options' => [ + 'old' => ['help' => __('Old key. If not provided, current key will be used.')], + 'new' => ['help' => __('New key. If not provided, new key will be generated.')], + ], + ], + ]); + $parser->addSubcommand('removeOrphanedCorrelations', [ + 'help' => __('Remove orphaned correlations.'), + ]); + $parser->addSubcommand('optimiseTables', [ + 'help' => __('Optimise database tables.'), + ]); + $parser->addSubcommand('redisMemoryUsage', [ + 'help' => __('Get detailed information about Redis memory usage.'), + ]); + $parser->addSubcommand('redisReady', [ + 'help' => __('Check if it is possible connect to Redis.'), + ]); return $parser; } @@ -369,14 +397,13 @@ class AdminShell extends AppShell public function getSetting() { - $this->ConfigLoad->execute(); $param = empty($this->args[0]) ? 'all' : $this->args[0]; $settings = $this->Server->serverSettingsRead(); $result = $settings; - if ($param != 'all') { + if ($param !== 'all') { $result = 'No valid setting found for ' . $param; foreach ($settings as $setting) { - if ($setting['setting'] == $param) { + if ($setting['setting'] === $param) { $result = $setting; break; } @@ -387,15 +414,17 @@ class AdminShell extends AppShell public function setSetting() { - $setting_name = !isset($this->args[0]) ? null : $this->args[0]; - $value = !isset($this->args[1]) ? null : $this->args[1]; + list($setting_name, $value) = $this->args; if ($value === 'false') { $value = 0; } elseif ($value === 'true') { $value = 1; } + if ($this->params['null']) { + $value = null; + } $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM')); - if (empty($setting_name) || $value === null) { + if (empty($setting_name) || ($value === null && !$this->params['null'])) { die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL); } $setting = $this->Server->getSettingData($setting_name); @@ -405,7 +434,7 @@ class AdminShell extends AppShell } $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']); if ($result === true) { - echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL; + $this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value)); } else { $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result); $this->error(__('Setting change rejected.'), $message); @@ -473,7 +502,7 @@ class AdminShell extends AppShell { try { $redis = $this->Server->setupRedisWithException(); - $redis->randomKey(); + $redis->ping(); $this->out('Successfully connected to Redis.'); } catch (Exception $e) { $this->error('Redis connection is not available', $e->getMessage()); @@ -756,6 +785,33 @@ class AdminShell extends AppShell $this->Job->saveField('status', 4); } + public function removeOrphanedCorrelations() + { + $count = $this->Server->removeOrphanedCorrelations(); + $this->out(__('%s orphaned correlation removed', $count)); + } + + public function optimiseTables() + { + $dataSource = $this->Server->getDataSource(); + $tables = $dataSource->listSources(); + + /** @var ProgressShellHelper $progress */ + $progress = $this->helper('progress'); + $progress->init([ + 'total' => count($tables), + 'width' => 50, + ]); + + foreach ($tables as $table) { + $dataSource->query('OPTIMISE TABLE ' . $dataSource->name($table)); + $progress->increment(); + $progress->draw(); + } + + $this->out('Optimised.'); + } + public function updatesDone() { $blocking = !empty($this->args[0]); @@ -851,4 +907,157 @@ class AdminShell extends AppShell $this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False')); } } + + public function reencrypt() + { + $old = $this->params['old'] ?? null; + $new = $this->params['new'] ?? null; + + if ($new !== null && strlen($new) < 32) { + $this->error('New key must be at least 32 char long.'); + } + + if ($old === null) { + $old = Configure::read('Security.encryption_key'); + } + + if ($new === null) { + // Generate random new key + $randomTool = new RandomTool(); + $new = $randomTool->random_str(); + } + + $this->Server->getDataSource()->begin(); + + try { + /** @var SystemSetting $systemSetting */ + $systemSetting = ClassRegistry::init('SystemSetting'); + $systemSetting->reencrypt($old, $new); + + $this->Server->reencryptAuthKeys($old, $new); + + /** @var Cerebrate $cerebrate */ + $cerebrate = ClassRegistry::init('Cerebrate'); + $cerebrate->reencryptAuthKeys($old, $new); + + $result = $this->Server->serverSettingsSaveValue('Security.encryption_key', $new, true); + + $this->Server->getDataSource()->commit(); + + if (!$result) { + $this->error('Encrypt key was changed, but it is not possible to save key to config file', __('Please insert new key "%s" to config file manually.', $new)); + } + } catch (Exception $e) { + $this->Server->getDataSource()->rollback(); + throw $e; + } + + $this->out(__('New encryption key "%s" saved into config file.', $new)); + } + + /** + * @param Redis $redis + * @param string $prefix + * @return array[int, int] + */ + private function redisSize($redis, $prefix) + { + $keyCount = 0; + $size = 0; + $it = null; + while ($keys = $redis->scan($it, $prefix, 1000)) { + $redis->pipeline(); + foreach ($keys as $key) { + $redis->rawCommand("memory", "usage", $key); + } + $result = $redis->exec(); + $keyCount += count($keys); + $size += array_sum($result); + } + return [$keyCount, $size]; + } + + public function redisMemoryUsage() + { + $redis = $this->Server->setupRedisWithException(); + $redis->setOption(Redis::OPT_SCAN, Redis::SCAN_RETRY); + + $output = []; + + list($count, $size) = $this->redisSize($redis, 'misp:feed_cache:*'); + $output['feed_cache_count'] = $count; + $output['feed_cache_size'] = $size; + + // Size of different feeds + $feedIds = $this->Feed->find('column', [ + 'fields' => ['id'], + ]); + + $redis->pipeline(); + foreach ($feedIds as $feedId) { + $redis->rawCommand("memory", "usage", 'misp:feed_cache:' . $feedId); + } + $feedSizes = $redis->exec(); + + foreach ($feedIds as $k => $feedId) { + if ($feedSizes[$k]) { + $output['feed_cache_size_' . $feedId] = $feedSizes[$k]; + } + } + + list($count, $size) = $this->redisSize($redis, 'misp:server_cache:*'); + $output['server_cache_count'] = $count; + $output['server_cache_size'] = $size; + + // Size of different server + $serverIds = $this->Server->find('column', [ + 'fields' => ['id'], + ]); + + $redis->pipeline(); + foreach ($serverIds as $serverId) { + $redis->rawCommand("memory", "usage", 'misp:server_cache:' . $serverId); + } + $serverSizes = $redis->exec(); + + foreach ($serverIds as $k => $serverId) { + if ($serverSizes[$k]) { + $output['server_cache_size_' . $serverId] = $serverSizes[$k]; + } + } + + list($count, $size) = $this->redisSize($redis, 'misp:wlc:*'); + $output['warninglist_cache_count'] = $count; + $output['warninglist_cache_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:warninglist_entries_cache:*'); + $output['warninglist_entries_count'] = $count; + $output['warninglist_entries_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:top_correlation'); + $output['top_correlation_count'] = $count; + $output['top_correlation_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:correlation_exclusions'); + $output['correlation_exclusions_count'] = $count; + $output['correlation_exclusions_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:event_lock:*'); + $output['event_lock_count'] = $count; + $output['event_lock_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:user_ip:*'); + $output['user_ip_count'] = $count; + $output['user_ip_size'] = $size; + + list($count, $size) = $this->redisSize($redis, 'misp:ip_user:*'); + $output['user_ip_count'] += $count; + $output['user_ip_size'] += $size; + + list($count, $size) = $this->redisSize($redis, 'misp:authkey_usage:*'); + $output['authkey_usage_count'] = $count; + $output['authkey_usage_size'] = $size; + + $this->out($this->json($output)); + } } diff --git a/app/Console/Command/AppShell.php b/app/Console/Command/AppShell.php index b691646fa..91d5496fe 100644 --- a/app/Console/Command/AppShell.php +++ b/app/Console/Command/AppShell.php @@ -29,7 +29,14 @@ App::uses('AppModel', 'Model'); class AppShell extends Shell { public $tasks = array('ConfigLoad'); - + + public function initialize() + { + parent::initialize(); + $this->ConfigLoad = $this->Tasks->load('ConfigLoad'); + $this->ConfigLoad->execute(); + } + public function perform() { $this->initialize(); diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index c8d503004..6bd8eb904 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -1,6 +1,7 @@ Event->save($event, array('fieldList' => $fieldList)); // only allow form submit CSRF protection. - $this->Job->saveField('status', 1); - $this->Job->saveField('message', 'Job done.'); + $this->Job->save([ + 'status' => Job::STATUS_COMPLETED, + 'message' => 'Job done.' + ]); } public function correlateValue() { $this->ConfigLoad->execute(); $value = $this->args[0]; - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'correlateValue', - 'job_input' => $value, - 'status' => 0, - 'retries' => 0, - 'org' => 0, - 'message' => 'Job created.', - ); - $this->Job->save($data); + + if (!empty($this->args[1])) { + $this->Job->id = intval($this->args[1]); + } else { + $this->Job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'correlateValue', + $value, + 'Job created.' + ); + } + $this->Correlation->correlateValue($value, $this->Job->id); - $this->Job->saveField('status', 1); - $this->Job->saveField('message', 'Job done.'); + $this->Job->save([ + 'status' => Job::STATUS_COMPLETED, + 'message' => 'Job done.', + 'progress' => 100 + ]); } public function cache() @@ -299,23 +307,35 @@ class EventShell extends AppShell public function postsemail() { $this->ConfigLoad->execute(); - if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || - empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])) { + if ( + empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || + empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5]) + ) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Posts email'] . PHP_EOL); } - $userId = $this->args[0]; - $postId = $this->args[1]; - $eventId = $this->args[2]; + $userId = intval($this->args[0]); + $postId = intval($this->args[1]); + $eventId = intval($this->args[2]); $title = $this->args[3]; $message = $this->args[4]; - $processId = $this->args[5]; - $this->Job->id = $processId; + $this->Job->id = intval($this->args[5]); + $result = $this->Post->sendPostsEmail($userId, $postId, $eventId, $title, $message); - $job['Job']['progress'] = 100; - $job['Job']['message'] = 'Emails sent.'; - $job['Job']['date_modified'] = date("Y-m-d H:i:s"); - $this->Job->save($job); + + if ($result) { + $this->Job->save([ + 'progress' => 100, + 'message' => 'Emails sent.', + 'date_modified' => date('Y-m-d H:i:s'), + 'status' => Job::STATUS_COMPLETED + ]); + } else { + $this->Job->save([ + 'date_modified' => date('Y-m-d H:i:s'), + 'status' => Job::STATUS_FAILED + ]); + } } public function enqueueCaching() @@ -391,6 +411,7 @@ class EventShell extends AppShell $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable'); $result = $this->Event->publish($id, $passAlong); $job['Job']['progress'] = 100; + $job['Job']['status'] = Job::STATUS_COMPLETED; $job['Job']['date_modified'] = date("Y-m-d H:i:s"); if ($result) { $job['Job']['message'] = 'Event published.'; @@ -404,7 +425,6 @@ class EventShell extends AppShell public function publish_sightings() { - $this->ConfigLoad->execute(); if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish sightings'] . PHP_EOL); } @@ -414,14 +434,8 @@ class EventShell extends AppShell $sightingsUuidsToPush = []; if (isset($this->args[4])) { // push just specific sightings - $path = APP . 'tmp/cache/ingest' . DS . $this->args[4]; - $tempFile = new File($path); - $inputData = $tempFile->read(); - if ($inputData === false) { - $this->error("File `$path` not found."); - } - $sightingsUuidsToPush = $this->Event->jsonDecode($inputData); - $tempFile->delete(); + $path = $this->args[4][0] === '/' ? $this->args[4] : (APP . 'tmp/cache/ingest' . DS . $this->args[4]); + $sightingsUuidsToPush = $this->Event->jsonDecode(FileAccessTool::readAndDelete($path)); } $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable'); @@ -444,7 +458,7 @@ class EventShell extends AppShell public function publish_galaxy_clusters() { $this->ConfigLoad->execute(); - if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) { + if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) { die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL); } @@ -528,9 +542,8 @@ class EventShell extends AppShell $inputFile = $this->args[0]; $inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile; - $inputData = FileAccessTool::readFromFile($inputFile); + $inputData = FileAccessTool::readAndDelete($inputFile); $inputData = $this->Event->jsonDecode($inputData); - FileAccessTool::deleteFile($inputFile); Configure::write('CurrentUserId', $inputData['user']['id']); $this->Event->processFreeTextData( $inputData['user'], @@ -552,9 +565,8 @@ class EventShell extends AppShell $inputFile = $this->args[0]; $inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile; - $inputData = FileAccessTool::readFromFile($inputFile); + $inputData = FileAccessTool::readAndDelete($inputFile); $inputData = $this->Event->jsonDecode($inputData); - FileAccessTool::deleteFile($inputFile); Configure::write('CurrentUserId', $inputData['user']['id']); $this->Event->processModuleResultsData( $inputData['user'], diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 49bae8a7f..183ff901c 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -1,6 +1,7 @@ BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs')); + } + public function list() { $servers = $this->Server->find('all', [ @@ -78,12 +88,19 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - $jobId = CakeResque::enqueue( - 'default', - 'ServerShell', - array('pull', $user['id'], $serverId, $technique) + + $backgroundJobId = $this->BackgroundJobsTool->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'pull', + $user['id'], + $serverId, + $technique + ] ); - $this->out("Enqueued pulling from $serverName server as job $jobId"); + + $this->out("Enqueued pulling from $serverName server as job $backgroundJobId"); } } @@ -105,14 +122,14 @@ class ServerShell extends AppShell if (!empty($this->args[3])) { $jobId = $this->args[3]; } else { - $jobId = $this->Job->createJob($user,Job::WORKER_DEFAULT, 'pull', 'Server: ' . $serverId, 'Pulling.'); + $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'pull', 'Server: ' . $serverId, 'Pulling.'); } $force = false; if (!empty($this->args[4]) && $this->args[4] === 'force') { $force = true; } try { - $result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force); + $result = $this->Server->pull($user, $technique, $server, $jobId, $force); if (is_array($result)) { $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); $this->Job->saveStatus($jobId, true, $message); @@ -137,23 +154,12 @@ class ServerShell extends AppShell $user = $this->getUser($userId); $serverId = $this->args[1]; $server = $this->getServer($serverId); - if (!empty($this->args[2])) { - $jobId = $this->args[2]; + $technique = empty($this->args[2]) ? 'full' : $this->args[2]; + if (!empty($this->args[3])) { + $jobId = $this->args[3]; } else { - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'push', - 'job_input' => 'Server: ' . $serverId, - 'status' => 0, - 'retries' => 0, - 'org' => $user['Organisation']['name'], - 'message' => 'Pushing.', - ); - $this->Job->save($data); - $jobId = $this->Job->id; + $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'push', 'Server: ' . $serverId, 'Pushing.'); } - $technique = empty($this->args[3]) ? 'full' : $this->args[3]; $this->Job->read(null, $jobId); App::uses('SyncTool', 'Tools'); @@ -192,11 +198,18 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - $jobId = CakeResque::enqueue( - 'default', - 'ServerShell', - array('push', $user['id'], $serverId, $technique) + + $jobId = $this->BackgroundJobsTool->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'push', + $user['id'], + $serverId, + $technique + ] ); + $this->out("Enqueued pushing from $serverName server as job $jobId"); } } @@ -213,25 +226,13 @@ class ServerShell extends AppShell if (!empty($this->args[2])) { $jobId = $this->args[2]; } else { - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'fetch_feeds', - 'job_input' => 'Feed: ' . $feedId, - 'status' => 0, - 'retries' => 0, - 'org' => $user['Organisation']['name'], - 'message' => 'Starting fetch from Feed.', - ); - $this->Job->save($data); - $jobId = $this->Job->id; + $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.'); } - if ($feedId == 'all') { - $feedIds = $this->Feed->find('list', array( - 'fields' => array('Feed.id', 'Feed.id'), + if ($feedId === 'all') { + $feedIds = $this->Feed->find('column', array( + 'fields' => array('Feed.id'), 'conditions' => array('Feed.enabled' => 1) )); - $feedIds = array_values($feedIds); $successes = 0; $fails = 0; foreach ($feedIds as $k => $feedId) { @@ -247,21 +248,21 @@ class ServerShell extends AppShell $this->Job->saveStatus($jobId, true, $message); echo $message . PHP_EOL; } else { - $temp = $this->Feed->find('first', array( - 'fields' => array('Feed.id', 'Feed.id'), - 'conditions' => array('Feed.enabled' => 1, 'Feed.id' => $feedId) - )); - if (!empty($temp)) { + $feedEnabled = $this->Feed->hasAny([ + 'Feed.enabled' => 1, + 'Feed.id' => $feedId, + ]); + if ($feedEnabled) { $result = $this->Feed->downloadFromFeedInitiator($feedId, $user, $jobId); if (!$result) { - $this->Job->saveStatus($jobId, false); + $this->Job->saveStatus($jobId, false, 'Job failed. See error log for more details.'); echo 'Job failed.' . PHP_EOL; } else { $this->Job->saveStatus($jobId, true); echo 'Job done.' . PHP_EOL; } } else { - $message = "Feed with ID $feedId not found."; + $message = "Feed with ID $feedId not found or not enabled."; $this->Job->saveStatus($jobId, false, $message); echo $message . PHP_EOL; } @@ -317,11 +318,17 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - $jobId = CakeResque::enqueue( - 'default', - 'ServerShell', - array('cacheServer', $user['id'], $serverId) + + $jobId = $this->BackgroundJobsTool->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheServer', + $user['id'], + $serverId + ] ); + $this->out("Enqueued cacheServer from {$serverName} server as job $jobId"); } } @@ -338,7 +345,7 @@ class ServerShell extends AppShell if (!empty($this->args[2])) { $jobId = $this->args[2]; } else { - $jobId = $this->Job->createJob($user,Job::WORKER_DEFAULT, 'cache_feeds', 'Feed: ' . $scope, 'Starting feed caching.'); + $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'cache_feeds', 'Feed: ' . $scope, 'Starting feed caching.'); } try { $result = $this->Feed->cacheFeedInitiator($user, $jobId, $scope); @@ -399,7 +406,7 @@ class ServerShell extends AppShell ); $this->Job->save($data); $jobId = $this->Job->id; - $result = $this->Server->pull($user, $server['Server']['id'], 'full', $server, $jobId); + $result = $this->Server->pull($user, 'full', $server, $jobId); $this->Job->save(array( 'id' => $jobId, 'message' => 'Job done.', diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php new file mode 100644 index 000000000..7be815aa4 --- /dev/null +++ b/app/Console/Command/StartWorkerShell.php @@ -0,0 +1,104 @@ +BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs')); + } + + public function getOptionParser(): ConsoleOptionParser + { + $parser = parent::getOptionParser(); + $parser + ->addArgument('queue', [ + 'help' => 'Name of the queue to process.', + 'choices' => $this->BackgroundJobsTool->getQueues(), + 'required' => true + ]) + ->addOption( + 'maxExecutionTime', + [ + 'help' => 'Worker maximum execution time (seconds) before it self-destruct. Zero means unlimited.', + 'default' => self::DEFAULT_MAX_EXECUTION_TIME, + 'required' => false + ] + ); + + return $parser; + } + + public function main(): void + { + $this->worker = new Worker( + [ + 'pid' => getmypid(), + 'queue' => $this->args[0], + 'user' => $this->whoami() + ] + ); + + $this->maxExecutionTime = (int)$this->params['maxExecutionTime']; + + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs..."); + + while (true) { + $this->checkMaxExecutionTime(); + + $job = $this->BackgroundJobsTool->dequeue($this->worker->queue()); + + if ($job) { + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}..."); + + try { + $this->BackgroundJobsTool->run($job); + } catch (Exception $exception) { + CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}"); + $job->setStatus(BackgroundJob::STATUS_FAILED); + $this->BackgroundJobsTool->update($job); + } + } + } + } + + /** + * Checks if worker maximum execution time is reached, and exits if so. + * + * @return void + */ + private function checkMaxExecutionTime(): void + { + if ($this->maxExecutionTime === 0) { + return; + } + if ((time() - $this->worker->createdAt()) > $this->maxExecutionTime) { + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - worker max execution time reached, exiting gracefully worker..."); + exit; + } + } + + private function whoami(): string + { + if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) { + return posix_getpwuid(posix_geteuid())['name']; + } else { + return trim(shell_exec('whoami')); + } + } +} diff --git a/app/Console/Command/Task/ConfigLoadTask.php b/app/Console/Command/Task/ConfigLoadTask.php index 78e506803..b624dc573 100644 --- a/app/Console/Command/Task/ConfigLoadTask.php +++ b/app/Console/Command/Task/ConfigLoadTask.php @@ -1,7 +1,13 @@ +} diff --git a/app/Console/Command/UserShell.php b/app/Console/Command/UserShell.php index 6ce3720de..e7983fb7a 100644 --- a/app/Console/Command/UserShell.php +++ b/app/Console/Command/UserShell.php @@ -82,28 +82,31 @@ class UserShell extends AppShell public function list() { - // do not fetch sensitive or big values - $schema = $this->User->schema(); - unset($schema['authkey']); - unset($schema['password']); - unset($schema['gpgkey']); - unset($schema['certif_public']); - - $fields = array_keys($schema); - $fields[] = 'Role.*'; - $fields[] = 'Organisation.*'; - - $users = $this->User->find('all', [ - 'recursive' => -1, - 'fields' => $fields, - 'contain' => ['Organisation', 'Role'], - ]); - if ($this->params['json']) { + // do not fetch sensitive or big values + $schema = $this->User->schema(); + unset($schema['authkey']); + unset($schema['password']); + unset($schema['gpgkey']); + unset($schema['certif_public']); + + $fields = array_keys($schema); + $fields[] = 'Role.*'; + $fields[] = 'Organisation.*'; + + $users = $this->User->find('all', [ + 'recursive' => -1, + 'fields' => $fields, + 'contain' => ['Organisation', 'Role', 'UserSetting'], + ]); + $this->out($this->json($users)); } else { + $users = $this->User->find('column', [ + 'fields' => ['email'], + ]); foreach ($users as $user) { - $this->out($user['User']['email']); + $this->out($user); } } } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f8c7fdcce..b658933ac 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -31,10 +31,10 @@ class AppController extends Controller */ public $defaultModel = ''; - public $helpers = array('OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector'); + public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); private $__queryVersion = '131'; - public $pyMispVersion = '2.4.148'; + public $pyMispVersion = '2.4.151'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; @@ -69,34 +69,39 @@ class AppController extends Controller } public $components = array( - 'Session', - 'Auth' => array( - 'authError' => 'Unauthorised access.', - 'authenticate' => array( - 'Form' => array( - 'passwordHasher' => 'BlowfishConstant', - 'fields' => array( - 'username' => 'email' - ) + 'Session', + 'Auth' => array( + 'authError' => 'Unauthorised access.', + 'authenticate' => array( + 'Form' => array( + 'passwordHasher' => 'BlowfishConstant', + 'fields' => array( + 'username' => 'email' ) ) - ), - 'Security', - 'ACL', - 'CompressedRequestHandler', - 'RestResponse', - 'Flash', - 'Toolbox', - 'RateLimit', - 'IndexFilter', - 'Deprecation', - 'RestSearch', - 'CRUD' - //,'DebugKit.Toolbar' + ) + ), + 'Security', + 'ACL', + 'CompressedRequestHandler', + 'RestResponse', + 'Flash', + 'Toolbox', + 'RateLimit', + 'IndexFilter', + 'Deprecation', + 'RestSearch', + 'CRUD' + //,'DebugKit.Toolbar' ); public function beforeFilter() { + if (Configure::read('MISP.system_setting_db')) { + App::uses('SystemSetting', 'Model'); + SystemSetting::setGlobalSetting(); + } + $this->_setupBaseurl(); $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin'; @@ -132,7 +137,8 @@ class AppController extends Controller $this->_setupDatabaseConnection(); $this->set('debugMode', Configure::read('debug') >= 1 ? 'debugOn' : 'debugOff'); - $this->set('ajax', $this->request->is('ajax')); + $isAjax = $this->request->is('ajax'); + $this->set('ajax', $isAjax); $this->set('queryVersion', $this->__queryVersion); $this->User = ClassRegistry::init('User'); @@ -245,7 +251,7 @@ class AppController extends Controller $this->__logAccess($user); // Try to run updates - if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && $this->_isLive())) { + if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && !$isAjax && $this->_isLive())) { $this->User->runUpdates(); } @@ -265,7 +271,7 @@ class AppController extends Controller $user = $this->Auth->user(); // user info in session could change (see __verifyUser) method, so reload user variable if (isset($user['logged_by_authkey']) && $user['logged_by_authkey'] && !($this->_isRest() || $this->_isAutomation())) { - throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed"); + throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed."); } // Put token expiration time to response header that can be processed by automation tool @@ -321,12 +327,15 @@ class AppController extends Controller $preAuthActions[] = 'email_otp'; } if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) { - if (!$this->request->is('ajax')) { + if ($isAjax) { + $response = $this->RestResponse->throwException(401, "Unauthorized"); + $response->send(); + $this->_stop(); + } else { $this->Session->write('pre_login_requested_url', $this->request->here); + $this->_redirectToLogin(); } - $this->_redirectToLogin(); } - $this->set('me', false); } @@ -338,7 +347,7 @@ class AppController extends Controller $this->User->Server->updateDatabase('cleanSessionTable'); } } - if (Configure::read('site_admin_debug') && (Configure::read('debug') < 2)) { + if (Configure::read('site_admin_debug') && Configure::read('debug') < 2) { Configure::write('debug', 1); } } @@ -361,19 +370,15 @@ class AppController extends Controller } // Notifications and homepage is not necessary for AJAX or REST requests - if ($user && !$this->_isRest() && !$this->request->is('ajax')) { - if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') { - $notifications = $this->User->populateNotifications($user); - } else { - $notifications = $this->User->populateNotifications($user, 'fast'); - } - $this->set('notifications', $notifications); + if ($user && !$this->_isRest() && !$isAjax) { + $hasNotifications = $this->User->hasNotifications($user); + $this->set('hasNotifications', $hasNotifications); $homepage = $this->User->UserSetting->getValueForUser($user['id'], 'homepage'); if (!empty($homepage)) { $this->set('homepage', $homepage); } - if (version_compare(phpversion(), '8.0') >= 0) { + if (PHP_MAJOR_VERSION >= 8) { $this->Flash->error(__('WARNING: MISP is currently running under PHP 8.0, which is unsupported. Background jobs will fail, so please contact your administrator to run a supported PHP version (such as 7.4)')); } } @@ -906,22 +911,29 @@ class AppController extends Controller return $user; } - // generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params - protected function _harvestParameters($options, &$exception, $data = array()) + /** + * generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params + * @param array $options + * @param $exception + * @param array $data + * @return array|false|mixed + */ + protected function _harvestParameters($options, &$exception = null, $data = array()) { - if (!empty($options['request']->is('post'))) { - if (empty($options['request']->data)) { + $request = $options['request'] ?? $this->request; + if ($request->is('post')) { + if (empty($request->data)) { $exception = $this->RestResponse->throwException( 400, __('Either specify the search terms in the url, or POST a json with the filter parameters.'), - '/' . $this->request->params['controller'] . '/' . $this->request->action + '/' . $request->params['controller'] . '/' . $request->action ); return false; } else { - if (isset($options['request']->data['request'])) { - $data = array_merge($data, $options['request']->data['request']); + if (isset($request->data['request'])) { + $data = array_merge($data, $request->data['request']); } else { - $data = array_merge($data, $options['request']->data); + $data = array_merge($data, $request->data); } } } @@ -1224,6 +1236,9 @@ class AppController extends Controller if ($user === false) { return $exception; } + + session_write_close(); // Rest search can be longer, so close session to allow concurrent requests + if (isset($filters['returnFormat'])) { $returnFormat = $filters['returnFormat']; if ($returnFormat === 'download') { diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index e068b7cbb..7a019d06d 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -11,12 +11,23 @@ class AttributesController extends AppController { public $components = array('Security', 'RequestHandler'); - public $paginate = array( - 'limit' => 60, - 'maxLimit' => 9999, - 'conditions' => array('AND' => array('Attribute.deleted' => 0)), - 'order' => 'Attribute.event_id DESC' - ); + public $paginate = [ + 'limit' => 60, + 'maxLimit' => 9999, + 'conditions' => array('AND' => array('Attribute.deleted' => 0)), + 'order' => 'Attribute.event_id DESC', + 'recursive' => -1, + 'contain' => array( + 'Event' => array( + 'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'), + ), + 'AttributeTag', + 'Object' => array( + 'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id') + ), + 'SharingGroup' => ['fields' => ['SharingGroup.name']], + ), + ]; public function beforeFilter() { @@ -53,36 +64,19 @@ class AttributesController extends AppController $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now } } - // do not show private to other orgs - if (!$this->_isSiteAdmin()) { - $this->paginate = Set::merge($this->paginate, array('conditions' => $this->Attribute->buildConditions($this->Auth->user()))); - } } public function index() { - $this->Attribute->recursive = -1; - $this->paginate['recursive'] = -1; - $this->paginate['contain'] = array( - 'Event' => array( - 'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'), - ), - 'AttributeTag' => array('Tag'), - 'Object' => array( - 'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id') - ), - 'SharingGroup' => ['fields' => ['SharingGroup.name']], - ); - $this->Attribute->contain(array('AttributeTag' => array('Tag'))); - $this->set('isSearch', 0); + $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user()); $attributes = $this->paginate(); + if ($this->_isRest()) { - foreach ($attributes as $k => $attribute) { - $attributes[$k] = $attribute['Attribute']; - } + $attributes = array_column($attributes, 'Attribute'); return $this->RestResponse->viewData($attributes, $this->response->type()); } + $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]); $orgTable = $this->Attribute->Event->Orgc->find('all', [ 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], ]); @@ -94,6 +88,7 @@ class AttributesController extends AppController } list($attributes, $sightingsData) = $this->__searchUI($attributes); + $this->set('isSearch', 0); $this->set('sightingsData', $sightingsData); $this->set('orgTable', array_column($orgTable, 'name', 'id')); $this->set('shortDist', $this->Attribute->shortDist); @@ -920,28 +915,25 @@ class AttributesController extends AppController if (empty($attribute)) { return new CakeResponse(array('body'=> json_encode(array('fail' => false, 'errors' => 'Invalid attribute')), 'status' => 200, 'type' => 'json')); } - $this->Attribute->data = $attribute; - $this->Attribute->id = $attribute['Attribute']['id']; if (!$this->__canModifyEvent($attribute)) { return new CakeResponse(array('body' => json_encode(array('fail' => false, 'errors' => 'You do not have permission to do that')), 'status' => 200, 'type' => 'json')); } if (!$this->_isRest()) { $this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']); } - $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen'); - $changed = false; if (empty($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); if (empty($this->request->data['Attribute'])) { throw new MethodNotAllowedException(__('Invalid input.')); } } + $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen'); + $changed = false; foreach ($this->request->data['Attribute'] as $changedKey => $changedField) { - if (!in_array($changedKey, $validFields)) { + if (!in_array($changedKey, $validFields, true)) { throw new MethodNotAllowedException(__('Invalid field.')); } if ($attribute['Attribute'][$changedKey] == $changedField) { - $this->autoRender = false; return new CakeResponse(array('body'=> json_encode(array('errors'=> array('value' => 'nochange'))), 'status'=>200, 'type' => 'json')); } $attribute['Attribute'][$changedKey] = $changedField; @@ -952,16 +944,23 @@ class AttributesController extends AppController } $date = new DateTime(); $attribute['Attribute']['timestamp'] = $date->getTimestamp(); - if ($this->Attribute->save($attribute)) { - $this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id']); + + $fieldsToSave = ['timestamp']; + if ($changedKey === 'value') { + $fieldsToSave[] = 'value1'; + $fieldsToSave[] = 'value2'; + } else { + $fieldsToSave[] = $changedKey; + } + + if ($this->Attribute->save($attribute, true, $fieldsToSave)) { + $this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id'], false, $date->getTimestamp()); if ($attribute['Attribute']['object_id'] != 0) { $this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp()); } - $this->autoRender = false; return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.', 'check_publish' => true)), 'status'=>200, 'type' => 'json')); } else { - $this->autoRender = false; return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $this->Attribute->validationErrors)), 'status'=>200, 'type' => 'json')); } } @@ -1577,25 +1576,9 @@ class AttributesController extends AppController if (!isset($params['conditions']['Attribute.deleted'])) { $params['conditions']['Attribute.deleted'] = 0; } - $this->paginate = $params; - if (empty($this->paginate['limit'])) { - $this->paginate['limit'] = 60; - } - if (empty($this->paginate['page'])) { - $this->paginate['page'] = 1; - } - $this->paginate['recursive'] = -1; - $this->paginate['contain'] = array( - 'Event' => array( - 'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'), - ), - 'AttributeTag' => array('Tag'), - 'Object' => array( - 'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id') - ), - 'SharingGroup' => ['fields' => ['SharingGroup.name']], - ); + $this->paginate['conditions'] = $params['conditions']; $attributes = $this->paginate(); + $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]); $orgTable = $this->Attribute->Event->Orgc->find('all', [ 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], @@ -1649,7 +1632,7 @@ class AttributesController extends AppController } } - private function __searchUI($attributes) + private function __searchUI(array $attributes) { if (empty($attributes)) { return [[], []]; @@ -1661,9 +1644,9 @@ class AttributesController extends AppController $this->loadModel('AttachmentScan'); $user = $this->Auth->user(); $attributeIds = []; - foreach ($attributes as $k => $attribute) { - $attributeId = $attribute['Attribute']['id']; - $attributeIds[] = $attributeId; + $galaxyTags = []; + foreach ($attributes as &$attribute) { + $attributeIds[] = $attribute['Attribute']['id']; if ($this->Attribute->isImage($attribute['Attribute'])) { if (extension_loaded('gd')) { // if extension is loaded, the data is not passed to the view because it is asynchronously fetched @@ -1671,20 +1654,33 @@ class AttributesController extends AppController } else { $attribute['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']); } - $attributes[$k] = $attribute; } if ($attribute['Attribute']['type'] === 'attachment' && $this->AttachmentScan->isEnabled()) { $infected = $this->AttachmentScan->isInfected(AttachmentScan::TYPE_ATTRIBUTE, $attribute['Attribute']['id']); - $attributes[$k]['Attribute']['infected'] = $infected; + $attribute['Attribute']['infected'] = $infected; } if ($attribute['Attribute']['distribution'] == 4) { - $attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup']; + $attribute['Attribute']['SharingGroup'] = $attribute['SharingGroup']; } - $attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag']; - $attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($this->Auth->user(), $attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true); - unset($attributes[$k]['AttributeTag']); + $attribute['Attribute']['AttributeTag'] = $attribute['AttributeTag']; + foreach ($attribute['Attribute']['AttributeTag'] as $at) { + if (substr($at['Tag']['name'], 0, 12) === 'misp-galaxy:') { + $galaxyTags[] = $at['Tag']['name']; + } + } + unset($attribute['AttributeTag']); + } + unset($attribute); + + // Fetch galaxy clusters in one query + if (!empty($galaxyTags)) { + $this->loadModel('GalaxyCluster'); + $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false); + $clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id'); + } else { + $clusters = []; } // Fetch correlations in one query @@ -1696,6 +1692,27 @@ class AttributesController extends AppController $attributesWithFeedCorrelations = $this->Feed->attachFeedCorrelations(array_column($attributes, 'Attribute'), $user, $fakeEventArray); foreach ($attributes as $k => $attribute) { + // Assign galaxies + $galaxies = []; + foreach ($attribute['Attribute']['AttributeTag'] as $k2 => $attributeTag) { + if (!isset($clusters[$attributeTag['Tag']['id']])) { + continue; + } + $cluster = $clusters[$attributeTag['Tag']['id']]; + $galaxyId = $cluster['Galaxy']['id']; + $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false; + if (isset($attribute['Attribute']['Galaxy'][$galaxyId])) { + unset($cluster['Galaxy']); + $galaxies[$galaxyId]['GalaxyCluster'][] = $cluster; + } else { + $galaxies[$galaxyId] = $cluster['Galaxy']; + unset($cluster['Galaxy']); + $galaxies[$galaxyId]['GalaxyCluster'] = [$cluster]; + } + unset($attributes[$k]['Attribute']['AttributeTag'][$k2]); // remove galaxy tag + } + $attributes[$k]['Attribute']['Galaxy'] = array_values($galaxies); + if (isset($attributesWithFeedCorrelations[$k]['Feed'])) { $attributes[$k]['Attribute']['Feed'] = $attributesWithFeedCorrelations[$k]['Feed']; } @@ -1707,65 +1724,6 @@ class AttributesController extends AppController return array($attributes, $sightingsData); } - // If the checkbox for the alternate search is ticked, then this method is called to return the data to be represented - // This alternate view will show a list of events with matching search results and the percentage of those matched attributes being marked as to_ids - // events are sorted based on relevance (as in the percentage of matches being flagged as indicators for IDS) - public function searchAlternate($data) - { - $attributes = $this->Attribute->fetchAttributes( - $this->Auth->user(), - array( - 'conditions' => array( - 'AND' => $data - ), - 'contain' => array('Event' => array('Orgc' => array('fields' => array('Orgc.name')))), - 'fields' => array( - 'Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids', 'Attribute.value', 'Attribute.distribution', - 'Event.id', 'Event.org_id', 'Event.orgc_id', 'Event.info', 'Event.distribution', 'Event.attribute_count', 'Event.date', - ) - ) - ); - $events = array(); - foreach ($attributes as $attribute) { - if (isset($events[$attribute['Event']['id']])) { - if ($attribute['Attribute']['to_ids']) { - $events[$attribute['Event']['id']]['to_ids']++; - } else { - $events[$attribute['Event']['id']]['no_ids']++; - } - } else { - $events[$attribute['Event']['id']]['Event'] = $attribute['Event']; - $events[$attribute['Event']['id']]['to_ids'] = 0; - $events[$attribute['Event']['id']]['no_ids'] = 0; - if ($attribute['Attribute']['to_ids']) { - $events[$attribute['Event']['id']]['to_ids']++; - } else { - $events[$attribute['Event']['id']]['no_ids']++; - } - } - } - foreach ($events as $key => $event) { - $events[$key]['relevance'] = 100 * $event['to_ids'] / ($event['no_ids'] + $event['to_ids']); - } - if (!empty($events)) { - $events = $this->__subval_sort($events, 'relevance'); - } - return $events; - } - - // Sort the array of arrays based on a value of a sub-array - private function __subval_sort($a, $subkey) - { - foreach ($a as $k=>$v) { - $b[$k] = strtolower($v[$subkey]); - } - arsort($b); - foreach ($b as $key=>$val) { - $c[] = $a[$key]; - } - return $c; - } - public function checkComposites() { if (!self::_isAdmin()) { @@ -1960,26 +1918,27 @@ class AttributesController extends AppController $this->Flash->success(__('All done. ' . $k . ' attributes processed.')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } else { + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generate correlation', - 'job_input' => 'All attributes', - 'status' => 0, - 'retries' => 0, - 'org' => 'ADMIN', - 'message' => 'Job created.', + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate correlation', + 'All attributes', + 'Job created.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('jobGenerateCorrelation', $jobId), - true + + $this->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateCorrelation', + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } @@ -2673,16 +2632,11 @@ class AttributesController extends AppController } } } else { - $conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id))); - if (!$this->_isSiteAdmin()) { - $conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id')); - $conditions['Tag.user_id'] = array('0', $this->Auth->user('id')); - } - $tag = $this->Attribute->AttributeTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions)); - if (empty($tag)) { + $tagId = $this->Attribute->AttributeTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id)); + if (empty($tagId)) { return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json')); } - $tag_id = $tag['Tag']['id']; + $tag_id = $tagId; } } } diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php index 3b2fcf354..74d1e93e8 100644 --- a/app/Controller/AuditLogsController.php +++ b/app/Controller/AuditLogsController.php @@ -37,6 +37,7 @@ class AuditLogsController extends AppController 'Server', 'ShadowAttribute', 'SharingGroup', + 'SystemSetting', 'Tag', 'TagCollection', 'TagCollectionTag', @@ -484,10 +485,12 @@ class AuditLogsController extends AppController if (!empty($eventIds)) { $this->loadModel('Event'); - $events = $this->Event->fetchSimpleEvents($this->Auth->user(), [ - 'conditions' => ['Event.id' => array_unique($eventIds)], + $conditions = $this->Event->createEventConditions($this->Auth->user()); + $conditions['Event.id'] = array_unique($eventIds); + $events = $this->Event->find('list', [ + 'conditions' => $conditions, + 'fields' => ['Event.id', 'Event.info'], ]); - $events = array_column(array_column($events, 'Event'), null, 'id'); } $links = [ @@ -525,7 +528,7 @@ class AuditLogsController extends AppController case 'Event': if (isset($events[$modelId])) { $url = '/events/view/' . $modelId; - $eventInfo = $events[$modelId]['info']; + $eventInfo = $events[$modelId]; } break; case 'ObjectReference': @@ -535,7 +538,7 @@ class AuditLogsController extends AppController $url .= '/deleted:2'; } if (isset($events[$objects[$objectReferences[$modelId]]['event_id']])) { - $eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']]['info']; + $eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']]; } } break; @@ -546,7 +549,7 @@ class AuditLogsController extends AppController $url .= '/deleted:2'; } if (isset($events[$objects[$modelId]['event_id']])) { - $eventInfo = $events[$objects[$modelId]['event_id']]['info']; + $eventInfo = $events[$objects[$modelId]['event_id']]; } } break; @@ -557,7 +560,7 @@ class AuditLogsController extends AppController $url .= '/deleted:2'; } if (isset($events[$attributes[$modelId]['event_id']])) { - $eventInfo = $events[$attributes[$modelId]['event_id']]['info']; + $eventInfo = $events[$attributes[$modelId]['event_id']]; } } break; @@ -565,7 +568,7 @@ class AuditLogsController extends AppController if (isset($shadowAttributes[$modelId])) { $url = '/events/view/' . $shadowAttributes[$modelId]['event_id'] . '/focus:' . $shadowAttributes[$modelId]['uuid']; if (isset($events[$shadowAttributes[$modelId]['event_id']])) { - $eventInfo = $events[$shadowAttributes[$modelId]['event_id']]['info']; + $eventInfo = $events[$shadowAttributes[$modelId]['event_id']]; } } break; diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 440408f79..778d60f4d 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -54,7 +54,6 @@ class ACLComponent extends Component 'returnAttributes' => array('*'), 'rpz' => array('*'), 'search' => array('*'), - 'searchAlternate' => array('*'), 'toggleCorrelation' => array('perm_add'), 'text' => array('*'), 'toggleToIDS' => array('perm_add'), diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index b1c902c31..6dd49a708 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -512,6 +512,7 @@ class RestResponseComponent extends Component * @param bool $download * @param array $headers * @return CakeResponse + * @throws Exception */ private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array()) { @@ -535,7 +536,7 @@ class RestResponseComponent extends Component $type = 'xml'; } elseif ($format === 'openioc') { $type = 'xml'; - } elseif ($format === 'csv') { + } elseif ($format === 'csv' || $format === 'text/csv') { $type = 'csv'; } else { if (empty($format)) { @@ -582,6 +583,12 @@ class RestResponseComponent extends Component } App::uses('TmpFileTool', 'Tools'); + if ($response instanceof Generator) { + $tmpFile = new TmpFileTool(); + $tmpFile->writeWithSeparator($response, null); + $response = $tmpFile; + } + if ($response instanceof TmpFileTool) { App::uses('CakeResponseTmp', 'Tools'); $cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 63e2fdbdf..726ad56ac 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -28,20 +28,22 @@ class EventsController extends AppController ) ); - private $acceptedFilteringNamedParams = array( - 'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'extended', 'page', + // private + const ACCEPTED_FILTERING_NAMED_PARAMS = array( + 'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page', 'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution', - 'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'focus', 'extended', 'overrideLimit', - 'filterColumnsOverwrite', 'feed', 'server', 'toIDS', 'sighting', 'includeSightingdb', 'warninglistId' + 'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS', + 'sighting', 'includeSightingdb', 'warninglistId' ); - public $defaultFilteringRules = array( + // private + const DEFAULT_FILTERING_RULE = array( 'searchFor' => '', 'attributeFilter' => 'all', 'proposal' => 0, 'correlation' => 0, 'warning' => 0, - 'deleted' => 2, + 'deleted' => 0, 'includeRelatedTags' => 0, 'includeDecayScore' => 0, 'toIDS' => 0, @@ -115,7 +117,7 @@ class EventsController extends AppController $excludeIDs = []; if (!empty($value)) { if (!is_array($value)) { - $pieces = explode('|', strtolower($value)); + $pieces = explode('|', mb_strtolower($value)); } else { $pieces = $value; } @@ -131,8 +133,8 @@ class EventsController extends AppController if (!empty($include)) { $includeConditions = []; foreach ($include as $i) { - $includeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $i); - $includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i); + $includeConditions['OR'][] = array('Attribute.value1 LIKE' => $i); + $includeConditions['OR'][] = array('Attribute.value2 LIKE' => $i); } $includeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( @@ -146,8 +148,8 @@ class EventsController extends AppController if (!empty($exclude)) { $excludeConditions = []; foreach ($exclude as $e) { - $excludeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $e); - $excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e); + $excludeConditions['OR'][] = array('Attribute.value1 LIKE' => $e); + $excludeConditions['OR'][] = array('Attribute.value2 LIKE' => $e); } $excludeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( @@ -257,9 +259,10 @@ class EventsController extends AppController /** * @param array $passedArgs * @param string $urlparams + * @param bool $nothing True when nothing should be fetched from database * @return array */ - private function __setIndexFilterConditions(array $passedArgs, &$urlparams) + private function __setIndexFilterConditions(array $passedArgs, &$urlparams, &$nothing = false) { $passedArgsArray = array(); foreach ($passedArgs as $k => $v) { @@ -275,8 +278,8 @@ class EventsController extends AppController $searchTerm = strtolower(substr($k, 6)); switch ($searchTerm) { case 'all': - if (!empty($passedArgs['searchall'])) { - $this->paginate['conditions']['AND'][] = array('Event.id' => $this->__quickFilter($passedArgs['searchall'])); + if (!empty($v)) { + $this->paginate['conditions']['AND'][] = array('Event.id' => $this->__quickFilter($v)); } break; case 'attribute': @@ -292,6 +295,9 @@ class EventsController extends AppController if ($v === 2 || $v === '2') { // both continue 2; } + if (is_array($v) && in_array(0, $v) && in_array(1, $v)) { + continue 2; // both + } $this->paginate['conditions']['AND'][] = array('Event.published' => $v); break; case 'hasproposal': @@ -312,7 +318,7 @@ class EventsController extends AppController $eventidConditions = array(); foreach ($pieces as $piece) { $piece = trim($piece); - if ($piece[0] == '!') { + if ($piece[0] === '!') { if (strlen($piece) === 37) { $eventidConditions['NOT']['uuid'][] = substr($piece, 1); } else { @@ -385,16 +391,15 @@ class EventsController extends AppController $this->Event->Org->virtualFields = [ 'upper_name' => 'UPPER(name)', - 'lower_uuid' => 'LOWER(name)', + 'lower_uuid' => 'LOWER(uuid)', ]; $orgs = array_column($this->Event->Org->find('all', [ 'fields' => ['Org.id', 'Org.upper_name', 'Org.lower_uuid'], 'recursive' => -1, ]), 'Org'); - unset($this->Event->Org->virtualFields['upper_name']); - unset($this->Event->Org->virtualFields['lower_uuid']); - $orgByName = array_column($orgs, null, 'upper_name'); - $orgByUuid = array_column($orgs, null, 'lower_uuid'); + $this->Event->Org->virtualFields = []; + $orgByName = array_column($orgs, 'id', 'upper_name'); + $orgByUuid = array_column($orgs, 'id', 'lower_uuid'); // if the first character is '!', search for NOT LIKE the rest of the string (excluding the '!' itself of course) $pieces = is_array($v) ? $v : explode('|', $v); $test = array(); @@ -404,28 +409,28 @@ class EventsController extends AppController if (is_numeric($piece)) { $orgId = $piece; } else if (Validation::uuid($piece)) { - $orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece]['id'] : null; + $orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece] : null; } else { $orgName = mb_strtoupper($piece); - $orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName]['id'] : null; + $orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName] : null; } if ($orgId) { $this->paginate['conditions']['AND'][] = array('Event.orgc_id !=' => $orgId); } } else { if (is_numeric($piece)) { - $test['OR'][] = array('Event.orgc_id' => array('Event.orgc_id' => $piece)); + $test['OR'][] = array('Event.orgc_id' => $piece); } else { if (Validation::uuid($piece)) { - $orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece]['id'] : null; + $orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece] : null; } else { $orgName = mb_strtoupper($piece); - $orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName]['id'] : null; + $orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName] : null; } if ($orgId) { $test['OR'][] = array('Event.orgc_id' => $orgId); } else { - $test['OR'][] = array('Event.orgc_id' => -1); + $nothing = true; } } } @@ -436,7 +441,7 @@ class EventsController extends AppController $pieces = explode('|', $v); $test = array(); foreach ($pieces as $piece) { - if ($piece[0] == '!') { + if ($piece[0] === '!') { $this->paginate['conditions']['AND'][] = array('Event.sharing_group_id !=' => substr($piece, 1)); } else { $test['OR'][] = array('Event.sharing_group_id' => $piece); @@ -454,10 +459,10 @@ class EventsController extends AppController $pieces = explode('|', $v); $test = array(); foreach ($pieces as $piece) { - if ($piece[0] == '!') { - $this->paginate['conditions']['AND'][] = array('lower(Event.info) NOT LIKE' => '%' . strtolower(substr($piece, 1)) . '%'); + if ($piece[0] === '!') { + $this->paginate['conditions']['AND'][] = array('lower(Event.info) NOT LIKE' => '%' . mb_strtolower(substr($piece, 1)) . '%'); } else { - $test['OR'][] = array('lower(Event.info) LIKE' => '%' . strtolower($piece) . '%'); + $test['OR'][] = array('lower(Event.info) LIKE' => '%' . mb_strtolower($piece) . '%'); } } $this->paginate['conditions']['AND'][] = $test; @@ -470,14 +475,13 @@ class EventsController extends AppController $pieces = is_array($v) ? $v : explode('|', $v); $filterString = ""; $expectOR = false; - $setOR = false; $tagRules = []; foreach ($pieces as $piece) { - if ($piece[0] == '!') { + if ($piece[0] === '!') { if (is_numeric(substr($piece, 1))) { - $conditions = array('OR' => array('Tag.id' => substr($piece, 1))); + $conditions = array('Tag.id' => substr($piece, 1)); } else { - $conditions = array('OR' => array('Tag.name' => substr($piece, 1))); + $conditions = array('Tag.name' => substr($piece, 1)); } $tagName = $this->Event->EventTag->Tag->find('first', array( 'conditions' => $conditions, @@ -492,26 +496,18 @@ class EventsController extends AppController $filterString .= '!' . $piece; continue; } - $block = $this->Event->EventTag->find('column', array( - 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), - 'fields' => ['EventTag.event_id'], - )); - if (!empty($block)) { - $sqlSubQuery = 'Event.id NOT IN (' . implode(",", $block) . ')'; - $tagRules['AND'][] = $sqlSubQuery; - } + $tagRules['block'][] = $tagName['Tag']['id']; if ($filterString != "") { $filterString .= "|"; } - $filterString .= '!' . (isset($tagName['Tag']['name']) ? $tagName['Tag']['name'] : $piece); + $filterString .= '!' . $tagName['Tag']['name']; } else { $expectOR = true; if (is_numeric($piece)) { - $conditions = array('OR' => array('Tag.id' => $piece)); + $conditions = array('Tag.id' => $piece); } else { - $conditions = array('OR' => array('Tag.name' => $piece)); + $conditions = array('Tag.name' => $piece); } - $tagName = $this->Event->EventTag->Tag->find('first', array( 'conditions' => $conditions, 'fields' => array('id', 'name'), @@ -524,61 +520,87 @@ class EventsController extends AppController $filterString .= $piece; continue; } - - $allow = $this->Event->EventTag->find('column', array( - 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), - 'fields' => ['EventTag.event_id'], - )); - if (!empty($allow)) { - $setOR = true; - $sqlSubQuery = 'Event.id IN (' . implode(",", $allow) . ')'; - $tagRules['OR'][] = $sqlSubQuery; - } + $tagRules['include'][] = $tagName['Tag']['id']; if ($filterString != "") { $filterString .= "|"; } - $filterString .= isset($tagName['Tag']['name']) ? $tagName['Tag']['name'] : $piece; + $filterString .= $tagName['Tag']['name']; } } - $this->paginate['conditions']['AND'][] = $tagRules; - // If we have a list of OR-d arguments, we expect to end up with a list of allowed event IDs - // If we don't however, it means that none of the tags was found. To prevent displaying the entire event index in this case: - if ($expectOR && !$setOR) { - $this->paginate['conditions']['AND'][] = array('Event.id' => -1); + + if (!empty($tagRules['block'])) { + $block = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagRules['block']), + 'fields' => ['EventTag.event_id'], + )); + if (!empty($block)) { + $this->paginate['conditions']['AND'][] = 'Event.id NOT IN (' . implode(",", $block) . ')'; + } } + + if (!empty($tagRules['include'])) { + $include = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagRules['include']), + 'fields' => ['EventTag.event_id'], + )); + if (!empty($include)) { + $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')'; + } else { + $nothing = true; + } + } else if ($expectOR) { + // If we have a list of OR-d arguments, we expect to end up with a list of allowed event IDs + // If we don't however, it means that none of the tags was found. To prevent displaying the entire event index in this case: + $nothing = true; + } + $v = $filterString; break; case 'email': - if ($v == "" || (strtolower($this->Auth->user('email')) !== strtolower(trim($v)) && !$this->_isSiteAdmin())) { + if ($v == "") { continue 2; } + + if (!$this->_isSiteAdmin()) { + // Special case to filter own events + if (strtolower($this->Auth->user('email')) === strtolower(trim($v))) { + $this->paginate['conditions']['AND'][] = ['Event.user_id' => $this->Auth->user('id')]; + break; + } else { + $nothing = true; + continue 2; + } + } + // if the first character is '!', search for NOT LIKE the rest of the string (excluding the '!' itself of course) $pieces = explode('|', $v); - $test = array(); + $usersToMatch = array(); + $positiveQuery = false; foreach ($pieces as $piece) { if ($piece[0] === '!') { $users = $this->Event->User->find('column', array( - 'recursive' => -1, 'fields' => array('User.id'), - 'conditions' => array('lower(User.email) LIKE' => '%' . strtolower(substr($piece, 1)) . '%') + 'conditions' => array('User.email LIKE' => '%' . strtolower(substr($piece, 1)) . '%') )); if (!empty($users)) { $this->paginate['conditions']['AND'][] = array('Event.user_id !=' => $users); } } else { + $positiveQuery = true; $users = $this->Event->User->find('column', array( - 'recursive' => -1, 'fields' => array('User.id'), - 'conditions' => array('lower(User.email) LIKE' => '%' . strtolower($piece) . '%') + 'conditions' => array('User.email LIKE' => '%' . strtolower($piece) . '%') )); - if (!empty($users)) { - $test['OR'][] = array('Event.user_id' => $users); - } + $usersToMatch = array_merge($usersToMatch, $users); } } - if (!empty($test)) { - $this->paginate['conditions']['AND'][] = $test; + if ($positiveQuery) { + if (empty($usersToMatch)) { + $nothing = true; + } else { + $this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch)]; + } } break; case 'distribution': @@ -597,7 +619,7 @@ class EventsController extends AppController } else { $terms = $this->Event->distributionLevels; } - $pieces = is_array($v) ? $v : explode('|', $v); + $pieces = is_array($v) ? $v : explode('|', $v); $test = array(); foreach ($pieces as $piece) { if ($filterString != "") { @@ -616,7 +638,7 @@ class EventsController extends AppController break; case 'minimal': $tableName = $this->Event->EventReport->table; - $eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName); + $eventReportQuery = sprintf('EXISTS (SELECT id FROM %s WHERE %s.event_id = Event.id AND %s.deleted = 0)', $tableName, $tableName, $tableName); $this->paginate['conditions']['AND'][] = [ 'OR' => [ ['Event.attribute_count >' => 0], @@ -639,15 +661,15 @@ class EventsController extends AppController $overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal'); $paginationParams = array('limit', 'page', 'sort', 'direction', 'order'); $passedArgs = $this->passedArgs; - if (isset($this->request->data)) { + if (!empty($this->request->data)) { if (isset($this->request->data['request'])) { $this->request->data = $this->request->data['request']; } foreach ($this->request->data as $k => $v) { - if (substr($k, 0, 6) === 'search' && in_array(strtolower(substr($k, 6)), $overrideAbleParams)) { + if (substr($k, 0, 6) === 'search' && in_array(strtolower(substr($k, 6)), $overrideAbleParams, true)) { unset($this->request->data[$k]); $this->request->data[strtolower(substr($k, 6))] = $v; - } else if (in_array(strtolower($k), $overrideAbleParams)) { + } else if (in_array(strtolower($k), $overrideAbleParams, true)) { unset($this->request->data[$k]); $this->request->data[strtolower($k)] = $v; } @@ -665,150 +687,38 @@ class EventsController extends AppController } // check each of the passed arguments whether they're a filter (could also be a sort for example) and if yes, add it to the pagination conditions - $passedArgsArray = $this->__setIndexFilterConditions($passedArgs, $urlparams); + $nothing = false; + $passedArgsArray = $this->__setIndexFilterConditions($passedArgs, $urlparams, $nothing); $this->loadModel('GalaxyCluster'); // for REST, don't use the pagination. With this, we'll escape the limit of events shown on the index. if ($this->_isRest()) { - $rules = array(); - $fieldNames = array_keys($this->Event->getColumnTypes()); - $directions = array('ASC', 'DESC'); - if (isset($passedArgs['sort']) && in_array($passedArgs['sort'], $fieldNames)) { - if (isset($passedArgs['direction']) && in_array(strtoupper($passedArgs['direction']), $directions)) { - $rules['order'] = array('Event.' . $passedArgs['sort'] => $passedArgs['direction']); - } else { - $rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC'); - } + if ($nothing) { + return $this->RestResponse->viewData([], $this->response->type(), false, false, false, ['X-Result-Count' => 0]); } - $rules['contain'] = $this->paginate['contain']; - if (isset($this->paginate['conditions'])) { - $rules['conditions'] = $this->paginate['conditions']; - } - $minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']); - if ($minimal) { - $rules['recursive'] = -1; - $rules['fields'] = array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid'); - $rules['contain'] = array('Orgc.uuid'); - } else { - $rules['contain'][] = 'EventTag'; - } - $paginationRules = array('page', 'limit', 'sort', 'direction', 'order'); - foreach ($paginationRules as $paginationRule) { - if (isset($passedArgs[$paginationRule])) { - $rules[$paginationRule] = $passedArgs[$paginationRule]; - } - } - - if (empty($rules['limit'])) { - $events = array(); - $i = 1; - $rules['limit'] = 20000; - while (true) { - $rules['page'] = $i; - $temp = $this->Event->find('all', $rules); - $resultCount = count($temp); - if ($resultCount !== 0) { - $events = array_merge($events, $temp); - } - if ($resultCount < $rules['limit']) { - break; - } - $i += 1; - } - $absolute_total = count($events); - } else { - $counting_rules = $rules; - unset($counting_rules['limit']); - unset($counting_rules['page']); - $absolute_total = $this->Event->find('count', $counting_rules); - - $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules); - } - - if (!$minimal) { - $tagIds = []; - foreach (array_column($events, 'EventTag') as $eventTags) { - foreach (array_column($eventTags, 'tag_id') as $tagId) { - $tagIds[$tagId] = true; - } - } - if (!empty($tagIds)) { - $tags = $this->Event->EventTag->Tag->find('all', [ - 'conditions' => [ - 'Tag.id' => array_keys($tagIds), - 'Tag.exportable' => 1, - ], - 'recursive' => -1, - 'fields' => ['Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy'], - ]); - unset($tagIds); - $tags = array_column(array_column($tags, 'Tag'), null, 'id'); - - foreach ($events as $k => $event) { - if (empty($event['EventTag'])) { - continue; - } - foreach ($event['EventTag'] as $k2 => $et) { - if (!isset($tags[$et['tag_id']])) { - unset($events[$k]['EventTag'][$k2]); // tag not exists or is not exportable - } else { - $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]; - } - } - $events[$k]['EventTag'] = array_values($events[$k]['EventTag']); - } - $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, false); - } - foreach ($events as $key => $event) { - if (empty($event['SharingGroup']['name'])) { - unset($event['SharingGroup']); - } - - $temp = $event['Event']; - $temp['Org'] = $event['Org']; - $temp['Orgc'] = $event['Orgc']; - unset($temp['user_id']); - $rearrangeObjects = array('GalaxyCluster', 'EventTag', 'SharingGroup'); - foreach ($rearrangeObjects as $ro) { - if (isset($event[$ro])) { - $temp[$ro] = $event[$ro]; - } - } - $events[$key] = $temp; - } - if ($this->response->type() === 'application/xml') { - $events = array('Event' => $events); - } - } else { - foreach ($events as $key => $event) { - $event['Event']['orgc_uuid'] = $event['Orgc']['uuid']; - $events[$key] = $event['Event']; - } - } - return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]); + return $this->__indexRestResponse($passedArgs); } $this->paginate['contain']['ThreatLevel'] = [ 'fields' => array('ThreatLevel.name') ]; - $this->paginate['contain'][] = 'EventTag'; + $this->paginate['contain']['EventTag'] = [ + 'fields' => ['EventTag.event_id', 'EventTag.tag_id', 'EventTag.local'], + ]; if ($this->_isSiteAdmin()) { $this->paginate['contain'][] = 'User.email'; } + if ($nothing) { + $this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event + } + $events = $this->paginate(); if (count($events) === 1 && isset($this->passedArgs['searchall'])) { $this->redirect(array('controller' => 'events', 'action' => 'view', $events[0]['Event']['id'])); } - if ($this->params['ext'] === 'csv') { - $events = $this->__attachInfoToEvents(['tags'], $events); - App::uses('CsvExport', 'Export'); - $export = new CsvExport(); - return $this->RestResponse->viewData($export->eventIndex($events), 'csv'); - } - list($possibleColumns, $enabledColumns) = $this->__indexColumns(); $events = $this->__attachInfoToEvents($enabledColumns, $events); @@ -832,6 +742,183 @@ class EventsController extends AppController } } + /** + * @param array $passedArgs + * @return CakeResponse + */ + private function __indexRestResponse(array $passedArgs) + { + $fieldNames = $this->Event->schema(); + $minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']); + if ($minimal) { + $rules = [ + 'recursive' => -1, + 'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid'), + 'contain' => array('Orgc.uuid'), + ]; + } else { + // Remove user ID from fetched fields + unset($fieldNames['user_id']); + $rules = [ + 'contain' => ['EventTag'], + 'fields' => array_keys($fieldNames), + ]; + } + if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) { + if (isset($passedArgs['direction']) && in_array(strtoupper($passedArgs['direction']), ['ASC', 'DESC'])) { + $rules['order'] = array('Event.' . $passedArgs['sort'] => $passedArgs['direction']); + } else { + $rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC'); + } + } + if (isset($this->paginate['conditions'])) { + $rules['conditions'] = $this->paginate['conditions']; + } + $paginationRules = array('page', 'limit', 'sort', 'direction', 'order'); + foreach ($paginationRules as $paginationRule) { + if (isset($passedArgs[$paginationRule])) { + $rules[$paginationRule] = $passedArgs[$paginationRule]; + } + } + + if (empty($rules['limit'])) { + $events = array(); + $i = 1; + $rules['limit'] = 20000; + while (true) { + $rules['page'] = $i; + $temp = $this->Event->find('all', $rules); + $resultCount = count($temp); + if ($resultCount !== 0) { + // this is faster and memory efficient than array_merge + foreach ($temp as $tempEvent) { + $events[] = $tempEvent; + } + } + if ($resultCount < $rules['limit']) { + break; + } + $i++; + } + unset($temp); + $absolute_total = count($events); + } else { + $counting_rules = $rules; + unset($counting_rules['limit']); + unset($counting_rules['page']); + $absolute_total = $this->Event->find('count', $counting_rules); + + $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules); + } + + $isCsvResponse = $this->response->type() === 'text/csv'; + + if (!$minimal) { + // Collect all tag IDs that are events + $tagIds = []; + foreach (array_column($events, 'EventTag') as $eventTags) { + foreach (array_column($eventTags, 'tag_id') as $tagId) { + $tagIds[$tagId] = true; + } + } + + if (!empty($tagIds)) { + $tags = $this->Event->EventTag->Tag->find('all', [ + 'conditions' => [ + 'Tag.id' => array_keys($tagIds), + 'Tag.exportable' => 1, + ], + 'recursive' => -1, + 'fields' => ['Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy'], + ]); + unset($tagIds); + $tags = array_column(array_column($tags, 'Tag'), null, 'id'); + + foreach ($events as $k => $event) { + if (empty($event['EventTag'])) { + continue; + } + foreach ($event['EventTag'] as $k2 => $et) { + if (!isset($tags[$et['tag_id']])) { + unset($events[$k]['EventTag'][$k2]); // tag not exists or is not exportable + } else { + $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]; + } + } + $events[$k]['EventTag'] = array_values($events[$k]['EventTag']); + } + if (!$isCsvResponse) { + $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, false); + } + } + + // Fetch all org and sharing groups that are in events + $orgIds = []; + $sharingGroupIds = []; + foreach ($events as $event) { + $orgIds[$event['Event']['org_id']] = true; + $orgIds[$event['Event']['orgc_id']] = true; + $sharingGroupIds[$event['Event']['sharing_group_id']] = true; + } + if (!empty($orgIds)) { + $orgs = $this->Event->Org->find('all', [ + 'conditions' => ['Org.id' => array_keys($orgIds)], + 'recursive' => -1, + 'fields' => $this->paginate['contain']['Org']['fields'], + ]); + unset($orgIds); + $orgs = array_column(array_column($orgs, 'Org'), null, 'id'); + } else { + $orgs = []; + } + + unset($sharingGroupIds[0]); + if (!empty($sharingGroupIds)) { + $sharingGroups = $this->Event->SharingGroup->find('all', [ + 'conditions' => ['SharingGroup.id' => array_keys($sharingGroupIds)], + 'recursive' => -1, + 'fields' => $this->paginate['contain']['SharingGroup']['fields'], + ]); + unset($sharingGroupIds); + $sharingGroups = array_column(array_column($sharingGroups, 'SharingGroup'), null, 'id'); + } + + foreach ($events as $key => $event) { + $temp = $event['Event']; + $temp['Org'] = $orgs[$temp['org_id']]; + $temp['Orgc'] = $orgs[$temp['orgc_id']]; + if ($temp['sharing_group_id'] != 0) { + $temp['SharingGroup'] = $sharingGroups[$temp['sharing_group_id']]; + } + $rearrangeObjects = array('GalaxyCluster', 'EventTag'); + foreach ($rearrangeObjects as $ro) { + if (isset($event[$ro])) { + $temp[$ro] = $event[$ro]; + } + } + $events[$key] = $temp; + } + unset($sharingGroups); + unset($orgs); + if ($this->response->type() === 'application/xml') { + $events = array('Event' => $events); + } + } else { + foreach ($events as $key => $event) { + $event['Event']['orgc_uuid'] = $event['Orgc']['uuid']; + $events[$key] = $event['Event']; + } + } + + if ($isCsvResponse) { + App::uses('CsvExport', 'Export'); + $export = new CsvExport(); + $events = $export->eventIndex($events); + } + + return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]); + } + private function __indexColumns() { $possibleColumns = []; @@ -1054,13 +1141,11 @@ class EventsController extends AppController * * @param array $attribute An attribute * @param array $fields List of keys in attribute to search in - * @param string $searchValue Values to search ( '|' is the separator) + * @param array $searchParts Values to search * @return bool Returns true on match */ - private function __valueInFieldAttribute($attribute, $fields, $searchValue) + private function __valueInFieldAttribute($attribute, $fields, $searchParts) { - $searchParts = explode('|', mb_strtolower($searchValue)); - foreach ($fields as $field) { if (strpos($field, 'Tag') === 0) { if (empty($attribute['AttributeTag'])) { @@ -1076,11 +1161,10 @@ class EventsController extends AppController } } } else { - $fieldValue = isset($attribute[$field]) ? $attribute[$field] : null; - if (empty($fieldValue)) { + if (!isset($attribute[$field])) { continue; } - $fieldValue = mb_strtolower($fieldValue); + $fieldValue = mb_strtolower($attribute[$field]); foreach ($searchParts as $s) { if (strpos($fieldValue, $s) !== false) { return true; @@ -1094,12 +1178,17 @@ class EventsController extends AppController public function viewEventAttributes($id, $all = false) { $filterData = array( - 'request' => $this->request, - 'paramArray' => $this->acceptedFilteringNamedParams, + 'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS, 'named_params' => $this->request->params['named'] ); - $exception = false; - $filters = $this->_harvestParameters($filterData, $exception); + $filters = $this->_harvestParameters($filterData); + + // Remove default filters + foreach ($filters as $filterName => $filterValue) { + if (isset(self::DEFAULT_FILTERING_RULE[$filterName]) && self::DEFAULT_FILTERING_RULE[$filterName] == $filterValue) { + unset($filters[$filterName]); + } + } if (isset($filters['focus'])) { $this->set('focus', $filters['focus']); @@ -1114,6 +1203,7 @@ class EventsController extends AppController 'includeEventCorrelations' => false, 'noEventReports' => true, // event reports for view are loaded dynamically 'noSightings' => true, + 'includeServerCorrelations' => $filters['includeServerCorrelations'] ?? 1. ]; if (isset($filters['extended'])) { $conditions['extended'] = 1; @@ -1128,19 +1218,14 @@ class EventsController extends AppController if ($filters['deleted'] == 1) { // both $conditions['deleted'] = [0, 1]; } elseif ($filters['deleted'] == 0) { // not-deleted only - $conditions['deleted'] = 1; - } else { // only deleted $conditions['deleted'] = 0; + } else { // only deleted + $conditions['deleted'] = 1; } } if (isset($filters['toIDS']) && $filters['toIDS'] != 0) { $conditions['to_ids'] = $filters['toIDS'] == 2 ? 0 : 1; } - if (!isset($filters['includeServerCorrelations'])) { - $conditions['includeServerCorrelations'] = 1; - } else { - $conditions['includeServerCorrelations'] = $filters['includeServerCorrelations']; - } if (!empty($filters['includeRelatedTags'])) { $this->set('includeRelatedTags', 1); $conditions['includeRelatedTags'] = 1; @@ -1170,7 +1255,10 @@ class EventsController extends AppController } $event = $results[0]; - $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both'); + $emptyEvent = empty($event['Object']) && empty($event['Attribute']); + $this->set('emptyEvent', $emptyEvent); + + $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event); $this->set('attributeTags', array_values($attributeTagsName['tags'])); $this->set('attributeClusters', array_values($attributeTagsName['clusters'])); @@ -1195,45 +1283,35 @@ class EventsController extends AppController if (isset($filters['galaxyAttachedAttributes']) && $filters['galaxyAttachedAttributes'] !== '') { $this->__applyQueryString($event, $filters['galaxyAttachedAttributes'], 'Tag.name'); } - $emptyEvent = (empty($event['Object']) && empty($event['Attribute'])); - $this->set('emptyEvent', $emptyEvent); // remove galaxies tags $this->loadModel('Taxonomy'); foreach ($event['Object'] as $k => $object) { if (isset($object['Attribute'])) { foreach ($object['Attribute'] as $k2 => $attribute) { - $this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]); + if (!empty($attribute['AttributeTag'])) { + $this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]); - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); - foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } - foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; - } - $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } - foreach ($event['Attribute'] as $k => $attribute) { - $this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]); - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); - foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + foreach ($event['Attribute'] as &$attribute) { + if (!empty($attribute['AttributeTag'])) { + $this->Event->Attribute->removeGalaxyClusterTags($attribute); + + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + $attribute['tagConflicts'] = $tagConflicts; } - foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; - } - $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } if (empty($this->passedArgs['sort'])) { $filters['sort'] = 'timestamp'; $filters['direction'] = 'desc'; } - $this->loadModel('Sighting'); - $sightingsData = $this->Sighting->eventsStatistic([$event], $this->Auth->user()); + $sightingsData = $this->Event->Sighting->eventsStatistic([$event], $user); $this->set('sightingsData', $sightingsData); $params = $this->Event->rearrangeEventForView($event, $filters, $all, $sightingsData); if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) { @@ -1242,35 +1320,30 @@ class EventsController extends AppController } $this->params->params['paging'] = array($this->modelClass => $params); $this->set('event', $event); - - $deleted = 0; - if (isset($filters['deleted'])) { - $deleted = $filters['deleted'] != 2 ? 1 : 0; - } $this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable'))); - $this->set('deleted', $deleted); + $this->set('deleted', isset($filters['deleted']) && $filters['deleted'] != 0); $this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all'); $this->set('filters', $filters); $advancedFiltering = $this->__checkIfAdvancedFiltering($filters); $this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0); $this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']); $this->response->disableCache(); - $uriArray = explode('/', $this->params->here); + + // Remove `focus` attribute from URI + $uriArray = explode('/', $this->request->here); foreach ($uriArray as $k => $v) { - if (strpos($v, ':')) { - $temp = explode(':', $v); - if ($temp[0] == 'focus') { - unset($uriArray[$k]); - } + if (strpos($v, 'focus:') === 0) { + unset($uriArray[$k]); } - $this->params->here = implode('/', $uriArray); + $this->request->here = implode('/', $uriArray); } + if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) { $this->set('sightingdbs', $this->Sightingdb->getSightingdbList($this->Auth->user())); } - $this->set('currentUri', $this->params->here); + $this->set('currentUri', $this->request->here); $this->layout = false; - $this->__eventViewCommon($this->Auth->user()); + $this->__eventViewCommon($user); $this->render('/Elements/eventattribute'); } @@ -1285,7 +1358,7 @@ class EventsController extends AppController $this->loadModel('Taxonomy'); $filterData = array( 'request' => $this->request, - 'paramArray' => $this->acceptedFilteringNamedParams, + 'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS, 'named_params' => $this->request->params['named'] ); $exception = false; @@ -1367,16 +1440,18 @@ class EventsController extends AppController $this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]); - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); - foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + if (!empty($attribute['AttributeTag'])) { + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } - foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; - } - $event['Attribute'][$k]['tagConflicts'] = $tagConflicts; } - $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both'); + $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event); $this->set('attributeTags', array_values($attributeTagsName['tags'])); $this->set('attributeClusters', array_values($attributeTagsName['clusters'])); @@ -1395,14 +1470,16 @@ class EventsController extends AppController $this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]); - $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); - foreach ($tagConflicts['global'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + if (!empty($attribute['AttributeTag'])) { + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); + foreach ($tagConflicts['global'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + foreach ($tagConflicts['local'] as $tagConflict) { + $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; + } + $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } - foreach ($tagConflicts['local'] as $tagConflict) { - $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; - } - $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts; } } } @@ -1505,7 +1582,7 @@ class EventsController extends AppController private function __eventViewCommon(array $user) { - $this->set('defaultFilteringRules', $this->defaultFilteringRules); + $this->set('defaultFilteringRules', self::DEFAULT_FILTERING_RULE); $this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings)); $orgTable = $this->Event->Orgc->find('list', array( @@ -1580,20 +1657,18 @@ class EventsController extends AppController if (isset($this->request->data['deleted'])) { $deleted = $this->request->data['deleted']; } - if (isset($deleted)) { - // workaround for old instances trying to pull events with both deleted / non deleted data - if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) { - $conditions['deleted'] = array(0, 1); - } else { - if (is_array($deleted)) { - $conditions['deleted'] = $deleted; - } else if ($deleted == 1) { // both - $conditions['deleted'] = [0, 1]; - } elseif ($deleted == 0) { // not-deleted only - $conditions['deleted'] = 0; - } else { // only deleted - $conditions['deleted'] = 1; - } + // workaround for old instances trying to pull events with both deleted / non deleted data + if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) { + $conditions['deleted'] = array(0, 1); + } else { + if (is_array($deleted)) { + $conditions['deleted'] = $deleted; + } else if ($deleted == 1) { // both + $conditions['deleted'] = [0, 1]; + } elseif ($deleted == 0) { // not-deleted only + $conditions['deleted'] = 0; + } else { // only deleted + $conditions['deleted'] = 1; } } if (isset($namedParams['toIDS']) && $namedParams['toIDS'] != 0) { @@ -1703,7 +1778,7 @@ class EventsController extends AppController return $this->__restResponse($event); } - $this->set('deleted', isset($deleted) ? ($deleted > 0 ? 1 : 0) : 0); + $this->set('deleted', $deleted > 0); $this->set('includeRelatedTags', (!empty($namedParams['includeRelatedTags'])) ? 1 : 0); $this->set('includeDecayScore', (!empty($namedParams['includeDecayScore'])) ? 1 : 0); @@ -1837,6 +1912,11 @@ class EventsController extends AppController $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId, true, $eventId)); } + /** + * @param array $event + * @param string $searchFor + * @param string|false $filterColumnsOverwrite + */ private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false) { // filtering on specific columns is specified @@ -1847,15 +1927,17 @@ class EventsController extends AppController $filterValue = array_map('trim', explode(",", $filterColumnsOverwrite)); $validFilters = array('id', 'uuid', 'value', 'comment', 'type', 'category', 'Tag.name'); foreach ($filterValue as $k => $v) { - if (!in_array($v, $validFilters)) { + if (!in_array($v, $validFilters, true)) { unset($filterValue[$k]); } } } + $searchParts = explode('|', mb_strtolower($searchFor)); + // search in all attributes foreach ($event['Attribute'] as $k => $attribute) { - if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchFor)) { + if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) { unset($event['Attribute'][$k]); } } @@ -1863,7 +1945,7 @@ class EventsController extends AppController // search in all attributes foreach ($event['ShadowAttribute'] as $k => $proposals) { - if (!$this->__valueInFieldAttribute($proposals, $filterValue, $searchFor)) { + if (!$this->__valueInFieldAttribute($proposals, $filterValue, $searchParts)) { unset($event['ShadowAttribute'][$k]); } } @@ -1871,11 +1953,11 @@ class EventsController extends AppController // search for all attributes in object foreach ($event['Object'] as $k => $object) { - if ($this->__valueInFieldAttribute($object, ['id', 'uuid', 'name', 'comment'], $searchFor)) { + if ($this->__valueInFieldAttribute($object, ['id', 'uuid', 'name', 'comment'], $searchParts)) { continue; } foreach ($object['Attribute'] as $k2 => $attribute) { - if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchFor)) { + if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) { unset($event['Object'][$k]['Attribute'][$k2]); } } @@ -1890,7 +1972,8 @@ class EventsController extends AppController } // look in the parameters if we are doing advanced filtering or not - private function __checkIfAdvancedFiltering($filters) { + private function __checkIfAdvancedFiltering($filters) + { $advancedFilteringActive = array_diff_key($filters, array('sort'=>0, 'direction'=>0, 'focus'=>0, 'overrideLimit'=>0, 'filterColumnsOverwrite'=>0, 'attributeFilter'=>0, 'extended' => 0, 'page' => 0)); if (count($advancedFilteringActive) > 0) { @@ -1913,24 +1996,13 @@ class EventsController extends AppController unset($filters['direction']); $activeRules = array(); foreach ($filters as $k => $v) { - if (isset($this->defaultFilteringRules[$k]) && $this->defaultFilteringRules[$k] != $v) { - $activeRules[$k] = 1; + if (isset(self::DEFAULT_FILTERING_RULE[$k]) && self::DEFAULT_FILTERING_RULE[$k] != $v) { + $activeRules[$k] = $v; } } return array('active' => $activeRules > 0 ? $res : false, 'activeRules' => $activeRules); } - private function __removeChildren(&$pivot, $id) - { - if ($pivot['id'] == $id) { - $pivot['children'] = array(); - } else { - foreach ($pivot['children'] as $k => $v) { - $this->__removeChildren($v, $id); - } - } - } - private function __doRemove(&$pivot, $id) { foreach ($pivot['children'] as $k => $v) { @@ -2190,24 +2262,20 @@ class EventsController extends AppController } } - public function upload_stix($stix_version = '1') + public function upload_stix($stix_version = '1', $publish = false) { if ($this->request->is('post')) { - $scriptDir = APP . 'files' . DS . 'scripts'; if ($this->_isRest()) { - $randomFileName = $this->Event->generateRandomFileName(); - $tempFile = new File($scriptDir . DS . 'tmp' . DS . $randomFileName, true, 0644); - if (!$tempFile->write($this->request->input())) { - throw new Exception("Could not write content of STIX file."); + if (isset($this->params['named']['publish'])) { + $publish = $this->params['named']['publish']; } - $tempFile->close(); + $filePath = FileAccessTool::writeToTempFile($this->request->input()); $result = $this->Event->upload_stix( $this->Auth->user(), - $scriptDir, - $randomFileName, + $filePath, $stix_version, 'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'), - false + $publish ); if (is_numeric($result)) { $event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result)); @@ -2222,14 +2290,13 @@ class EventsController extends AppController } else { $original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : ''; if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) { - $randomFileName = $this->Event->generateRandomFileName(); - if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $scriptDir . DS . 'tmp' . DS . $randomFileName)) { + $filePath = FileAccessTool::createTempFile(); + if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) { throw new Exception("Could not move uploaded STIX file."); } $result = $this->Event->upload_stix( $this->Auth->user(), - $scriptDir, - $randomFileName, + $filePath, $stix_version, $original_file, $this->data['Event']['publish'] @@ -2534,8 +2601,7 @@ class EventsController extends AppController foreach ($idList as $eid) { $event = $this->Event->find('first', array( 'conditions' => Validation::uuid($eid) ? ['Event.uuid' => $eid] : ['Event.id' => $eid], - 'fields' => array('Event.orgc_id', 'Event.id', 'Event.user_id'), - 'recursive' => -1 + 'recursive' => -1, )); if (empty($event)) { $fails[] = $eid; // event not found @@ -3127,10 +3193,7 @@ class EventsController extends AppController $iocData = FileAccessTool::readFromFile($this->data['Event']['submittedioc']['tmp_name'], $this->data['Event']['submittedioc']['size']); // write - $attachments_dir = Configure::read('MISP.attachments_dir'); - if (empty($attachments_dir)) { - $attachments_dir = $this->Event->getDefaultAttachments_dir(); - } + $attachments_dir = Configure::read('MISP.attachments_dir') ?: (APP . 'files'); $rootDir = $attachments_dir . DS . $id . DS; App::uses('Folder', 'Utility'); $dir = new Folder($rootDir . 'ioc', true); @@ -3396,16 +3459,11 @@ class EventsController extends AppController } } } else { - $conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id))); - if (!$this->_isSiteAdmin()) { - $conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id')); - $conditions['Tag.user_id'] = array('0', $this->Auth->user('id')); - } - $tag = $this->Event->EventTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions)); - if (empty($tag)) { + $tagId = $this->Event->EventTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id)); + if (empty($tagId)) { return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json')); } - $tag_id = $tag['Tag']['id']; + $tag_id = $tagId; } } } @@ -3430,7 +3488,7 @@ class EventsController extends AppController $tag = $this->Event->EventTag->Tag->find('first', array( 'conditions' => $conditions, 'recursive' => -1, - 'fields' => array('Tag.name') + 'fields' => array('Tag.name', 'Tag.local_only') )); if (!$tag) { $fails[$tag_id] = __('Tag not found.'); @@ -3461,6 +3519,10 @@ class EventsController extends AppController $fails[$tag_id] = __('Tag is not allowed due to taxonomy exclusivity settings'); continue; } + if ($tag['Tag']['local_only'] && !$local) { + $fails[$tag_id] = __('Invalid Tag. This tag can only be set as a local tag.'); + continue; + } $this->Event->EventTag->create(); if ($this->Event->EventTag->save(array('event_id' => $id, 'tag_id' => $tag_id, 'local' => $local))) { if (!$local) { @@ -5590,13 +5652,13 @@ class EventsController extends AppController $objectRef['object_id'] = $ObjectResult; $objectRef['relationship_type'] = "preceded-by"; $this->loadModel('MispObject'); - $result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false); + $result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId); $objectRef['referenced_id'] = $temp['Object']['id']; $objectRef['referenced_uuid'] = $temp['Object']['uuid']; $objectRef['object_id'] = $PreviousObjRef['Object']['id']; $objectRef['relationship_type'] = "followed-by"; $this->loadModel('MispObject'); - $result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false); + $result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId); $PreviousObjRef = $temp; } else { $PreviousObjRef = $temp; @@ -5714,13 +5776,18 @@ class EventsController extends AppController ); $job->save($data); $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'prio', - 'EventShell', - array($function, $jobId, $id), - true + + $this->Event->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + $function, + $jobId, + $id + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); $message = __('Recover event job queued. Job ID: %s', $jobId); if ($this->_isRest()) { diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index e340aad7e..f46b17145 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -488,26 +488,30 @@ class FeedsController extends AppController } } if (Configure::read('MISP.background_jobs')) { - $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'fetch_feeds', - 'job_input' => 'Feed: ' . $feedId, - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Starting fetch from Feed.'), + + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'fetch_feeds', + 'Feed: ' . $feedId, + __('Starting fetch from Feed.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('fetchFeed', $this->Auth->user('id'), $feedId, $jobId), - true + + $this->Feed->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'fetchFeed', + $this->Auth->user('id'), + $feedId, + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $message = __('Pull queued for background execution.'); } else { $result = $this->Feed->downloadFromFeedInitiator($feedId, $this->Auth->user()); @@ -554,26 +558,30 @@ class FeedsController extends AppController continue; } if (Configure::read('MISP.background_jobs')) { - $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'fetch_feed', - 'job_input' => 'Feed: ' . $feedId, - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Starting fetch from Feed.'), + + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'fetch_feed', + 'Feed: ' . $feedId, + __('Starting fetch from Feed.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('fetchFeed', $this->Auth->user('id'), $feedId, $jobId), - true + + $this->Feed->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'fetchFeed', + $this->Auth->user('id'), + $feedId, + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $message = 'Pull queued for background execution.'; } else { $result = $this->Feed->downloadFromFeedInitiator($feedId, $this->Auth->user()); @@ -747,21 +755,31 @@ class FeedsController extends AppController } else { $currentPage = 1; } - $urlparams = ''; - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); if (!in_array($feed['Feed']['source_format'], array('freetext', 'csv'))) { throw new MethodNotAllowedException(__('Invalid feed type.')); } - $HttpSocket = $syncTool->setupHttpSocketFeed($feed); - $params = array(); + App::uses('SyncTool', 'Tools'); + $syncTool = new SyncTool(); + $HttpSocket = $syncTool->setupHttpSocketFeed(); // params is passed as reference here, the pagination happens in the method, which isn't ideal but considering the performance gains here it's worth it try { - $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], $currentPage, 60, $params); + $resultArray = $this->Feed->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); } catch (Exception $e) { $this->Flash->error("Could not fetch feed: {$e->getMessage()}"); $this->redirect(array('controller' => 'feeds', 'action' => 'index')); } + + App::uses('CustomPaginationTool', 'Tools'); + $customPagination = new CustomPaginationTool(); + $params = $customPagination->createPaginationRules($resultArray, array('page' => $currentPage, 'limit' => 60), 'Feed', $sort = false); + if (!empty($currentPage) && $currentPage !== 'all') { + $start = ($currentPage - 1) * 60; + if ($start > count($resultArray)) { + return false; + } + $resultArray = array_slice($resultArray, $start, 60); + } + $this->params->params['paging'] = array($this->modelClass => $params); $resultArray = $this->Feed->getFreetextFeedCorrelations($resultArray, $feed['Feed']['id']); // remove all duplicates @@ -923,26 +941,30 @@ class FeedsController extends AppController public function cacheFeeds($scope = 'freetext') { if (Configure::read('MISP.background_jobs')) { - $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'cache_feeds', - 'job_input' => $scope, - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Starting feed caching.'), + + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'cache_feeds', + $scope, + __('Starting feed caching.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('cacheFeed', $this->Auth->user('id'), $scope, $jobId), - true + + $this->Feed->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheFeed', + $this->Auth->user('id'), + $scope, + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $message = 'Feed caching job initiated.'; } else { $result = $this->Feed->cacheFeedInitiator($this->Auth->user(), false, $scope); diff --git a/app/Controller/GalaxiesController.php b/app/Controller/GalaxiesController.php index 8877df9dd..06dcf4666 100644 --- a/app/Controller/GalaxiesController.php +++ b/app/Controller/GalaxiesController.php @@ -339,6 +339,11 @@ class GalaxiesController extends AppController $conditions[] = [ 'enabled' => true ]; + if(!$local) { + $conditions[] = [ + 'local_only' => 0 + ]; + } $galaxies = $this->Galaxy->find('all', array( 'recursive' => -1, 'fields' => array('MAX(Galaxy.version) as latest_version', 'id', 'kill_chain_order', 'name', 'icon', 'description'), @@ -528,8 +533,9 @@ class GalaxiesController extends AppController public function attachCluster($target_id, $target_type = 'event') { + $local = !empty($this->params['named']['local']); $cluster_id = $this->request->data['Galaxy']['target_id']; - $result = $this->Galaxy->attachCluster($this->Auth->user(), $target_type, $target_id, $cluster_id); + $result = $this->Galaxy->attachCluster($this->Auth->user(), $target_type, $target_id, $cluster_id, $local); return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $result, 'check_publish' => true)), 'status'=>200, 'type' => 'json')); } diff --git a/app/Controller/JobsController.php b/app/Controller/JobsController.php index 3b0c9d122..2f5550014 100644 --- a/app/Controller/JobsController.php +++ b/app/Controller/JobsController.php @@ -6,7 +6,7 @@ App::uses('AppController', 'Controller'); */ class JobsController extends AppController { - public $components = array('Security' ,'RequestHandler', 'Session'); + public $components = array('Security', 'RequestHandler', 'Session'); public $paginate = array( 'limit' => 20, @@ -30,13 +30,18 @@ class JobsController extends AppController } $jobs = $this->paginate(); foreach ($jobs as &$job) { - if ($job['Job']['process_id'] !== false) { - $job['Job']['job_status'] = $this->__jobStatusConverter(CakeResque::getJobStatus($job['Job']['process_id'])); + if (!empty($job['Job']['process_id'])) { + $job['Job']['job_status'] = $this->__getJobStatus($job['Job']['process_id']); $job['Job']['failed'] = $job['Job']['job_status'] === 'Failed'; } else { $job['Job']['job_status'] = 'Unknown'; + $job['Job']['failed'] = null; + } + if(Configure::read('SimpleBackgroundJobs.enabled')){ + $job['Job']['worker_status'] = true; + }else{ + $job['Job']['worker_status'] = isset($workers[$job['Job']['worker']]) && $workers[$job['Job']['worker']]['ok']; } - $job['Job']['worker_status'] = isset($workers[$job['Job']['worker']]) && $workers[$job['Job']['worker']]['ok']; } if ($this->_isRest()) { return $this->RestResponse->viewData($jobs, $this->response->type()); @@ -53,7 +58,7 @@ class JobsController extends AppController 'Error' => 'error' ); $this->set('fields', $fields); - $this->set('response', CakeResque::getFailedJobLog($id)); + $this->set('response', $this->__getFailedJobLog($id)); $this->render('/Jobs/ajax/error'); } @@ -84,7 +89,7 @@ class JobsController extends AppController throw new NotFoundException("Job with ID `$id` not found"); } $output = [ - 'job_status' => $this->__jobStatusConverter(CakeResque::getJobStatus($job['Job']['process_id'])), + 'job_status' => $this->__getJobStatus($job['Job']['process_id']), 'progress' => (int)$job['Job']['progress'], ]; return $this->RestResponse->viewData($output, 'json'); @@ -136,7 +141,7 @@ class JobsController extends AppController if ($this->_isSiteAdmin()) { $target = 'All events.'; } else { - $target = 'Events visible to: '.$this->Auth->user('Organisation')['name']; + $target = 'Events visible to: ' . $this->Auth->user('Organisation')['name']; } $id = $this->Job->cache($type, $this->Auth->user()); if ($this->_isRest()) { @@ -161,4 +166,35 @@ class JobsController extends AppController $this->redirect(array('action' => 'index')); } } + + private function __getJobStatus(?string $id): string + { + if (!Configure::read('SimpleBackgroundJobs.enabled')) { + return $this->__jobStatusConverter(CakeResque::getJobStatus($id)); + } + + $status = null; + if (!empty($id)) { + $job = $this->Job->getBackgroundJobsTool()->getJob($id); + $status = $job ? $job->status() : $status; + } + + return $this->__jobStatusConverter($status); + } + + private function __getFailedJobLog(string $id): array + { + if (!Configure::read('SimpleBackgroundJobs.enabled')) { + return CakeResque::getFailedJobLog($id); + } + + $job = $this->Job->getBackgroundJobsTool()->getJob($id); + $output = $job ? $job->output() : __('Job status not found.'); + $backtrace = $job ? explode("\n", $job->error()) : []; + + return [ + 'error' => $output ?? $backtrace[0] ?? '', + 'backtrace' => $backtrace + ]; + } } diff --git a/app/Controller/ObjectReferencesController.php b/app/Controller/ObjectReferencesController.php index 973c1f6fa..1d4d593ff 100644 --- a/app/Controller/ObjectReferencesController.php +++ b/app/Controller/ObjectReferencesController.php @@ -70,7 +70,7 @@ class ObjectReferencesController extends AppController $this->ObjectReference->create(); $result = $this->ObjectReference->save(array('ObjectReference' => $data)); if ($result) { - $this->ObjectReference->updateTimestamps($this->id, $data); + $this->ObjectReference->updateTimestamps($data); if ($this->_isRest()) { $object = $this->ObjectReference->find("first", array( 'recursive' => -1, @@ -203,10 +203,10 @@ class ObjectReferencesController extends AppController throw new NotFoundException(__('Invalid object reference.')); } // Check if user can view object that contains this reference - $object = $this->ObjectReference->Object->find($this->Auth->user(), [ - 'conditions' => $objectReference['ObjectReference']['object_id'], + $object = $this->ObjectReference->Object->fetchObjectSimple($this->Auth->user(), [ + 'conditions' => ['Object.id' => $objectReference['ObjectReference']['object_id']], ]); - if (!$object) { + if (empty($object)) { throw new NotFoundException(__('Invalid object reference.')); } return $this->RestResponse->viewData($objectReference, 'json'); diff --git a/app/Controller/ObjectTemplatesController.php b/app/Controller/ObjectTemplatesController.php index 8d498a6be..a2a078045 100644 --- a/app/Controller/ObjectTemplatesController.php +++ b/app/Controller/ObjectTemplatesController.php @@ -57,8 +57,7 @@ class ObjectTemplatesController extends AppController $templates_raw = $this->ObjectTemplate->find('all', array( 'recursive' => -1, 'conditions' => $conditions, - 'fields' => array('id', 'meta-category', 'name', 'description', 'org_id'), - 'contain' => array('Organisation.name'), + 'fields' => array('id', 'meta-category', 'name', 'description'), 'order' => array('ObjectTemplate.name asc') )); @@ -76,10 +75,9 @@ class ObjectTemplatesController extends AppController ); } - $fun = 'redirectAddObject'; $this->set('items', $items); $this->set('options', array( - 'functionName' => $fun, + 'functionName' => 'redirectAddObject', 'multiple' => 0, 'select_options' => array( 'additionalData' => array('event_id' => $event_id), diff --git a/app/Controller/OrganisationsController.php b/app/Controller/OrganisationsController.php index 705b2736f..78ca7dc79 100644 --- a/app/Controller/OrganisationsController.php +++ b/app/Controller/OrganisationsController.php @@ -372,19 +372,15 @@ class OrganisationsController extends AppController } $idList = json_decode($idList, true); $id_exclusion_list = array_merge($idList, array($this->Auth->user('Organisation')['id'])); - $temp = $this->Organisation->find('all', array( - 'conditions' => array( - 'local' => $local, - 'id !=' => $id_exclusion_list, - ), - 'recursive' => -1, - 'fields' => array('id', 'name'), - 'order' => array('lower(name) ASC') + $orgs = $this->Organisation->find('list', array( + 'conditions' => array( + 'local' => $local, + 'id !=' => $id_exclusion_list, + ), + 'recursive' => -1, + 'fields' => array('id', 'name'), + 'order' => array('lower(name) ASC') )); - $orgs = array(); - foreach ($temp as $org) { - $orgs[] = array('id' => $org['Organisation']['id'], 'name' => $org['Organisation']['name']); - } $this->set('local', $local); $this->layout = false; $this->autoRender = false; @@ -402,10 +398,13 @@ class OrganisationsController extends AppController $this->render('ajax/sg_org_row_empty'); } + /** + * @deprecated Probably not used anywhere. + */ public function getUUIDs() { - if (!$this->Auth->user('Role')['perm_sync']) { - throw new MethodNotAllowedException(__('This action is restricted to sync users')); + if (Configure::read('Security.hide_organisation_index_from_users')) { + throw new MethodNotAllowedException(__('This action is not enabled on this instance.')); } $temp = $this->Organisation->find('all', array( 'recursive' => -1, diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 6724e683b..d320afaa8 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -126,8 +126,16 @@ class ServersController extends AppController try { list($events, $total_count) = $this->Server->previewIndex($server, $this->Auth->user(), $combinedArgs); } catch (Exception $e) { - $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage()); - $this->redirect(array('action' => 'index')); + if ($this->_isRest()) { + return $this->RestResponse->throwException(500, $e->getMessage()); + } else { + $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage()); + $this->redirect(array('action' => 'index')); + } + } + + if ($this->_isRest()) { + return $this->RestResponse->viewData($events, $this->response->type()); } $this->loadModel('Event'); @@ -704,32 +712,36 @@ class ServersController extends AppController * incremental - only new events * - specific id of the event to pull */ - public function pull($id = null, $technique='full') + public function pull($id = null, $technique = 'full') { - if (!empty($id)) { - $this->Server->id = $id; - } else if (!empty($this->request->data['id'])) { - $this->Server->id = $this->request->data['id']; - } else { + if (empty($id)) { + if (!empty($this->request->data['id'])) { + $id = $this->request->data['id']; + } else { + throw new NotFoundException(__('Invalid server')); + } + } + + $s = $this->Server->find('first', [ + 'conditions' => ['id' => $id], + 'recursive' => -1, + ]); + if (empty($s)) { throw new NotFoundException(__('Invalid server')); } - if (!$this->Server->exists()) { - throw new NotFoundException(__('Invalid server')); - } - $s = $this->Server->read(null, $id); $error = false; if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) { throw new MethodNotAllowedException(__('You are not authorised to do that.')); } - if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) { + if (false == $s['Server']['pull'] && ($technique === 'full' || $technique === 'incremental')) { $error = __('Pull setting not enabled for this server.'); } - if (false == $this->Server->data['Server']['pull_galaxy_clusters'] && ($technique == 'pull_relevant_clusters')) { + if (false == $s['Server']['pull_galaxy_clusters'] && ($technique === 'pull_relevant_clusters')) { $error = __('Pull setting not enabled for this server.'); } if (empty($error)) { if (!Configure::read('MISP.background_jobs')) { - $result = $this->Server->pull($this->Auth->user(), $id, $technique, $s); + $result = $this->Server->pull($this->Auth->user(), $technique, $s); if (is_array($result)) { $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]); } else { @@ -741,24 +753,28 @@ class ServersController extends AppController $this->set('pulledSightings', $result[3]); } else { $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'pull', - 'job_input' => 'Server: ' . $id, - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Pulling.'), + $jobId = $this->Job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'pull', + 'Server: ' . $id, + __('Pulling.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('pull', $this->Auth->user('id'), $id, $technique, $jobId) + + $this->Server->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'pull', + $this->Auth->user('id'), + $id, + $technique, + $jobId + ], + false, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $success = __('Pull queued for background execution. Job ID: %s', $jobId); } } @@ -824,25 +840,30 @@ class ServersController extends AppController } } else { $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'push', - 'job_input' => 'Server: ' . $id, - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Pushing.'), + $jobId = $this->Job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'push', + 'Server: ' . $id, + __('Pushing.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('push', $this->Auth->user('id'), $id, $jobId) + + $this->Server->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'push', + $this->Auth->user('id'), + $id, + $technique, + $jobId + ], + false, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $message = sprintf(__('Push queued for background execution. Job ID: %s'), $jobId); + if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('Servers', 'push', $message, $this->response->type()); } @@ -946,8 +967,10 @@ class ServersController extends AppController 'Encryption' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'Proxy' => array('count' => 0, 'errors' => 0, 'severity' => 5), 'Security' => array('count' => 0, 'errors' => 0, 'severity' => 5), - 'Plugin' => array('count' => 0, 'errors' => 0, 'severity' => 5) + 'Plugin' => array('count' => 0, 'errors' => 0, 'severity' => 5), + 'SimpleBackgroundJobs' => array('count' => 0, 'errors' => 0, 'severity' => 5) ); + $writeableErrors = array(0 => __('OK'), 1 => __('not found'), 2 => __('is not writeable')); $readableErrors = array(0 => __('OK'), 1 => __('not readable')); $gpgErrors = array(0 => __('OK'), 1 => __('FAIL: settings not set'), 2 => __('FAIL: Failed to load GnuPG'), 3 => __('FAIL: Issues with the key/passphrase'), 4 => __('FAIL: sign failed')); @@ -955,6 +978,13 @@ class ServersController extends AppController $zmqErrors = array(0 => __('OK'), 1 => __('not enabled (so not tested)'), 2 => __('Python ZeroMQ library not installed correctly.'), 3 => __('ZeroMQ script not running.')); $sessionErrors = array(0 => __('OK'), 1 => __('High'), 2 => __('Alternative setting used'), 3 => __('Test failed')); $moduleErrors = array(0 => __('OK'), 1 => __('System not enabled'), 2 => __('No modules found')); + $backgroundJobsErrors = array( + 0 => __('OK'), + 1 => __('Not configured (so not tested)'), + 2 => __('Error connecting to Redis.'), + 3 => __('Error connecting to Supervisor.'), + 4 => __('Error connecting to Redis and Supervisor.') + ); $finalSettings = $this->Server->serverSettingsRead(); $issues = array( @@ -1020,20 +1050,19 @@ class ServersController extends AppController $attachmentTool = new AttachmentTool(); try { - $advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion()); + $advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus(); } catch (Exception $e) { $this->log($e->getMessage(), LOG_NOTICE); $advanced_attachments = false; } $this->set('advanced_attachments', $advanced_attachments); - // check if the current version of MISP is outdated or not - $version = $this->__checkVersion(); - $this->set('version', $version); - $gitStatus = $this->Server->getCurrentGitStatus(); + + $gitStatus = $this->Server->getCurrentGitStatus(true); $this->set('branch', $gitStatus['branch']); $this->set('commit', $gitStatus['commit']); $this->set('latestCommit', $gitStatus['latestCommit']); + $this->set('version', $gitStatus['version']); $phpSettings = array( 'max_execution_time' => array( @@ -1067,7 +1096,7 @@ class ServersController extends AppController } $this->set('phpSettings', $phpSettings); - if ($version && (!$version['upToDate'] || $version['upToDate'] == 'older')) { + if ($gitStatus['version'] && $gitStatus['version']['upToDate'] === 'older') { $diagnostic_errors++; } @@ -1085,6 +1114,9 @@ class ServersController extends AppController // if Proxy is set up in the settings, try to connect to a test URL $proxyStatus = $this->Server->proxyDiagnostics($diagnostic_errors); + // if SimpleBackgroundJobs is set up in the settings, try to connect to Redis + $backgroundJobsStatus = $this->Server->backgroundJobsDiagnostics($diagnostic_errors); + // get the DB diagnostics $dbDiagnostics = $this->Server->dbSpaceUsage(); $dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic(); @@ -1139,7 +1171,7 @@ class ServersController extends AppController unset($dumpResults[$key]['description']); } $dump = array( - 'version' => $version, + 'version' => $gitStatus['version'], 'phpSettings' => $phpSettings, 'gpgStatus' => $gpgErrors[$gpgStatus['status']], 'proxyStatus' => $proxyErrors[$proxyStatus], @@ -1154,7 +1186,8 @@ class ServersController extends AppController 'redisInfo' => $redisInfo, 'finalSettings' => $dumpResults, 'extensions' => $extensions, - 'workers' => $worker_array + 'workers' => $worker_array, + 'backgroundJobsStatus' => $backgroundJobsErrors[$backgroundJobsStatus] ); foreach ($dump['finalSettings'] as $k => $v) { if (!empty($v['redacted'])) { @@ -1178,7 +1211,6 @@ class ServersController extends AppController $this->set('phptoonew', $this->phptoonew); $this->set('pythonmin', $this->pythonmin); $this->set('pythonrec', $this->pythonrec); - $this->set('pymisp', $this->pymisp); $this->set('title_for_layout', __('Diagnostics')); } @@ -1187,10 +1219,25 @@ class ServersController extends AppController if (!$this->request->is('post')) { throw new MethodNotAllowedException(); } + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $message = __('Worker start signal sent'); + $this->Server->getBackgroundJobsTool()->startWorkerByQueue($type); + + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'startWorker', $type, $this->response->type(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + // CakeResque $validTypes = array('default', 'email', 'scheduler', 'cache', 'prio', 'update'); if (!in_array($type, $validTypes)) { throw new MethodNotAllowedException('Invalid worker type.'); } + $prepend = ''; if ($type != 'scheduler') { $workerIssueCount = 0; @@ -1222,8 +1269,21 @@ class ServersController extends AppController if (!$this->request->is('post')) { throw new MethodNotAllowedException(); } - $this->Server->killWorker($pid, $this->Auth->user()); + $message = __('Worker stop signal sent'); + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Server->getBackgroundJobsTool()->stopWorker($pid); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->type(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + // CakeResque + $this->Server->killWorker($pid, $this->Auth->user()); if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->type(), $message); } else { @@ -1243,32 +1303,6 @@ class ServersController extends AppController return $this->RestResponse->viewData($worker_array); } - private function __checkVersion() - { - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); - try { - $HttpSocket = $syncTool->setupHttpSocket(); - $response = $HttpSocket->get('https://api.github.com/repos/MISP/MISP/tags'); - $tags = $response->body; - } catch (Exception $e) { - return false; - } - if ($response->isOK() && !empty($tags)) { - $json_decoded_tags = json_decode($tags); - - // find the latest version tag in the v[major].[minor].[hotfix] format - for ($i = 0; $i < count($json_decoded_tags); $i++) { - if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $json_decoded_tags[$i]->name)) { - break; - } - } - return $this->Server->checkVersion($json_decoded_tags[$i]->name); - } else { - return false; - } - } - public function idTranslator($localId = null) { // We retrieve the list of remote servers that we can query @@ -1436,18 +1470,18 @@ class ServersController extends AppController } } $this->autoRender = false; - $this->loadModel('Log'); - if (!is_writeable(APP . 'Config/config.php')) { + if (!Configure::read('MISP.system_setting_db') && !is_writeable(APP . 'Config/config.php')) { + $this->loadModel('Log'); $this->Log->create(); $this->Log->save(array( - 'org' => $this->Auth->user('Organisation')['name'], - 'model' => 'Server', - 'model_id' => 0, - 'email' => $this->Auth->user('email'), - 'action' => 'serverSettingsEdit', - 'user_id' => $this->Auth->user('id'), - 'title' => 'Server setting issue', - 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value'] . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.', + 'org' => $this->Auth->user('Organisation')['name'], + 'model' => 'Server', + 'model_id' => 0, + 'email' => $this->Auth->user('email'), + 'action' => 'serverSettingsEdit', + 'user_id' => $this->Auth->user('id'), + 'title' => 'Server setting issue', + 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value'] . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.', )); if ($this->_isRest()) { return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'app/Config.config.php is not writeable to the apache user.', $this->response->type()); @@ -1489,7 +1523,14 @@ class ServersController extends AppController if (!$this->request->is('post')) { throw new MethodNotAllowedException(); } - $this->Server->restartWorkers($this->Auth->user()); + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Server->getBackgroundJobsTool()->restartWorkers(); + } else { + // CakeResque + $this->Server->restartWorkers($this->Auth->user()); + } + if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('Server', 'restartWorkers', false, $this->response->type(), __('Restarting workers.')); } @@ -1501,7 +1542,14 @@ class ServersController extends AppController if (!$this->request->is('post')) { throw new MethodNotAllowedException(); } - $this->Server->restartDeadWorkers($this->Auth->user()); + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Server->getBackgroundJobsTool()->restartDeadWorkers(); + } else { + // CakeResque + $this->Server->restartDeadWorkers($this->Auth->user()); + } + if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('Server', 'restartDeadWorkers', false, $this->response->type(), __('Restarting workers.')); } @@ -1708,8 +1756,8 @@ class ServersController extends AppController if (!empty($result)) { $this->set('events', $result['publishCount']); $this->set('messages', $result['messageCount']); - $this->set('time', date('Y/m/d H:i:s', $result['timestamp'])); - $this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings'])); + $this->set('time', $result['timestamp']); + $this->set('time2', $result['timestampSettings']); } $this->render('ajax/zeromqstatus'); } @@ -1727,12 +1775,19 @@ class ServersController extends AppController if (!$this->request->is('Post') || $this->request->is('ajax')) { throw new MethodNotAllowedException(); } - $worker_array = array('cache', 'default', 'email', 'prio'); - if (!in_array($worker, $worker_array)) { - throw new MethodNotAllowedException('Invalid worker'); + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->Server->getBackgroundJobsTool()->purgeQueue($worker); + } else { + // CakeResque + $worker_array = array('cache', 'default', 'email', 'prio'); + if (!in_array($worker, $worker_array)) { + throw new MethodNotAllowedException('Invalid worker'); + } + $redis = Resque::redis(); + $redis->del('queue:' . $worker); } - $redis = Resque::redis(); - $redis->del('queue:' . $worker); + $this->Flash->success('Queue cleared.'); $this->redirect($this->referer()); } @@ -1769,12 +1824,11 @@ class ServersController extends AppController public function update($branch = false) { if ($this->request->is('post')) { - $branch = false; $filterData = array( 'request' => $this->request, 'named_params' => $this->params['named'], 'paramArray' => ['branch'], - 'ordered_url_params' => @compact($paramArray), + 'ordered_url_params' => [], 'additional_delimiters' => PHP_EOL ); $exception = false; @@ -1797,8 +1851,8 @@ class ServersController extends AppController return new CakeResponse(array('body' => $update, 'type' => 'txt')); } } else { - $branch = $this->Server->getCurrentBranch(); - $this->set('branch', $branch); + $this->set('isUpdatePossible', $this->Server->isUpdatePossible()); + $this->set('branch', $this->Server->getCurrentBranch()); $this->render('ajax/update'); } } @@ -2172,26 +2226,29 @@ misp.direct_call(relative_path, body) public function cache($id = 'all') { if (Configure::read('MISP.background_jobs')) { + $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'cache_servers', - 'job_input' => intval($id) ? $id : 'all', - 'status' => 0, - 'retries' => 0, - 'org' => $this->Auth->user('Organisation')['name'], - 'message' => __('Starting server caching.'), + $jobId = $this->Job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'cache_servers', + intval($id) ? $id : 'all', + __('Starting server caching.') ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'ServerShell', - array('cacheServer', $this->Auth->user('id'), $id, $jobId), - true + + $this->Server->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_SERVER, + [ + 'cacheServer', + $this->Auth->user('id'), + $id, + $jobId + ], + false, + $jobId ); - $this->Job->saveField('process_id', $process_id); + $message = 'Server caching job initiated.'; } else { $result = $this->Server->cacheServerInitiator($this->Auth->user(), $id); @@ -2438,8 +2495,8 @@ misp.direct_call(relative_path, body) public function removeOrphanedCorrelations() { - $success = $this->Server->removeOrphanedCorrelations(); - $message = __('Orphaned correlation removed'); + $count = $this->Server->removeOrphanedCorrelations(); + $message = __('%s orphaned correlation removed', $count); if ($this->_isRest()) { return $this->RestResponse->viewData($message, $this->response->type()); } else { @@ -2553,26 +2610,28 @@ misp.direct_call(relative_path, body) $this->Flash->success('Done. For more details check the audit logs.'); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } else { - $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'upgrade_24', - 'job_input' => 'Old database', - 'status' => 0, - 'retries' => 0, - 'org_id' => 0, - 'message' => 'Job created.', + + $this->loadModel('Job'); + $jobId = $this->Job->createJob( + $this->Auth->user(), + Job::WORKER_DEFAULT, + 'upgrade_24', + 'Old database', + __('Job created.') ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('jobUpgrade24', $jobId, $this->Auth->user('id')), - true + + $this->Server->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobUpgrade24', + $jobId, + $this->Auth->user('id'), + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 4142db415..0c34cb134 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -708,7 +708,7 @@ class ShadowAttributesController extends AppController } throw new InternalErrorException(__('Could not save the proposal. Errors: %s', $message)); } else { - $this->Flash->error(__('The ShadowAttribute could not be saved. Please, try again.')); + $this->Flash->error(__('The proposed Attribute could not be saved. Please, try again.')); } } } else { @@ -1074,25 +1074,27 @@ class ShadowAttributesController extends AppController $this->Flash->success(__('All done. ' . $k . ' proposals processed.')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } else { + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generate proposal correlation', - 'job_input' => 'All attributes', - 'status' => 0, - 'retries' => 0, - 'org' => 'ADMIN', - 'message' => 'Job created.', + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate proposal correlation', + 'All attributes', + 'Job created.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('jobGenerateShadowAttributeCorrelation', $jobId) + + $this->Attribute->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateShadowAttributeCorrelation', + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + $this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).')); $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } diff --git a/app/Controller/SightingsController.php b/app/Controller/SightingsController.php index 44d699523..7387ef34c 100644 --- a/app/Controller/SightingsController.php +++ b/app/Controller/SightingsController.php @@ -9,11 +9,6 @@ class SightingsController extends AppController { public $components = array('Session', 'RequestHandler'); - public function beforeFilter() - { - parent::beforeFilter(); - } - public $paginate = array( 'limit' => 60, 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page. @@ -81,7 +76,7 @@ class SightingsController extends AppController $error_message = 'Could not add the Sighting. Reason: ' . $error; return new CakeResponse(array('body' => json_encode(array('saved' => false, 'errors' => $error_message)), 'status' => 200, 'type' => 'json')); } else { - return new CakeResponse(array('body' => json_encode(array('saved' => true, 'success' => $result . ' ' . $this->Sighting->type[$type] . (($result == 1) ? '' : 's') . ' added.')), 'status' => 200, 'type' => 'json')); + return new CakeResponse(array('body' => json_encode(array('saved' => true, 'success' => $result . ' ' . Sighting::TYPE[$type] . (($result == 1) ? '' : 's') . ' added.')), 'status' => 200, 'type' => 'json')); } } else { if ($error) { @@ -137,8 +132,7 @@ class SightingsController extends AppController throw new MethodNotAllowedException('Invalid attribute.'); } } else { - $this->loadModel('Event'); - $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true)); + $events = $this->Sighting->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true)); if (empty($events)) { throw new MethodNotAllowedException('Invalid event.'); } @@ -254,24 +248,16 @@ class SightingsController extends AppController public function index($eventid = false) { - $this->loadModel('Event'); - $sightingConditions = array(); - if ($eventid) { - $sightingConditions = array('Sighting.event_id' => $eventid); - } - $sightedEvents = $this->Sighting->find('list', array( - 'group' => ['Sighting.id', 'Sighting.event_id'], + $sightingConditions = $eventid ? array('Sighting.event_id' => $eventid) : []; + $sightedEvents = $this->Sighting->find('column', array( 'fields' => array('Sighting.event_id'), - 'conditions' => $sightingConditions + 'conditions' => $sightingConditions, + 'unique' => true, )); if (empty($sightedEvents)) { $this->RestResponse->viewData(array()); } - $conditions = array('metadata' => true, 'contain' => false); - if ($eventid) { - $conditions['eventid'] = $sightedEvents; - } - $events = $this->Event->fetchEventIds($this->Auth->user(), [ + $events = $this->Sighting->Event->fetchEventIds($this->Auth->user(), [ 'eventIdList' => $sightedEvents ]); $sightings = array(); @@ -312,10 +298,9 @@ class SightingsController extends AppController public function viewSightings($id, $context = 'attribute') { - $this->loadModel('Event'); $id = $this->Sighting->explodeIdList($id); if ($context === 'attribute') { - $objects = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1)); + $objects = $this->Sighting->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1)); if (empty($objects)) { throw new MethodNotAllowedException('Invalid object.'); } @@ -323,7 +308,7 @@ class SightingsController extends AppController } elseif ($context === 'event') { // let's set the context to event here, since we reuse the variable later on for some additional lookups. // Passing $context = 'org' could have interesting results otherwise... - $events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]); + $events = $this->Sighting->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]); $statistics = $this->Sighting->eventsStatistic($events, $this->Auth->user(), true); } else { throw new MethodNotAllowedException('Invalid context'); diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 96324e08c..49c79d2b2 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -505,6 +505,7 @@ class TagsController extends AppController $this->loadModel('Taxonomy'); $expanded = array(); $this->set('taxonomy_id', $taxonomy_id); + $local_tag = !empty($this->params['named']['local']); if ($taxonomy_id === 'collections') { $this->loadModel('TagCollection'); // This method removes banned and hidden tags @@ -528,7 +529,7 @@ class TagsController extends AppController } } else { if ($taxonomy_id === '0') { - $temp = $this->Taxonomy->getAllTaxonomyTags(true, $this->Auth->user(), true); + $temp = $this->Taxonomy->getAllTaxonomyTags(true, $this->Auth->user(), true, true, $local_tag); $tags = array(); foreach ($temp as $tag) { $tags[$tag['Tag']['id']] = $tag['Tag']; @@ -543,6 +544,9 @@ class TagsController extends AppController 'Tag.user_id' => array(0, $this->Auth->user('id')), 'Tag.hide_tag' => 0, ); + if (!$local_tag) { + $conditions['Tag.local_only'] = 0; + } $favTags = $this->Tag->FavouriteTag->find('all', array( 'conditions' => $conditions, 'recursive' => -1, @@ -562,6 +566,9 @@ class TagsController extends AppController $conditions['Tag.org_id'] = array(0, $this->Auth->user('org_id')); $conditions['Tag.user_id'] = array(0, $this->Auth->user('id')); } + if (!$local_tag) { + $conditions['Tag.local_only'] = 0; + } $allTags = $this->Tag->find('all', array( 'conditions' => $conditions, 'recursive' => -1, @@ -585,6 +592,9 @@ class TagsController extends AppController if ($tag['hide_tag']) { continue; // do not include hidden tags } + if ($tag['local_only'] && !$local_tag) { + continue; // we skip the local tags for global entries + } if (!$isSiteAdmin) { // Skip all tags that this user cannot use for tagging, determined by the org restriction on tags if ($tag['org_id'] != '0' && $tag['org_id'] != $this->Auth->user('org_id')) { @@ -706,19 +716,19 @@ class TagsController extends AppController return $this->RestResponse->viewData($results, 'json'); } + /** + * @param string $object_uuid Attribute or Event UUID + * @param string $type + * @param string $scope + * @return array + * @throws MethodNotAllowedException + */ private function __findObjectByUuid($object_uuid, &$type, $scope = 'modify') { $this->loadModel('Event'); - if (!$this->userRole['perm_tagger']) { - throw new MethodNotAllowedException(__('This functionality requires tagging permission.')); - } - $object = $this->Event->fetchEvent($this->Auth->user(), array( - 'event_uuid' => $object_uuid, - 'metadata' => 1 - )); - $type = 'Event'; + $object = $this->Event->fetchSimpleEvent($this->Auth->user(), $object_uuid); if (!empty($object)) { - $object = $object[0]; + $type = 'Event'; if ( $scope !== 'view' && !$this->_isSiteAdmin() && @@ -732,17 +742,12 @@ class TagsController extends AppController } } else { $type = 'Attribute'; - $object = $this->Event->Attribute->fetchAttributes( - $this->Auth->user(), - array( - 'conditions' => array( - 'Attribute.uuid' => $object_uuid - ), - 'flatten' => 1 - ) - ); + $object = $this->Event->Attribute->fetchAttributeSimple($this->Auth->user(), [ + 'conditions' => array( + 'Attribute.uuid' => $object_uuid + ), + ]); if (!empty($object)) { - $object = $object[0]; if ( $scope !== 'view' && !$this->_isSiteAdmin() && @@ -789,7 +794,7 @@ class TagsController extends AppController $successes = 0; $fails = array(); $existingRelations = array(); - foreach ($tags as $k => $tag) { + foreach ($tags as $tag) { if (is_numeric($tag)) { $conditions = array('Tag.id' => $tag); } else { @@ -813,13 +818,12 @@ class TagsController extends AppController $fails[] = __('Tag not found and insufficient privileges to create it.'); continue; } - $this->Tag->create(); - $result = $this->Tag->save(array('Tag' => array('name' => $tag, 'colour' => $this->Tag->random_color()))); - if (!$result) { + $createdTagId = $this->Tag->quickAdd($tag); + if (!$createdTagId) { $fails[] = __('Unable to create tag. Reason: ' . json_encode($this->Tag->validationErrors)); continue; } - $existingTag = $this->Tag->find('first', array('recursive' => -1, 'conditions' => array('Tag.id' => $this->Tag->id))); + $existingTag = $this->Tag->find('first', array('recursive' => -1, 'conditions' => array('Tag.id' => $createdTagId))); } else { $fails[] = __('Invalid Tag.'); continue; @@ -835,30 +839,30 @@ class TagsController extends AppController continue; } } + if ($existingTag['Tag']['local_only'] && !$local) { + $fails[] = __('Invalid Tag. This tag can only be set as a local tag.'); + continue; + } $this->loadModel($objectType); $connectorObject = $objectType . 'Tag'; $conditions = array( strtolower($objectType) . '_id' => $object[$objectType]['id'], 'tag_id' => $existingTag['Tag']['id'], - 'local' => ($local ? 1 : 0) ); - $existingAssociation = $this->$objectType->$connectorObject->find('first', array( - 'conditions' => $conditions - )); - if (!empty($existingAssociation)) { + $existingAssociation = $this->$objectType->$connectorObject->hasAny($conditions); + if ($existingAssociation) { $message = __('%s already has the requested tag attached, no changes had to be made for tag %s.', $objectType, $existingTag['Tag']['name']); $existingRelations[] = $existingTag['Tag']['name']; $successes++; continue; } $this->$objectType->$connectorObject->create(); - $data = array( - $connectorObject => $conditions - ); - if ($objectType == 'Attribute') { - $data[$connectorObject]['event_id'] = $object['Event']['id']; + $data = $conditions; + $data['local'] = $local ? 1 : 0; + if ($objectType === 'Attribute') { + $data['event_id'] = $object['Event']['id']; } - $result = $this->$objectType->$connectorObject->save($data); + $result = $this->$objectType->$connectorObject->save([$connectorObject => $data]); if ($result) { if ($local) { $message = 'Local tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').'; @@ -867,8 +871,7 @@ class TagsController extends AppController 'recursive' => -1, 'conditions' => array($objectType . '.id' => $object[$objectType]['id']) )); - $date = new DateTime(); - $tempObject[$objectType]['timestamp'] = $date->getTimestamp(); + $tempObject[$objectType]['timestamp'] = time(); $this->$objectType->save($tempObject); if ($objectType === 'Attribute') { $this->$objectType->Event->unpublishEvent($object['Event']['id']); diff --git a/app/Controller/UserSettingsController.php b/app/Controller/UserSettingsController.php index 3ce34a653..32788bfb6 100644 --- a/app/Controller/UserSettingsController.php +++ b/app/Controller/UserSettingsController.php @@ -114,8 +114,8 @@ class UserSettingsController extends AppController $this->paginate['conditions'] = $conditions; $data = $this->paginate(); foreach ($data as $k => $v) { - if (!empty($this->UserSetting->validSettings[$v['UserSetting']['setting']])) { - $data[$k]['UserSetting']['restricted'] = empty($this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']) ? '' : $this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']; + if (!empty(UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']])) { + $data[$k]['UserSetting']['restricted'] = empty(UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']]['restricted']) ? '' : UserSetting::VALID_SETTINGS[$v['UserSetting']['setting']]['restricted']; } else { $data[$k]['UserSetting']['restricted'] = array(); } @@ -215,7 +215,6 @@ class UserSettingsController extends AppController return $this->RestResponse->describe('UserSettings', 'setSetting', false, $this->response->type()); } else { // load the valid settings from the model - $validSettings = $this->UserSetting->validSettings; if ($this->_isSiteAdmin()) { $users = $this->UserSetting->User->find('list', array( 'recursive' => -1, @@ -235,33 +234,46 @@ class UserSettingsController extends AppController } $this->set('setting', $setting); $this->set('users', $users); - $this->set('validSettings', $validSettings); + $this->set('validSettings', UserSetting::VALID_SETTINGS); } } - public function getSetting($user_id, $setting) + public function getSetting($userId = null, $setting = null) { - if (!$this->UserSetting->checkSettingValidity($setting)) { - throw new MethodNotAllowedException(__('Invalid setting.')); + if ($this->request->is('post')) { + if (empty($this->request->data['setting'])) { + throw new BadRequestException("No setting name provided."); + } + $setting = $this->request->data['setting']; + $userId = $this->request->data['user_id'] ?? $this->Auth->user('id'); + } else { + if (empty($userId) || empty($setting)) { + throw new BadRequestException("No setting name or user ID provided."); + } } + + if (!$this->UserSetting->checkSettingValidity($setting)) { + throw new NotFoundException(__('Invalid setting.')); + } + $userSetting = $this->UserSetting->find('first', array( 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $user_id, - 'UserSetting.setting' => $setting - ), + 'conditions' => [ + 'UserSetting.user_id' => $userId, + 'UserSetting.setting' => $setting, + ], 'contain' => array('User.id', 'User.org_id') )); - $checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting, $user_id); - if (empty($checkAccess)) { - throw new MethodNotAllowedException(__('Invalid setting.')); + + if (empty($userSetting)) { + throw new NotFoundException(__('Invalid setting.')); } - if (!empty($userSetting)) { - $userSetting = json_encode($userSetting['UserSetting']['value']); - } else { - $userSetting = '[]'; + + $checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting, $userId); + if (!$checkAccess) { + throw new NotFoundException(__('Invalid setting.')); } - return $this->RestResponse->viewData($userSetting, $this->response->type(), false, true); + return $this->RestResponse->viewData($userSetting['UserSetting'], $this->response->type()); } public function delete($id = false) @@ -274,15 +286,30 @@ class UserSettingsController extends AppController */ return $this->RestResponse->describe('UserSettings', 'delete', false, $this->response->type()); } - // check if the ID is valid and whether a user setting with the given ID exists - if (empty($id) || !is_numeric($id)) { - throw new InvalidArgumentException(__('Invalid ID passed.')); + + if (!$this->request->is('post') && !$this->request->is('delete')) { + throw new MethodNotAllowedException(__('Expecting POST or DELETE request.')); } + + if (empty($id)) { + if (empty($this->request->data['setting'])) { + throw new BadRequestException("No setting name to delete provided."); + } + $conditions = ['UserSetting.setting' => $this->request->data['setting']]; + if (!empty($this->request->data['user_id'])) { + $conditions['UserSetting.user_id'] = $this->request->data['user_id']; + } else { + $conditions['UserSetting.user_id'] = $this->Auth->user('id'); // current user + } + } else if (is_numeric($id)) { + $conditions = ['UserSetting.id' => $id]; + } else { + throw new BadRequestException(__('Invalid ID passed.')); + } + $userSetting = $this->UserSetting->find('first', array( 'recursive' => -1, - 'conditions' => array( - 'UserSetting.id' => $id - ), + 'conditions' => $conditions, 'contain' => array('User.id', 'User.org_id') )); if (empty($userSetting)) { @@ -296,34 +323,30 @@ class UserSettingsController extends AppController if ($settingPermCheck !== true) { throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck)); } - if ($this->request->is('post') || $this->request->is('delete')) { - // Delete the setting that we were after. - $result = $this->UserSetting->delete($userSetting['UserSetting']['id']); - if ($result) { - // set the response for both the UI and API - $message = __('Setting deleted.'); - if ($this->_isRest()) { - return $this->RestResponse->saveSuccessResponse('UserSettings', 'delete', $id, $this->response->type(), $message); - } else { - $this->Flash->success($message); - } + // Delete the setting that we were after. + $result = $this->UserSetting->delete($userSetting['UserSetting']['id']); + if ($result) { + // set the response for both the UI and API + $message = __('Setting deleted.'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('UserSettings', 'delete', $userSetting['UserSetting']['id'], $this->response->type(), $message); } else { - // set the response for both the UI and API - $message = __('Setting could not be deleted.'); - if ($this->_isRest()) { - return $this->RestResponse->saveFailResponse('UserSettings', 'delete', $id, $message, $this->response->type()); - } else { - $this->Flash->error($message); - } + $this->Flash->success($message); } - /* - * The API responses stopped executing this function and returned a serialised response to the user. - * For UI users, redirect to where they issued the request from. - */ - $this->redirect($this->referer()); } else { - throw new MethodNotAllowedException(__('Expecting POST or DELETE request.')); + // set the response for both the UI and API + $message = __('Setting could not be deleted.'); + if ($this->_isRest()) { + return $this->RestResponse->saveFailResponse('UserSettings', 'delete', $userSetting['UserSetting']['id'], $message, $this->response->type()); + } else { + $this->Flash->error($message); + } } + /* + * The API responses stopped executing this function and returned a serialised response to the user. + * For UI users, redirect to where they issued the request from. + */ + $this->redirect($this->referer()); } public function setHomePage() diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index bae80efe8..8eb081912 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -741,7 +741,7 @@ class UsersController extends AppController $notification_message .= ' ' . __('User notification of new credentials could not be send.'); } } - if (!empty(Configure::read('Security.advanced_authkeys'))) { + if (!empty(Configure::read('Security.advanced_authkeys')) && $this->_isRest()) { $this->loadModel('AuthKey'); $newKey = $this->AuthKey->createnewkey($this->User->id); } @@ -1210,8 +1210,7 @@ class UsersController extends AppController } } // populate the DB with the first role (site admin) if it's empty - $this->loadModel('Role'); - if ($this->Role->find('count') == 0) { + if (!$this->User->Role->hasAny()) { $siteAdmin = array('Role' => array( 'id' => 1, 'name' => 'Site Admin', @@ -1230,14 +1229,14 @@ class UsersController extends AppController 'perm_template' => 1, 'perm_tagger' => 1, )); - $this->Role->save($siteAdmin); + $this->User->Role->save($siteAdmin); // PostgreSQL: update value of auto incremented serial primary key after setting the column by force - if ($dataSource == 'Database/Postgres') { + if ($dataSource === 'Database/Postgres') { $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));"; - $this->Role->query($sql); + $this->User->Role->query($sql); } } - if ($this->User->Organisation->find('count', array('conditions' => array('Organisation.local' => true))) == 0) { + if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) { $this->User->runUpdates(); $date = date('Y-m-d H:i:s'); $org = array('Organisation' => array( @@ -1253,23 +1252,25 @@ class UsersController extends AppController )); $this->User->Organisation->save($org); // PostgreSQL: update value of auto incremented serial primary key after setting the column by force - if ($dataSource == 'Database/Postgres') { + if ($dataSource === 'Database/Postgres') { $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));"; $this->User->Organisation->query($sql); } $org_id = $this->User->Organisation->id; - } else { - $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1)); - if (!empty($hostOrg)) { - $org_id = $hostOrg['Organisation']['id']; - } else { - $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC')); - $org_id = $firstOrg['Organisation']['id']; - } } // populate the DB with the first user if it's empty - if ($this->User->find('count') == 0) { + if (!$this->User->hasAny()) { + if (!isset($org_id)) { + $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1)); + if (!empty($hostOrg)) { + $org_id = $hostOrg['Organisation']['id']; + } else { + $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC')); + $org_id = $firstOrg['Organisation']['id']; + } + } + $this->User->runUpdates(); $this->User->createInitialUser($org_id); } diff --git a/app/Lib/Dashboard/MispStatusWidget.php b/app/Lib/Dashboard/MispStatusWidget.php index b40eaed23..869debfce 100644 --- a/app/Lib/Dashboard/MispStatusWidget.php +++ b/app/Lib/Dashboard/MispStatusWidget.php @@ -44,7 +44,7 @@ class MispStatusWidget 'View' ) ); - $notifications = $this->Event->populateNotifications($user); + $notifications = $this->Event->User->populateNotifications($user); if (!empty($notifications['proposalCount'])) { $data[] = array( 'title' => __('Pending proposals'), diff --git a/app/Lib/Export/CsvExport.php b/app/Lib/Export/CsvExport.php index 37dfcc0da..cf388a68b 100644 --- a/app/Lib/Export/CsvExport.php +++ b/app/Lib/Export/CsvExport.php @@ -64,7 +64,7 @@ class CsvExport $attribute['decay_score_score'] = 0; $attribute['decay_score_decayed'] = false; } - return $this->__addLine($attribute, $options); + return $this->__addLine($attribute); } private function __sightingsHandler($sighting, $options) @@ -89,7 +89,7 @@ class CsvExport $sighting['Sighting'][$new_key] = $attribute_val; } } - $lines .= $this->__addLine($sighting['Sighting'], $options); + $lines .= $this->__addLine($sighting['Sighting']); return $lines; } @@ -97,20 +97,20 @@ class CsvExport { $lines = ''; if (!empty($event['Attribute'])) { - foreach ($event['Attribute'] as $k => $attribute) { + foreach ($event['Attribute'] as $attribute) { $attribute = $this->__addMetadataToAttribute($event, $attribute); - $lines .= $this->__addLine($attribute, $options); + $lines .= $this->__addLine($attribute); } } if (!empty($event['Object'])) { - foreach ($event['Object'] as $k => $object) { + foreach ($event['Object'] as $object) { if (!empty($object['Attribute'])) { foreach ($object['Attribute'] as $attribute) { $attribute = $this->__addMetadataToAttribute($event, $attribute); $attribute['object_uuid'] = $object['uuid']; $attribute['object_name'] = $object['name']; $attribute['object_meta-category'] = $object['meta-category']; - $lines .= $this->__addLine($attribute, $options); + $lines .= $this->__addLine($attribute); } } } @@ -118,22 +118,27 @@ class CsvExport return $lines; } - private function __addLine($attribute, $options = array()) { - $line = ''; + /** + * @param array $attribute + * @return string + */ + private function __addLine($attribute) + { + $parts = []; foreach ($this->requested_fields as $req_att) { - if (empty($line)) { - $line = $this->__escapeCSVField($attribute[$req_att]); - } else { - $line .= ',' . $this->__escapeCSVField($attribute[$req_att]); - } + if (isset($attribute[$req_att])) { + $parts[] = $this->__escapeCSVField($attribute[$req_att]); + } else { + $parts[] = '""'; // keep it consistent with old CSV format + } } - return $line . PHP_EOL; + return implode(',', $parts) . PHP_EOL; } - private function __escapeCSVField(&$field) + private function __escapeCSVField($field) { if (is_bool($field)) { - return ($field ? '1' : '0'); + return $field ? '1' : '0'; } if (is_numeric($field)) { return $field; @@ -257,15 +262,18 @@ class CsvExport return ''; } - public function eventIndex($events) + /** + * @param array $events + * @return Generator[string] + */ + public function eventIndex(array $events) { $fields = array( 'id', 'date', 'info', 'tags', 'uuid', 'published', 'analysis', 'attribute_count', 'orgc_id', 'orgc_name', 'orgc_uuid', 'timestamp', 'distribution', 'sharing_group_id', 'threat_level_id', 'publish_timestamp', 'extends_uuid' ); - $result = implode(',', $fields) . PHP_EOL; - foreach ($events as $key => $event) { - $event['tags'] = ''; + yield implode(',', $fields) . PHP_EOL; + foreach ($events as $event) { if (!empty($event['EventTag'])) { $tags = array(); foreach ($event['EventTag'] as $et) { @@ -275,16 +283,18 @@ class CsvExport } else { $tags = ''; } - $event['Event']['tags'] = $tags; - $event['Event']['orgc_name'] = $event['Orgc']['name']; - $event['Event']['orgc_uuid'] = $event['Orgc']['uuid']; + $event['tags'] = $tags; + $event['orgc_name'] = $event['Orgc']['name']; + $event['orgc_uuid'] = $event['Orgc']['uuid']; $current = array(); foreach ($fields as $field) { - $current[] = $this->__escapeCSVField($event['Event'][$field]); + if (isset($event[$field])) { + $current[] = $this->__escapeCSVField($event[$field]); + } else { + $current[] = ''; + } } - $result .= implode(', ', $current) . PHP_EOL; + yield implode(',', $current) . PHP_EOL; } - return $result; } - } diff --git a/app/Lib/Export/OpendataExport.php b/app/Lib/Export/OpendataExport.php index b71b5067a..dd66b205f 100644 --- a/app/Lib/Export/OpendataExport.php +++ b/app/Lib/Export/OpendataExport.php @@ -3,7 +3,6 @@ class OpendataExport { public $non_restrictive_export = true; - public $use_default_filters = true; public $mock_query_only = true; private $__default_filters = null; diff --git a/app/Lib/Export/Stix1Export.php b/app/Lib/Export/Stix1Export.php index ea340cc20..8262f5fdf 100644 --- a/app/Lib/Export/Stix1Export.php +++ b/app/Lib/Export/Stix1Export.php @@ -1,5 +1,4 @@ __baseurl = escapeshellarg(Configure::read('MISP.baseurl')); - $this->__org = escapeshellarg(Configure::read('MISP.org')); - $my_server = ClassRegistry::init('Server'); - return $my_server->getPythonVersion() . ' ' . $this->__framing_script . ' stix1 -v ' . $this->__version . ' -n ' . $this->__baseurl . ' -o ' . $this->__org . ' -f ' . $this->__return_format . ' ' . $this->__end_of_cmd; + return [ + ProcessTool::pythonBin(), + $this->__framing_script, + 'stix1', + '-v', $this->__version, + '-n', Configure::read('MISP.baseurl'), + '-o', Configure::read('MISP.org'), + '-f', $this->__return_format, + ]; } - protected function __parse_misp_events($filenames) + protected function __parse_misp_events(array $filenames) { - $scriptFile = $this->__scripts_dir . $this->__script_name; - $my_server = ClassRegistry::init('Server'); - return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -f ' . $this->__return_format . ' -o ' . $this->__org . ' -i ' . $this->__tmp_dir . $filenames . $this->__end_of_cmd); + $command = [ + ProcessTool::pythonBin(), + $this->__scripts_dir . 'misp2stix.py', + '-v', $this->__version, + '-f', $this->__return_format, + '-o', Configure::read('MISP.org'), + '-i', + ]; + $command = array_merge($command, $filenames); + + return ProcessTool::execute($command, null, true); } } diff --git a/app/Lib/Export/Stix2Export.php b/app/Lib/Export/Stix2Export.php index 9f467e5a8..b9514adde 100644 --- a/app/Lib/Export/Stix2Export.php +++ b/app/Lib/Export/Stix2Export.php @@ -1,5 +1,4 @@ getPythonVersion() . ' ' . $this->__framing_script . ' stix2 -v ' . $this->__version . ' --uuid ' . escapeshellarg(CakeText::uuid()) . $this->__end_of_cmd; + return [ + ProcessTool::pythonBin(), + $this->__framing_script, + 'stix2', + '-v', $this->__version, + '--uuid', CakeText::uuid(), + ]; } - protected function __parse_misp_events($filenames) + protected function __parse_misp_events(array $filenames) { - $scriptFile = $this->__scripts_dir . $this->__script_name; - $filenames = implode(' ' . $this->__tmp_dir, $this->__filenames); - $my_server = ClassRegistry::init('Server'); - $result = shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -i ' . $this->__tmp_dir . $filenames . $this->__end_of_cmd); + $scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py'; + $command = [ + ProcessTool::pythonBin(), + $scriptFile, + '-v', $this->__version, + '-i', + ]; + $command = array_merge($command, $filenames); + $result = ProcessTool::execute($command, null, true); $result = preg_split("/\r\n|\n|\r/", trim($result)); return end($result); } diff --git a/app/Lib/Export/StixExport.php b/app/Lib/Export/StixExport.php index a80bd3d07..6b64a4c23 100644 --- a/app/Lib/Export/StixExport.php +++ b/app/Lib/Export/StixExport.php @@ -1,6 +1,10 @@ 1, @@ -8,60 +12,54 @@ class StixExport ); protected $__return_format = 'json'; protected $__scripts_dir = APP . 'files/scripts/'; - protected $__tmp_dir = APP . 'files/scripts/tmp/'; protected $__framing_script = APP . 'files/scripts/misp_framing.py'; - protected $__end_of_cmd = ' 2>' . APP . 'tmp/logs/exec-errors.log'; protected $__return_type = null; + + /** @var array Full paths to files to convert */ protected $__filenames = array(); - protected $__default_filters = null; protected $__version = null; private $__current_filename = null; private $__empty_file = null; - private $__framing = null; - private $__stix_file = null; + /** @var File */ private $__tmp_file = null; private $__n_attributes = 0; public $non_restrictive_export = true; - public $use_default_filters = true; + + private $Server; public function setDefaultFilters($filters) { - $sane_version = (!empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions)); + $sane_version = !empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions, true); $this->__version = $sane_version ? $filters['stix-version'] : $this->__default_version; } public function handler($data, $options = array()) { - $attributes_count = count($data['Attribute']); - foreach ($data['Object'] as $_object) { - if (isset($_object['Attribute'])) { - $attributes_count += count($_object['Attribute']); + $attributesCount = count($data['Attribute']); + foreach ($data['Object'] as $object) { + if (isset($object['Attribute'])) { + $attributesCount += count($object['Attribute']); } } - App::uses('JSONConverterTool', 'Tools'); + $converter = new JSONConverterTool(); - $event = $converter->convert($data); - if ($this->__n_attributes + $attributes_count < $this->__attributes_limit) { - $this->__tmp_file->append($this->__n_attributes == 0 ? $event : ',' . $event); - $this->__n_attributes += $attributes_count; + $event = JsonTool::encode($converter->convert($data, false, true)); // we don't need pretty printed JSON + if ($this->__n_attributes + $attributesCount < $this->__attributes_limit) { + $this->__tmp_file->append($this->__n_attributes === 0 ? $event : ',' . $event); + $this->__n_attributes += $attributesCount; $this->__empty_file = false; + } elseif ($attributesCount > $this->__attributes_limit) { + $filePath = FileAccessTool::writeToTempFile($event); + $this->__filenames[] = $filePath; } else { - if ($attributes_count > $this->__attributes_limit) { - $randomFileName = $this->__generateRandomFileName(); - $tmpFile = new File($this->__tmp_dir . $randomFileName, true, 0644); - $tmpFile->write($event); - $tmpFile->close(); - array_push($this->__filenames, $randomFileName); - } else { - $this->__tmp_file->append(']}'); - $this->__tmp_file->close(); - array_push($this->__filenames, $this->__current_filename); - $this->__initialize_misp_file(); - $this->__tmp_file->append($event); - $this->__n_attributes = $attributes_count; - } + $this->__tmp_file->append(']}'); + $this->__tmp_file->close(); + $this->__filenames[] = $this->__current_filename; + $this->__initialize_misp_file(); + $this->__tmp_file->append($event); + $this->__n_attributes = $attributesCount; } return ''; } @@ -69,21 +67,19 @@ class StixExport public function header($options = array()) { $this->__return_type = $options['returnFormat']; - if ($this->__return_type == 'stix-json') { + if ($this->__return_type === 'stix-json') { $this->__return_type = 'stix'; - } else if ($this->__return_type == 'stix') { + } else if ($this->__return_type === 'stix') { $this->__return_format = 'xml'; } - $framing_cmd = $this->__initiate_framing_params(); - $randomFileName = $this->__generateRandomFileName(); - $this->__framing = json_decode(shell_exec($framing_cmd), true); - $this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.' . $this->__return_type); - unset($randomFileName); - $this->__stix_file->write($this->__framing['header']); $this->__initialize_misp_file(); return ''; } + /** + * @return TmpFileTool + * @throws Exception + */ public function footer() { if ($this->__empty_file) { @@ -92,31 +88,27 @@ class StixExport } else { $this->__tmp_file->append(']}'); $this->__tmp_file->close(); - array_push($this->__filenames, $this->__current_filename); + $this->__filenames[] = $this->__current_filename; } - $filenames = implode(' ' . $this->__tmp_dir, $this->__filenames); - $result = $this->__parse_misp_events($filenames); + $result = $this->__parse_misp_events($this->__filenames); + $this->__delete_temporary_files(); $decoded = json_decode($result, true); if (!isset($decoded['success']) || !$decoded['success']) { - $this->__delete_temporary_files(); $error = $decoded && !empty($decoded['error']) ? $decoded['error'] : $result; - return 'Error while processing your query: ' . $error; + throw new Exception('Error while processing your query during STIX export: ' . $error); } - foreach ($this->__filenames as $f => $filename) { - $file = new File($this->__tmp_dir . $filename . '.out'); - $stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1); - $file->close(); - $file->delete(); - @unlink($this->__tmp_dir . $filename); - $this->__stix_file->append($stix_event . $this->__framing['separator']); - unset($stix_event); + + $framing = $this->getFraming(); + + $stixFile = new TmpFileTool(); + $stixFile->write($framing['header']); + foreach ($this->__filenames as $filename) { + $stixEvent = FileAccessTool::readAndDelete($filename . '.out'); + $stixEvent = $this->__return_type === 'stix' ? $stixEvent : substr($stixEvent, 1, -1); + $stixFile->writeWithSeparator($stixEvent, $framing['separator']); } - $stix_event = $this->__stix_file->read(); - $this->__stix_file->close(); - $this->__stix_file->delete(); - $sep_len = strlen($this->__framing['separator']); - $stix_event = (empty($this->__filenames) ? $stix_event : substr($stix_event, 0, -$sep_len)) . $this->__framing['footer']; - return $stix_event; + $stixFile->write($framing['footer']); + return $stixFile; } public function separator() @@ -126,23 +118,41 @@ class StixExport private function __initialize_misp_file() { - $this->__current_filename = $this->__generateRandomFileName(); - $this->__tmp_file = new File($this->__tmp_dir . $this->__current_filename, true, 0644); + $this->__current_filename = FileAccessTool::createTempFile(); + $this->__tmp_file = new File($this->__current_filename); $this->__tmp_file->write('{"response": ['); $this->__empty_file = true; } - private function __generateRandomFileName() - { - return (new RandomTool())->random_str(false, 12); - } - private function __delete_temporary_files() { - foreach ($this->__filenames as $f => $filename) { - @unlink($this->__tmp_dir . $filename); + foreach ($this->__filenames as $filename) { + FileAccessTool::deleteFileIfExists($filename); } - $this->__stix_file->close(); - $this->__stix_file->delete(); } + + /** + * @return array + * @throws Exception + */ + private function getFraming() + { + $framingCmd = $this->__initiate_framing_params(); + $framing = json_decode(ProcessTool::execute($framingCmd, null, true), true); + if ($framing === null || isset($framing['error'])) { + throw new Exception("Could not get results from framing cmd when exporting STIX file."); + } + return $framing; + } + + /** + * @param array $filenames Paths to files to process + * @return string|false|null + */ + abstract protected function __parse_misp_events(array $filenames); + + /** + * @return array + */ + abstract protected function __initiate_framing_params(); } diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index a9126c355..5f6891023 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -1,5 +1,7 @@ execute($exec); + ProcessTool::execute($exec); return FileAccessTool::readFromFile($zipFile); } catch (Exception $e) { @@ -360,13 +362,13 @@ class AttachmentTool * @return array * @throws Exception */ - public function advancedExtraction($pythonBin, $filePath) + public function advancedExtraction($filePath) { return $this->executeAndParseJsonOutput([ - $pythonBin, + ProcessTool::pythonBin(), self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-p', - escapeshellarg($filePath), + $filePath, ]); } @@ -375,9 +377,9 @@ class AttachmentTool * @return array * @throws Exception */ - public function checkAdvancedExtractionStatus($pythonBin) + public function checkAdvancedExtractionStatus() { - return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']); + return $this->executeAndParseJsonOutput([ProcessTool::pythonBin(), self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']); } /** @@ -496,50 +498,11 @@ class AttachmentTool */ private function executeAndParseJsonOutput(array $command) { - $output = $this->execute($command); - - $json = json_decode($output, true); - if ($json === null) { - throw new Exception("Command output is not valid JSON: " . json_last_error_msg()); + $output = ProcessTool::execute($command); + try { + return JsonTool::decode($output); + } catch (Exception $e) { + throw new Exception("Command output is not valid JSON.", 0, $e); } - return $json; - } - - /** - * This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions - * can be much more specific. - * - * @param array $command - * @return string - * @throws Exception - */ - private function execute(array $command) - { - $descriptorspec = [ - 1 => ["pipe", "w"], // stdout - 2 => ["pipe", "w"], // stderr - ]; - - $command = implode(' ', $command); - $process = proc_open($command, $descriptorspec, $pipes); - if (!$process) { - throw new Exception("Command '$command' could be started."); - } - - $stdout = stream_get_contents($pipes[1]); - if ($stdout === false) { - throw new Exception("Could not get STDOUT of command."); - } - fclose($pipes[1]); - - $stderr = stream_get_contents($pipes[2]); - fclose($pipes[2]); - - $returnCode = proc_close($process); - if ($returnCode !== 0) { - throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'"); - } - - return $stdout; } } diff --git a/app/Lib/Tools/AttributeValidationTool.php b/app/Lib/Tools/AttributeValidationTool.php new file mode 100644 index 000000000..15db1b758 --- /dev/null +++ b/app/Lib/Tools/AttributeValidationTool.php @@ -0,0 +1,698 @@ + 64, + 'md5' => 32, + 'imphash' => 32, + 'telfhash' => 70, + 'sha1' => 40, + 'git-commit-id' => 40, + 'x509-fingerprint-md5' => 32, + 'x509-fingerprint-sha1' => 40, + 'x509-fingerprint-sha256' => 64, + 'ja3-fingerprint-md5' => 32, + 'jarm-fingerprint' => 62, + 'hassh-md5' => 32, + 'hasshserver-md5' => 32, + 'pehash' => 40, + 'sha224' => 56, + 'sha256' => 64, + 'sha384' => 96, + 'sha512' => 128, + 'sha512/224' => 56, + 'sha512/256' => 64, + 'sha3-224' => 56, + 'sha3-256' => 64, + 'sha3-384' => 96, + 'sha3-512' => 128, + ]; + + /** + * Do some last second modifications before the validation + * @param string $type + * @param mixed $value + * @return string + */ + public static function modifyBeforeValidation($type, $value) + { + $value = self::handle4ByteUnicode($value); + switch ($type) { + case 'ip-src': + case 'ip-dst': + return self::compressIpv6($value); + case 'md5': + case 'sha1': + case 'sha224': + case 'sha256': + case 'sha384': + case 'sha512': + case 'sha512/224': + case 'sha512/256': + case 'sha3-224': + case 'sha3-256': + case 'sha3-384': + case 'sha3-512': + case 'ja3-fingerprint-md5': + case 'jarm-fingerprint': + case 'hassh-md5': + case 'hasshserver-md5': + case 'hostname': + case 'pehash': + case 'authentihash': + case 'vhash': + case 'imphash': + case 'telfhash': + case 'tlsh': + case 'anonymised': + case 'cdhash': + case 'email': + case 'email-src': + case 'email-dst': + case 'target-email': + case 'whois-registrant-email': + return strtolower($value); + case 'domain': + $value = strtolower($value); + $value = trim($value, '.'); + // Domain is not valid, try to convert to punycode + if (!self::isDomainValid($value) && function_exists('idn_to_ascii')) { + $punyCode = idn_to_ascii($value); + if ($punyCode !== false) { + $value = $punyCode; + } + } + return $value; + case 'domain|ip': + $value = strtolower($value); + $parts = explode('|', $value); + if (!isset($parts[1])) { + return $value; // not a composite + } + $parts[0] = trim($parts[0], '.'); + // Domain is not valid, try to convert to punycode + if (!self::isDomainValid($parts[0]) && function_exists('idn_to_ascii')) { + $punyCode = idn_to_ascii($parts[0]); + if ($punyCode !== false) { + $parts[0] = $punyCode; + } + } + $parts[1] = self::compressIpv6($parts[1]); + return "$parts[0]|$parts[1]"; + case 'filename|md5': + case 'filename|sha1': + case 'filename|imphash': + case 'filename|sha224': + case 'filename|sha256': + case 'filename|sha384': + case 'filename|sha512': + case 'filename|sha512/224': + case 'filename|sha512/256': + case 'filename|sha3-224': + case 'filename|sha3-256': + case 'filename|sha3-384': + case 'filename|sha3-512': + case 'filename|authentihash': + case 'filename|vhash': + case 'filename|pehash': + case 'filename|tlsh': + // Convert hash to lowercase + $pos = strpos($value, '|'); + return substr($value, 0, $pos) . strtolower(substr($value, $pos)); + case 'http-method': + case 'hex': + return strtoupper($value); + case 'vulnerability': + case 'weakness': + $value = str_replace('–', '-', $value); + return strtoupper($value); + case 'cc-number': + case 'bin': + return preg_replace('/[^0-9]+/', '', $value); + case 'iban': + case 'bic': + $value = strtoupper($value); + return preg_replace('/[^0-9A-Z]+/', '', $value); + case 'prtn': + case 'whois-registrant-phone': + case 'phone-number': + if (substr($value, 0, 2) == '00') { + $value = '+' . substr($value, 2); + } + $value = preg_replace('/\(0\)/', '', $value); + return preg_replace('/[^\+0-9]+/', '', $value); + case 'x509-fingerprint-md5': + case 'x509-fingerprint-sha256': + case 'x509-fingerprint-sha1': + $value = str_replace(':', '', $value); + return strtolower($value); + case 'ip-dst|port': + case 'ip-src|port': + if (substr_count($value, ':') >= 2) { // (ipv6|port) - tokenize ip and port + if (strpos($value, '|')) { // 2001:db8::1|80 + $parts = explode('|', $value); + } elseif (strpos($value, '[') === 0 && strpos($value, ']') !== false) { // [2001:db8::1]:80 + $ipv6 = substr($value, 1, strpos($value, ']')-1); + $port = explode(':', substr($value, strpos($value, ']')))[1]; + $parts = array($ipv6, $port); + } elseif (strpos($value, '.')) { // 2001:db8::1.80 + $parts = explode('.', $value); + } elseif (strpos($value, ' port ')) { // 2001:db8::1 port 80 + $parts = explode(' port ', $value); + } elseif (strpos($value, 'p')) { // 2001:db8::1p80 + $parts = explode('p', $value); + } elseif (strpos($value, '#')) { // 2001:db8::1#80 + $parts = explode('#', $value); + } else { // 2001:db8::1:80 this one is ambiguous + $temp = explode(':', $value); + $parts = array(implode(':', array_slice($temp, 0, count($temp)-1)), end($temp)); + } + } elseif (strpos($value, ':')) { // (ipv4:port) + $parts = explode(':', $value); + } elseif (strpos($value, '|')) { // (ipv4|port) + $parts = explode('|', $value); + } else { + return $value; + } + return self::compressIpv6($parts[0]) . '|' . $parts[1]; + case 'mac-address': + case 'mac-eui-64': + $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value)); + return wordwrap($value, 2, ':', true); + case 'hostname|port': + $value = strtolower($value); + return str_replace(':', '|', $value); + case 'boolean': + $value = trim(strtolower($value)); + if ('true' === $value) { + $value = 1; + } else if ('false' === $value) { + $value = 0; + } + return $value ? '1' : '0'; + case 'datetime': + try { + return (new DateTime($value, new DateTimeZone('GMT')))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formatting with microseconds + } catch (Exception $e) { + return $value; // silently skip. Rejection will be done in validation() + } + case 'AS': + if (strtoupper(substr($value, 0, 2)) === 'AS') { + $value = substr($value, 2); // remove 'AS' + } + if (strpos($value, '.') !== false) { // maybe value is in asdot notation + $parts = explode('.', $value); + if (self::isPositiveInteger($parts[0]) && self::isPositiveInteger($parts[1])) { + return $parts[0] * 65536 + $parts[1]; + } + } + return $value; + } + return $value; + } + + /** + * Validate if value is valid for given attribute type. + * At this point, we can be sure, that composite type is really composite. + * @param string $type + * @param string $value + * @return bool|string + */ + public static function validate($type, $value) + { + switch ($type) { + case 'md5': + case 'imphash': + case 'telfhash': + case 'sha1': + case 'sha224': + case 'sha256': + case 'sha384': + case 'sha512': + case 'sha512/224': + case 'sha512/256': + case 'sha3-224': + case 'sha3-256': + case 'sha3-384': + case 'sha3-512': + case 'authentihash': + case 'ja3-fingerprint-md5': + case 'jarm-fingerprint': + case 'hassh-md5': + case 'hasshserver-md5': + case 'x509-fingerprint-md5': + case 'x509-fingerprint-sha256': + case 'x509-fingerprint-sha1': + case 'git-commit-id': + if (self::isHashValid($type, $value)) { + return true; + } + $length = self::HASH_HEX_LENGTH[$type]; + return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length); + case 'tlsh': + if (self::isTlshValid($value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".'); + case 'pehash': + if (self::isHashValid('pehash', $value)) { + return true; + } + return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); + case 'ssdeep': + if (self::isSsdeep($value)) { + return true; + } + return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash'); + case 'impfuzzy': + if (substr_count($value, ':') === 2) { + $parts = explode(':', $value); + if (self::isPositiveInteger($parts[0])) { + return true; + } + } + return __('Invalid impfuzzy format. The format has to be imports:hash:hash'); + case 'cdhash': + if (preg_match("#^[0-9a-f]{40,}$#", $value)) { + return true; + } + return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)'); + case 'http-method': + if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) { + return true; + } + return __('Unknown HTTP method.'); + case 'filename|pehash': + // no newline + if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) { + return true; + } + return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); + case 'filename|md5': + case 'filename|sha1': + case 'filename|imphash': + case 'filename|sha224': + case 'filename|sha256': + case 'filename|sha384': + case 'filename|sha512': + case 'filename|sha512/224': + case 'filename|sha512/256': + case 'filename|sha3-224': + case 'filename|sha3-256': + case 'filename|sha3-384': + case 'filename|sha3-512': + case 'filename|authentihash': + $hashType = substr($type, 9); // strip `filename|` + $length = self::HASH_HEX_LENGTH[$hashType]; + if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length); + case 'filename|ssdeep': + $composite = explode('|', $value); + if (strpos($composite[0], "\n") !== false) { + return __('Filename must not contain new line character.'); + } + if (self::isSsdeep($composite[1])) { + return true; + } + return __('Invalid ssdeep hash (expected: blocksize:hash:hash).'); + case 'filename|tlsh': + $composite = explode('|', $value); + if (strpos($composite[0], "\n") !== false) { + return __('Filename must not contain new line character.'); + } + if (self::isTlshValid($composite[1])) { + return true; + } + return __('TLSH hash has an invalid length or format (expected: filename|at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".'); + case 'filename|vhash': + if (preg_match('#^.+\|.+$#', $value)) { + return true; + } + return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".'); + case 'ip-src': + case 'ip-dst': + if (strpos($value, '/') !== false) { + $parts = explode("/", $value); + if (count($parts) !== 2 || !self::isPositiveInteger($parts[1])) { + return __('Invalid CIDR notation value found.'); + } + + if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + if ($parts[1] > 32) { + return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.'); + } + } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + if ($parts[1] > 128) { + return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.'); + } + } else { + return __('IP address has an invalid format.'); + } + } else if (!filter_var($value, FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + return true; + case 'port': + if (!self::isPortValid($value)) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'ip-dst|port': + case 'ip-src|port': + $parts = explode('|', $value); + if (!filter_var($parts[0], FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + if (!self::isPortValid($parts[1])) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'mac-address': + return preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value) === 1; + case 'mac-eui-64': + return preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value) === 1; + case 'hostname': + case 'domain': + if (self::isDomainValid($value)) { + return true; + } + return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type)); + case 'hostname|port': + $parts = explode('|', $value); + if (!self::isDomainValid($parts[0])) { + return __('Hostname has an invalid format.'); + } + if (!self::isPortValid($parts[1])) { + return __('Port numbers have to be integers between 1 and 65535.'); + } + return true; + case 'domain|ip': + $parts = explode('|', $value); + if (!self::isDomainValid($parts[0])) { + return __('Domain has an invalid format.'); + } + if (!filter_var($parts[1], FILTER_VALIDATE_IP)) { + return __('IP address has an invalid format.'); + } + return true; + case 'email': + case 'email-src': + case 'eppn': + case 'email-dst': + case 'target-email': + case 'whois-registrant-email': + case 'dns-soa-email': + case 'jabber-id': + // we don't use the native function to prevent issues with partial email addresses + if (preg_match("#^.*\@.*\..*$#i", $value)) { + return true; + } + return __('Email address has an invalid format. Please double check the value or select type "other".'); + case 'vulnerability': + if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) { + return true; + } + return __('Invalid format. Expected: CVE-xxxx-xxxx...'); + case 'weakness': + if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) { + return true; + } + return __('Invalid format. Expected: CWE-x...'); + case 'windows-service-name': + case 'windows-service-displayname': + if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) { + return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.'); + } + return true; + case 'mutex': + case 'process-state': + case 'snort': + case 'bro': + case 'zeek': + case 'community-id': + case 'anonymised': + case 'pattern-in-file': + case 'pattern-in-traffic': + case 'pattern-in-memory': + case 'filename-pattern': + case 'pgp-public-key': + case 'pgp-private-key': + case 'yara': + case 'stix2-pattern': + case 'sigma': + case 'gene': + case 'kusto-query': + case 'mime-type': + case 'identity-card-number': + case 'cookie': + case 'attachment': + case 'malware-sample': + case 'comment': + case 'text': + case 'other': + case 'cpe': + case 'email-attachment': + case 'email-body': + case 'email-header': + case 'first-name': + case 'middle-name': + case 'last-name': + case 'full-name': + return true; + case 'link': + // Moved to a native function whilst still enforcing the scheme as a requirement + return (bool)filter_var($value, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED); + case 'hex': + return ctype_xdigit($value); + case 'target-user': + case 'campaign-name': + case 'campaign-id': + case 'threat-actor': + case 'target-machine': + case 'target-org': + case 'target-location': + case 'target-external': + case 'email-subject': + case 'malware-type': + // TODO: review url/uri validation + case 'url': + case 'uri': + case 'user-agent': + case 'regkey': + case 'regkey|value': + case 'filename': + case 'pdb': + case 'windows-scheduled-task': + case 'whois-registrant-name': + case 'whois-registrant-org': + case 'whois-registrar': + case 'whois-creation-date': + case 'date-of-birth': + case 'place-of-birth': + case 'gender': + case 'passport-number': + case 'passport-country': + case 'passport-expiration': + case 'redress-number': + case 'nationality': + case 'visa-number': + case 'issue-date-of-the-visa': + case 'primary-residence': + case 'country-of-residence': + case 'special-service-request': + case 'frequent-flyer-number': + case 'travel-details': + case 'payment-details': + case 'place-port-of-original-embarkation': + case 'place-port-of-clearance': + case 'place-port-of-onward-foreign-destination': + case 'passenger-name-record-locator-number': + case 'email-dst-display-name': + case 'email-src-display-name': + case 'email-reply-to': + case 'email-x-mailer': + case 'email-mime-boundary': + case 'email-thread-index': + case 'email-message-id': + case 'github-username': + case 'github-repository': + case 'github-organisation': + case 'twitter-id': + case 'dkim': + case 'dkim-signature': + case 'favicon-mmh3': + case 'chrome-extension-id': + case 'mobile-application-id': + case 'named pipe': + if (strpos($value, "\n") !== false) { + return __('Value must not contain new line character.'); + } + return true; + case 'ssh-fingerprint': + if (self::isSshFingerprint($value)) { + return true; + } + return __('SSH fingerprint must be in MD5 or SHA256 format.'); + case 'datetime': + if (strtotime($value) !== false) { + return true; + } + return __('Datetime has to be in the ISO 8601 format.'); + case 'size-in-bytes': + case 'counter': + if (self::isPositiveInteger($value)) { + return true; + } + return __('The value has to be a whole number greater or equal 0.'); + /* case 'targeted-threat-index': + if (!is_numeric($value) || $value < 0 || $value > 10) { + return __('The value has to be a number between 0 and 10.'); + } + return true;*/ + case 'iban': + case 'bic': + case 'btc': + case 'dash': + case 'xmr': + return preg_match('/^[a-zA-Z0-9]+$/', $value) === 1; + case 'vhash': + return preg_match('/^.+$/', $value) === 1; + case 'bin': + case 'cc-number': + case 'bank-account-nr': + case 'aba-rtn': + case 'prtn': + case 'phone-number': + case 'whois-registrant-phone': + case 'float': + return is_numeric($value); + case 'cortex': + json_decode($value); + return json_last_error() === JSON_ERROR_NONE; + case 'boolean': + return $value == 1 || $value == 0; + case 'AS': + if (self::isPositiveInteger($value) && $value <= 4294967295) { + return true; + } + return __('AS number have to be integer between 1 and 4294967295'); + } + throw new InvalidArgumentException("Unknown type $type."); + } + + /** + * @param string $value + * @return bool + */ + private static function isDomainValid($value) + { + return preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value) === 1; + } + + /** + * @param string $value + * @return bool + */ + private static function isPortValid($value) + { + return self::isPositiveInteger($value) && $value >= 1 && $value <= 65535; + } + + /** + * @param string $value + * @return bool + */ + private static function isTlshValid($value) + { + if ($value[0] === 't') { + $value = substr($value, 1); + } + return strlen($value) > 35 && ctype_xdigit($value); + } + + + /** + * @param string $type + * @param string $value + * @return bool + */ + private static function isHashValid($type, $value) + { + return strlen($value) === self::HASH_HEX_LENGTH[$type] && ctype_xdigit($value); + } + + /** + * Returns true if input value is positive integer or zero. + * @param int|string $value + * @return bool + */ + private static function isPositiveInteger($value) + { + return (is_int($value) && $value >= 0) || ctype_digit($value); + } + + /** + * @param $value + * @return bool + */ + private static function isSsdeep($value) + { + $parts = explode(':', $value); + if (count($parts) !== 3) { + return false; + } + return self::isPositiveInteger($parts[0]); + } + + /** + * @param string $value + * @return bool + */ + private static function isSshFingerprint($value) + { + if (substr($value, 0, 7) === 'SHA256:') { + $value = substr($value, 7); + $decoded = base64_decode($value, true); + return $decoded && strlen($decoded) === 32; + } else if (substr($value, 0, 4) === 'MD5:') { + $value = substr($value, 4); + } + + $value = str_replace(':', '', $value); + return self::isHashValid('md5', $value); + } + + /** + * @param string $value + * @return string + */ + private static function compressIpv6($value) + { + if (strpos($value, ':') && $converted = inet_pton($value)) { + return inet_ntop($converted); + } + return $value; + } + + /** + * Temporary solution for utf8 columns until we migrate to utf8mb4. + * via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars + * @param string $input + * @return string + */ + private static function handle4ByteUnicode($input) + { + return preg_replace( + '%(?: + \xF0[\x90-\xBF][\x80-\xBF]{2} + | [\xF1-\xF3][\x80-\xBF]{3} + | \xF4[\x80-\x8F][\x80-\xBF]{2} + )%xs', + '?', + $input + ); + } +} diff --git a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php new file mode 100644 index 000000000..5bb39008d --- /dev/null +++ b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php @@ -0,0 +1,211 @@ +id = $properties['id']; + $this->command = $properties['command']; + $this->args = $properties['args'] ?? []; + $this->createdAt = $properties['createdAt'] ?? time(); + $this->updatedAt = $properties['updatedAt'] ?? null; + $this->status = $properties['status'] ?? self::STATUS_WAITING; + $this->error = $properties['error'] ?? null; + $this->progress = $properties['progress'] ?? 0; + $this->metadata = $properties['metadata'] ?? []; + } + + /** + * Run the job command + * + * @return self + */ + public function run(): self + { + $descriptorSpec = [ + 1 => ["pipe", "w"], // stdout + 2 => ["pipe", "w"], // stderr + ]; + + $process = proc_open( + array_merge( + [ + ROOT . DS . 'app' . DS . 'Console' . DS . 'cake', + $this->command(), + ], + $this->args() + ), + $descriptorSpec, + $pipes, + null, + ['BACKGROUND_JOB_ID' => $this->id] + ); + + $stdout = stream_get_contents($pipes[1]); + $this->setOutput($stdout); + fclose($pipes[1]); + + $stderr = stream_get_contents($pipes[2]); + $this->setError($stderr); + fclose($pipes[2]); + + $this->returnCode = proc_close($process); + + if ($this->returnCode === 0 && empty($stderr)) { + $this->setStatus(BackgroundJob::STATUS_COMPLETED); + $this->setProgress(100); + + CakeLog::info("[JOB ID: {$this->id()}] - completed."); + } else { + $this->setStatus(BackgroundJob::STATUS_FAILED); + + CakeLog::error("[JOB ID: {$this->id()}] - failed with error code {$this->returnCode}. STDERR: {$stderr}. STDOUT: {$stdout}."); + } + + return $this; + } + + public function jsonSerialize(): array + { + return [ + 'id' => $this->id, + 'command' => $this->command, + 'args' => $this->args, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + 'status' => $this->status, + 'output' => $this->output, + 'error' => $this->error, + 'metadata' => $this->metadata, + ]; + } + + public function id(): string + { + return $this->id; + } + + public function command(): string + { + return $this->command; + } + + public function args(): array + { + return $this->args; + } + + public function progress(): int + { + return $this->progress; + } + + public function createdAt(): int + { + return $this->createdAt; + } + + public function updatedAt(): ?int + { + return $this->updatedAt; + } + + public function status(): int + { + return $this->status; + } + + public function output(): ?string + { + return $this->output; + } + + public function error(): ?string + { + return $this->error; + } + + public function metadata(): array + { + return $this->metadata; + } + + public function returnCode(): int + { + return $this->returnCode; + } + + public function setStatus(int $status): void + { + $this->status = $status; + } + + public function setOutput(?string $output): void + { + $this->output = $output; + } + + public function setError(?string $error): void + { + $this->error = $error; + } + + public function setProgress(int $progress): void + { + $this->progress = $progress; + } + + public function setUpdatedAt(int $updatedAt): void + { + $this->updatedAt = $updatedAt; + } +} diff --git a/app/Lib/Tools/BackgroundJobs/Worker.php b/app/Lib/Tools/BackgroundJobs/Worker.php new file mode 100644 index 000000000..232957ec7 --- /dev/null +++ b/app/Lib/Tools/BackgroundJobs/Worker.php @@ -0,0 +1,107 @@ +pid = $properties['pid']; + $this->queue = $properties['queue']; + $this->user = $properties['user']; + $this->createdAt = $properties['createdAt'] ?? time(); + $this->updatedAt = $properties['updatedAt'] ?? null; + $this->status = $properties['status'] ?? self::STATUS_UNKNOWN; + } + + public function jsonSerialize(): array + { + return [ + 'pid' => $this->pid, + 'queue' => $this->queue, + 'user' => $this->user, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + 'status' => $this->status, + ]; + } + + public function pid(): ?int + { + return $this->pid; + } + + public function queue(): string + { + return $this->queue; + } + + public function user(): ?string + { + return $this->user; + } + + public function createdAt(): int + { + return $this->createdAt; + } + + public function updatedAt(): ?int + { + return $this->updatedAt; + } + + public function status(): int + { + return $this->status; + } + + public function setStatus(int $status): void + { + $this->status = $status; + } + + public function setUpdatedAt(int $updatedAt): void + { + $this->updatedAt = $updatedAt; + } +} diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php new file mode 100644 index 000000000..42977ad56 --- /dev/null +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -0,0 +1,700 @@ + 'EventShell', + self::CMD_SERVER => 'ServerShell', + self::CMD_ADMIN => 'AdminShell' + ]; + + public const JOB_STATUS_PREFIX = 'job_status'; + + /** @var array */ + private $settings; + + /** + * Initialize + * + * Settings should have the following format: + * [ + * 'enabled' => true, + * 'redis_host' => 'localhost', + * 'redis_port' => 6379, + * 'redis_password' => '', + * 'redis_database' => 1, + * 'redis_namespace' => 'background_jobs', + * 'max_job_history_ttl' => 86400 + * 'supervisor_host' => 'localhost', + * 'supervisor_port' => '9001', + * 'supervisor_user' => '', + * 'supervisor_password' => '', + * ] + * + * @param array $settings + */ + public function __construct(array $settings) + { + $this->settings = $settings; + + if ($this->settings['enabled'] === true) { + $this->RedisConnection = $this->createRedisConnection(); + } + } + + /** + * Enqueue a Job. + * + * @param string $queue Queue name, e.g. 'default'. + * @param string $command Command of the job. + * @param array $args Arguments passed to the job. + * @param boolean|null $trackStatus Whether to track the status of the job. + * @param int|null $jobId Id of the relational database record representing the job. + * @param array $metadata Related to the job. + * @return string Background Job ID. + * @throws InvalidArgumentException + */ + public function enqueue( + string $queue, + string $command, + array $args = [], + $trackStatus = null, + int $jobId = null, + array $metadata = [] + ): string { + + if (!$this->settings['enabled']) { + return $this->resqueEnqueue($queue, self::CMD_TO_SHELL_DICT[$command], $args, $trackStatus, $jobId); + } + + $this->validateQueue($queue); + $this->validateCommand($command); + + $backgroundJob = new BackgroundJob( + [ + 'id' => CakeText::uuid(), + 'command' => $command, + 'args' => $args, + 'metadata' => $metadata + ] + ); + + $this->RedisConnection->rpush( + $queue, + $backgroundJob + ); + + $this->update($backgroundJob); + + if ($jobId) { + $this->updateJobProcessId($jobId, $backgroundJob->id()); + } + + return $backgroundJob->id(); + } + + /** + * Enqueue a Job using the CakeResque. + * @deprecated + * + * @param string $queue Name of the queue to enqueue the job to. + * @param string $class Class of the job. + * @param array $args Arguments passed to the job. + * @param boolean $trackStatus Whether to track the status of the job. + * @param int|null $jobId Id of the relational database record representing the job. + * @return string Job Id. + */ + private function resqueEnqueue( + string $queue, + string $class, + $args = [], + $trackStatus = null, + int $jobId = null + ): string { + + $process_id = CakeResque::enqueue( + $queue, + $class, + $args, + $trackStatus + ); + + if ($jobId) { + $this->updateJobProcessId($jobId, $process_id); + } + + return $process_id; + } + + /** + * Dequeue a BackgroundJob. + * If the queue is empty the read is blocked until a job is pushed to this queue or the timeout is reached. + * + * @param string $queue Queue name, e.g. 'default'. + * @param int $timeout Time to block the read if the queue is empty. + * Must be less than your configured `read_write_timeout` + * for the redis connection. + * + * @return BackgroundJob|null. + * @throws Exception + */ + public function dequeue($queue, int $timeout = 30): ?BackgroundJob + { + $this->validateQueue($queue); + + $rawJob = $this->RedisConnection->blpop($queue, $timeout); + + if (!empty($rawJob)) { + return new BackgroundJob($rawJob[1]); + } + + return null; + } + + /** + * Get the job status. + * + * @param string $jobId Background Job Id. + * + * @return BackgroundJob|null job status. + * + */ + public function getJob(string $jobId): ?BackgroundJob + { + $rawJob = $this->RedisConnection->get( + self::JOB_STATUS_PREFIX . ':' . $jobId + ); + + if ($rawJob) { + return new BackgroundJob($rawJob); + } + + return null; + } + + /** + * Get the queues's names. + * + * @return array Array containing the queues' names. + */ + public function getQueues(): array + { + return self::VALID_QUEUES; + } + + /** + * Clear all the queue's jobs. + * + * @param string $queue Queue name, e.g. 'default'. + * + * @return boolean True on success, false on failure. + */ + public function clearQueue($queue): bool + { + $this->validateQueue($queue); + + return (bool) $this->RedisConnection->del($queue); + } + + /** + * Get all workers' instances. + * + * @return Worker[] List of worker's instances. + */ + public function getWorkers(): array + { + $workers = []; + $procs = $this->getSupervisor()->getAllProcesses(); + + foreach ($procs as $proc) { + if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) { + if ($proc->offsetGet('pid') > 0) { + $workers[] = new Worker([ + 'pid' => $proc->offsetGet('pid'), + 'queue' => explode("_", $proc->offsetGet('name'))[0], + 'user' => $this->processUser((int) $proc->offsetGet('pid')), + 'createdAt' => $proc->offsetGet('start'), + 'updatedAt' => $proc->offsetGet('now'), + 'status' => $this->convertProcessStatus($proc->offsetGet('state')) + ]); + } + } + } + + return $workers; + } + + /** + * Get the number of jobs inside a queue. + * + * @param string $queue Queue name, e.g. 'default'. + * + * @return integer Number of jobs. + */ + public function getQueueSize(string $queue): int + { + $this->validateQueue($queue); + + if (!$this->settings['enabled']) { + return CakeResque::getQueueSize($queue); + } + + return $this->RedisConnection->llen($queue); + } + + /** + * Update job + * + * @param BackgroundJob $job + * + * @return void + */ + public function update(BackgroundJob $job): void + { + $job->setUpdatedAt(time()); + + $this->RedisConnection->setex( + self::JOB_STATUS_PREFIX . ':' . $job->id(), + $this->settings['max_job_history_ttl'], + $job + ); + } + + /** + * Run job + * + * @param BackgroundJob $job + * + * @return integer Process return code. + */ + public function run(BackgroundJob $job): int + { + $job->setStatus(BackgroundJob::STATUS_RUNNING); + CakeLog::info("[JOB ID: {$job->id()}] - started."); + + $this->update($job); + + $job = $job->run(); + + $this->update($job); + + return $job->returnCode(); + } + + /** + * Start worker by name + * + * @param string $name + * @param boolean $waitForRestart + * @return boolean + */ + public function startWorker(string $name, bool $waitForRestart = false): bool + { + $this->validateWorkerName($name); + + return $this->getSupervisor()->startProcess( + sprintf( + '%s:%s', + self::MISP_WORKERS_PROCESS_GROUP, + $name + ), + $waitForRestart + ); + } + + /** + * Start worker by queue + * + * @param string $name + * @param boolean $waitForRestart + * @return boolean + */ + public function startWorkerByQueue(string $queue, bool $waitForRestart = false): bool + { + $this->validateQueue($queue); + + $procs = $this->getSupervisor()->getAllProcesses(); + + foreach ($procs as $proc) { + if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) { + $name = explode("_", $proc->offsetGet('name'))[0]; + if ($name === $queue && $proc->offsetGet('state') != \Supervisor\Process::RUNNING) { + return $this->getSupervisor()->startProcess( + sprintf( + '%s:%s', + self::MISP_WORKERS_PROCESS_GROUP, + $proc->offsetGet('name') + ), + $waitForRestart + ); + } + } + } + + return false; + } + + /** + * Stop worker by name or pid + * + * @param string|int $id + * @param boolean $waitForRestart + * @return boolean + */ + public function stopWorker($id, bool $waitForRestart = false): bool + { + if (is_numeric($id)) { + $process = $this->getProcessByPid((int)$id); + $name = $process->offsetGet('name'); + } else { + $name = $id; + } + + $this->validateWorkerName($name); + + return $this->getSupervisor()->stopProcess( + sprintf( + '%s:%s', + self::MISP_WORKERS_PROCESS_GROUP, + $name + ), + $waitForRestart + ); + } + + /** + * Restarts workers + * + * @param boolean $waitForRestart + * @return void + */ + public function restartWorkers(bool $waitForRestart = false): void + { + $this->getSupervisor()->stopProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); + $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); + } + + /** + * Restarts workers with status != RUNNING + * + * @param boolean $waitForRestart + * @return void + */ + public function restartDeadWorkers(bool $waitForRestart = false): void + { + $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); + } + + /** + * Purge queue + * + * @param string $queue + * @return void + */ + public function purgeQueue(string $queue): void + { + $this->validateQueue($queue); + + $this->RedisConnection->del($queue); + } + + /** + * Return Background Jobs status + * + * @return integer + */ + public function getStatus(): int + { + if (!$this->settings['enabled']) { + return self::STATUS_NOT_ENABLED; + } + + try { + $redisStatus = $this->RedisConnection->ping(); + } catch (Exception $exception) { + CakeLog::error("SimpleBackgroundJobs Redis error: {$exception->getMessage()}"); + $redisStatus = false; + } + + try { + $supervisorStatus = $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING; + } catch (Exception $exception) { + CakeLog::error("SimpleBackgroundJobs Supervisor error: {$exception->getMessage()}"); + $supervisorStatus = false; + } + + if ($redisStatus && $supervisorStatus) { + return self::STATUS_RUNNING; + } elseif (!$redisStatus && !$supervisorStatus) { + return self::STATUS_REDIS_AND_SUPERVISOR_NOT_OK; + } elseif ($redisStatus && !$supervisorStatus) { + return self::STATUS_SUPERVISOR_NOT_OK; + } else { + return self::STATUS_REDIS_NOT_OK; + } + } + + /** + * Validate queue + * + * @return boolean + * @throws InvalidArgumentException + */ + private function validateQueue(string $queue): bool + { + if (!in_array($queue, self::VALID_QUEUES, true)) { + throw new InvalidArgumentException( + sprintf( + 'Invalid background job queue %s, must be one of: [%s]', + $queue, + implode(', ', self::VALID_QUEUES) + ) + ); + } + + return true; + } + + /** + * Validate command + * + * @return boolean + * @throws InvalidArgumentException + */ + private function validateCommand(string $command): bool + { + if (!in_array($command, self::ALLOWED_COMMANDS, true)) { + throw new InvalidArgumentException( + sprintf( + 'Invalid command %s, must be one of: [%s]', + $command, + implode(', ', self::ALLOWED_COMMANDS) + ) + ); + } + + return true; + } + + /** + * Validate worker name + * + * @param string $name + * @return boolean + * @throws InvalidArgumentException + */ + private function validateWorkerName(string $name): bool + { + list($queue, $id) = explode('_', $name); + + $this->validateQueue($queue); + + if (!$this->validateQueue($queue) || !is_numeric($id)) { + throw new InvalidArgumentException("Invalid worker name $name, must be one of format {queue_name}_{process_id}, example: default_00"); + } + + return true; + } + + /** + * @return Redis + */ + private function createRedisConnection(): Redis + { + $redis = new Redis(); + $redis->connect($this->settings['redis_host'], $this->settings['redis_port']); + $redis->setOption(Redis::OPT_SERIALIZER, Redis::SERIALIZER_JSON); + $redis->setOption(Redis::OPT_PREFIX, $this->settings['redis_namespace'] . ':'); + $redisPassword = $this->settings['redis_password']; + + if (!empty($redisPassword)) { + $redis->auth($redisPassword); + } + $redis->select($this->settings['redis_database']); + + return $redis; + } + + /** + * @return \Supervisor\Supervisor + */ + private function getSupervisor() + { + if (!$this->Supervisor) { + $this->Supervisor = $this->createSupervisorConnection(); + } + return $this->Supervisor; + } + + /** + * @return \Supervisor\Supervisor + */ + private function createSupervisorConnection(): \Supervisor\Supervisor + { + $httpOptions = []; + if (!empty($this->settings['supervisor_user']) && !empty($this->settings['supervisor_password'])) { + $httpOptions = [ + 'auth' => [ + $this->settings['supervisor_user'], + $this->settings['supervisor_password'], + ], + ]; + } + + $client = new \fXmlRpc\Client( + sprintf( + 'http://%s:%s/RPC2', + $this->settings['supervisor_host'], + $this->settings['supervisor_port'] + ), + new \fXmlRpc\Transport\HttpAdapterTransport( + new \Http\Message\MessageFactory\GuzzleMessageFactory(), + new \GuzzleHttp\Client($httpOptions) + ) + ); + + return new \Supervisor\Supervisor($client); + } + + private function updateJobProcessId(int $jobId, string $processId): void + { + $job = ClassRegistry::init('Job'); + $job->id = $jobId; + $job->save(['process_id' => $processId]); + } + + /** + * Get Supervisor process by PID + * + * @param integer $pid + * @return \Supervisor\Process + * + * @throws NotFoundException + */ + private function getProcessByPid(int $pid): \Supervisor\Process + { + $procs = $this->getSupervisor()->getAllProcesses(); + + foreach ($procs as $proc) { + if ( + $proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP && + $proc->offsetGet('pid') === $pid + ) { + return $proc; + } + } + + throw new NotFoundException("Worker with pid=$pid not found."); + } + + /** + * Convert process status to worker status + * + * @param integer $stateId + * @return integer + */ + private function convertProcessStatus(int $stateId): int + { + switch ($stateId) { + case \Supervisor\Process::RUNNING: + return Worker::STATUS_RUNNING; + case \Supervisor\Process::UNKNOWN: + return Worker::STATUS_UNKNOWN; + default: + return Worker::STATUS_FAILED; + } + } + + /** + * Get effective user name + * @param int $pid + * @return string + */ + private function processUser(int $pid) + { + if (function_exists('posix_getpwuid') && file_exists("/proc/$pid/status")) { + $content = file_get_contents("/proc/$pid/status"); + preg_match("/Uid:\t([0-9]+)\t([0-9]+)/", $content, $matches); + return posix_getpwuid((int)$matches[2])['name']; + } else { + return trim(shell_exec(sprintf("ps -o uname='' -p %s", $pid)) ?? ''); + } + } +} diff --git a/app/Lib/Tools/BetterSecurity.php b/app/Lib/Tools/BetterSecurity.php new file mode 100644 index 000000000..0e98f108c --- /dev/null +++ b/app/Lib/Tools/BetterSecurity.php @@ -0,0 +1,66 @@ + $sortedElement) { $sortArray[$k] = $items[$k]; } - $items = array(); $items = $sortArray; } if (!$escapeReindex) { @@ -113,11 +112,12 @@ class CustomPaginationTool $params = $this->createPaginationRules($items, $options, $model, $sort, $focusKey); $items = $this->sortArray($items, $params, $escapeReindex); if (!empty($params['options']['focus'])) { + $focus = $params['options']['focus']; foreach ($items as $k => $item) { - if ($item[$focusKey] == $params['options']['focus']) { + if ($item[$focusKey] === $focus) { $params['page'] = 1 + intval(floor($k / $params['limit'])); $params['current'] = 1 + ($params['page'] - 1) * $params['limit']; - continue; + break; } } unset($params['options']['focus']); diff --git a/app/Lib/Tools/EncryptedValue.php b/app/Lib/Tools/EncryptedValue.php new file mode 100644 index 000000000..540e9fc92 --- /dev/null +++ b/app/Lib/Tools/EncryptedValue.php @@ -0,0 +1,68 @@ +value = $value; + $this->isJson = $isJson; + } + + /** + * @return mixed + * @throws JsonException + * @throws Exception + */ + public function decrypt() + { + $decrypt = BetterSecurity::decrypt(substr($this->value, 2), Configure::read('Security.encryption_key')); + return $this->isJson ? JsonTool::decode($decrypt) : $decrypt; + } + + public function __toString() + { + return $this->decrypt(); + } + + public function jsonSerialize() + { + return $this->decrypt(); + } + + /** + * Encrypt provided string if encryption is enabled. If not enabled, input value will be returned. + * @param string $value + * @return string + * @throws Exception + */ + public static function encryptIfEnabled($value) + { + $key = Configure::read('Security.encryption_key'); + if ($key) { + return EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $key); + } + return $value; + } + + /** + * Check if value is encrypted (starts with encrypted magic) + * @param string $value + * @return bool + */ + public static function isEncrypted($value) + { + return substr($value, 0, 2) === EncryptedValue::ENCRYPTED_MAGIC; + } +} diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index c6e8a8342..47e184f16 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -1,7 +1,23 @@ get($url); + if (!$response->isOk()) { + throw new HttpSocketHttpException($response, $url); + } + return $response->json(); + } + + /** + * @param HttpSocketExtended $HttpSocket + * @return string + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public static function getLatestCommit(HttpSocketExtended $HttpSocket) + { + $url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1'; + $response = $HttpSocket->get($url); + if (!$response->isOk()) { + throw new HttpSocketHttpException($response, $url); + } + $data = $response->json(); + if (!isset($data[0]['sha'])) { + throw new Exception("Response do not contains requested data."); + } + return $data[0]['sha']; + } + + /** + * `git rev-parse HEAD` + * @return string + * @throws Exception + */ + public static function currentCommit() + { + $head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD')); + if (substr($head, 0, 5) === 'ref: ') { + $path = substr($head, 5); + return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path)); + } else if (strlen($head) === 40) { + return $head; + } else { + throw new Exception("Invalid head $head"); + } + } + + /** + * `git symbolic-ref HEAD` + * @return string + * @throws Exception + */ + public static function currentBranch() + { + $head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD')); + if (substr($head, 0, 5) === 'ref: ') { + $path = substr($head, 5); + return str_replace('refs/heads/', '', $path); + } else { + throw new Exception("ref HEAD is not a symbolic ref"); + } + } + + /** + * @return array + * @throws Exception + */ + public static function submoduleStatus() + { + $lines = ProcessTool::execute(['git', 'submodule', 'status', '--cached'], ROOT); + $output = []; + foreach (explode("\n", $lines) as $submodule) { + if ($submodule === '' || $submodule[0] === '-') { + continue; + } + $parts = explode(' ', $submodule); + $output[] = [ + 'name' => $parts[2], + 'commit' => $parts[1], + ]; + } + return $output; + } + + /** + * @param string $submodule Path to Git repo + * @return string|null + */ + public static function submoduleCurrentCommit($submodule) + { + try { + $commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule); + } catch (ProcessException $e) { + return null; + } + return rtrim($commit); + } + + /** + * @param string $commit + * @param string|null $submodule Path to Git repo + * @return int|null + */ + public static function commitTimestamp($commit, $submodule = null) + { + try { + $timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule); + } catch (ProcessException $e) { + return null; + } + return (int)rtrim($timestamp); + } +} diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php index 24cf8f60b..024bf880a 100644 --- a/app/Lib/Tools/JSONConverterTool.php +++ b/app/Lib/Tools/JSONConverterTool.php @@ -48,7 +48,7 @@ class JSONConverterTool } } - if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) { + if (empty($event['Event']['SharingGroup'])) { unset($event['Event']['SharingGroup']); } @@ -86,11 +86,6 @@ class JSONConverterTool } unset($tempSightings); unset($event['Event']['RelatedAttribute']); - if (isset($event['Event']['RelatedEvent'])) { - foreach ($event['Event']['RelatedEvent'] as $key => $value) { - unset($event['Event']['RelatedEvent'][$key]['Event']['user_id']); - } - } $result = array('Event' => $event['Event']); if (isset($event['errors'])) { $result = array_merge($result, array('errors' => $event['errors'])); @@ -143,7 +138,7 @@ class JSONConverterTool { // remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser foreach ($attributes as $key => $attribute) { - if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) { + if (empty($attribute['SharingGroup'])) { unset($attributes[$key]['SharingGroup']); } unset($attributes[$key]['value1']); diff --git a/app/Lib/Tools/JsonTool.php b/app/Lib/Tools/JsonTool.php new file mode 100644 index 000000000..8751c76f7 --- /dev/null +++ b/app/Lib/Tools/JsonTool.php @@ -0,0 +1,32 @@ +stderr = $stderr; + $this->stdout = $stdout; + parent::__construct($message, $returnCode); + } + + public function stderr() + { + return $this->stderr; + } + + public function stdout() + { + return $this->stdout; + } +} + +class ProcessTool +{ + const LOG_FILE = APP . 'tmp/logs/exec-errors.log'; + + /** + * @param array $command If command is array, it is not necessary to escape arguments + * @param string|null $cwd + * @param bool $stderrToFile IF true, log stderrr output to LOG_FILE + * @return string Stdout + * @throws ProcessException + * @throws Exception + */ + public static function execute(array $command, $cwd = null, $stderrToFile = false) + { + $descriptorSpec = [ + 1 => ["pipe", "w"], // stdout + 2 => ["pipe", "w"], // stderr + ]; + + if ($stderrToFile) { + self::logMessage('Running command ' . implode(' ', $command)); + $descriptorSpec[2] = ["file", self::LOG_FILE, 'a']; + } + + // PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually + if (PHP_VERSION_ID < 70400) { + $command = array_map('escapeshellarg', $command); + $command = implode(' ', $command); + } + $process = proc_open($command, $descriptorSpec, $pipes, $cwd); + if (!$process) { + $commandForException = self::commandFormat($command); + throw new Exception("Command '$commandForException' could be started."); + } + + $stdout = stream_get_contents($pipes[1]); + if ($stdout === false) { + $commandForException = self::commandFormat($command); + throw new Exception("Could not get STDOUT of command '$commandForException'."); + } + fclose($pipes[1]); + + if ($stderrToFile) { + $stderr = null; + } else { + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[2]); + } + + $returnCode = proc_close($process); + + if ($stderrToFile) { + self::logMessage("Process finished with return code $returnCode"); + } + + if ($returnCode !== 0) { + throw new ProcessException($command, $returnCode, $stderr, $stdout); + } + + return $stdout; + } + + /** + * @return string + */ + public static function pythonBin() + { + return Configure::read('MISP.python_bin') ?: 'python3'; + } + + private static function logMessage($message) + { + $logMessage = '[' . date("Y-m-d H:i:s") . ' ' . getmypid() . "] $message\n"; + file_put_contents(self::LOG_FILE, $logMessage, FILE_APPEND | LOCK_EX); + } + + /** + * @param array|string $command + * @return string + */ + private static function commandFormat($command) + { + return is_array($command) ? implode(' ', $command) : $command; + } +} diff --git a/app/Lib/Tools/PubSubTool.php b/app/Lib/Tools/PubSubTool.php index 078995f20..225d1c12b 100644 --- a/app/Lib/Tools/PubSubTool.php +++ b/app/Lib/Tools/PubSubTool.php @@ -1,4 +1,8 @@ redis) { $settings = $this->getSetSettings(); $this->setupPubServer($settings); $this->redis = $this->createRedisConnection($settings); - $this->settings = $settings; } } @@ -35,17 +33,19 @@ class PubSubTool */ public function checkIfRunning($pidFilePath = null) { - $pidFile = new File($pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid'); - if (!$pidFile->exists()) { + $pidFile = $pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid'; + clearstatcache(false, $pidFile); + if (!file_exists($pidFile)) { return false; } - $pid = $pidFile->read(); + $pid = file_get_contents($pidFile); if ($pid === false || $pid === '') { return false; } if (!is_numeric($pid)) { throw new Exception('Internal error (invalid PID file for the MISP zmq script)'); } + clearstatcache(false, "/proc/$pid"); $result = file_exists("/proc/$pid"); if ($result === false) { return false; @@ -57,8 +57,8 @@ class PubSubTool { $settings = $this->getSetSettings(); $redis = $this->createRedisConnection($settings); - $redis->rPush($settings['redis_namespace'] . ':command', 'status'); - $response = $redis->blPop($settings['redis_namespace'] . ':status', 5); + $redis->rPush( 'command', 'status'); + $response = $redis->blPop('status', 5); if ($response === null) { throw new Exception("No response from status command returned after 5 seconds."); } @@ -67,9 +67,9 @@ class PubSubTool public function checkIfPythonLibInstalled() { - $my_server = ClassRegistry::init('Server'); - $result = trim(shell_exec($my_server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py')); - if ($result === "OK") { + $script = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py'; + $result = ProcessTool::execute([ProcessTool::pythonBin(), $script]); + if (trim($result) === "OK") { return true; } return false; @@ -79,8 +79,8 @@ class PubSubTool { App::uses('JSONConverterTool', 'Tools'); $jsonTool = new JSONConverterTool(); - $json = $jsonTool->convert($event); - return $this->pushToRedis(':data:misp_json', $json); + $json = $jsonTool->convert($event, false, true); + return $this->pushToRedis('data:misp_json', $json); } public function event_save(array $event, $action) @@ -88,7 +88,7 @@ class PubSubTool if (!empty($action)) { $event['action'] = $action; } - return $this->pushToRedis(':data:misp_json_event', $event); + return $this->pushToRedis('data:misp_json_event', $event); } public function object_save(array $object, $action) @@ -96,7 +96,7 @@ class PubSubTool if (!empty($action)) { $object['action'] = $action; } - return $this->pushToRedis(':data:misp_json_object', $object); + return $this->pushToRedis('data:misp_json_object', $object); } public function object_reference_save(array $object_reference, $action) @@ -104,12 +104,12 @@ class PubSubTool if (!empty($action)) { $object_reference['action'] = $action; } - return $this->pushToRedis(':data:misp_json_object_reference', $object_reference); + return $this->pushToRedis('data:misp_json_object_reference', $object_reference); } public function publishConversation(array $message) { - return $this->pushToRedis(':data:misp_json_conversation', $message); + return $this->pushToRedis('data:misp_json_conversation', $message); } public function attribute_save(array $attribute, $action = false) @@ -117,7 +117,7 @@ class PubSubTool if (!empty($action)) { $attribute['action'] = $action; } - return $this->pushToRedis(':data:misp_json_attribute', $attribute); + return $this->pushToRedis('data:misp_json_attribute', $attribute); } public function tag_save(array $tag, $action = false) @@ -125,7 +125,7 @@ class PubSubTool if (!empty($action)) { $tag['action'] = $action; } - return $this->pushToRedis(':data:misp_json_tag', $tag); + return $this->pushToRedis('data:misp_json_tag', $tag); } public function sighting_save(array $sighting, $action = false) @@ -133,7 +133,7 @@ class PubSubTool if (!empty($action)) { $sighting['action'] = $action; } - return $this->pushToRedis(':data:misp_json_sighting', $sighting); + return $this->pushToRedis('data:misp_json_sighting', $sighting); } public function warninglist_save(array $warninglist, $action = false) @@ -141,7 +141,7 @@ class PubSubTool if (!empty($action)) { $warninglist['action'] = $action; } - return $this->pushToRedis(':data:misp_json_warninglist', $warninglist); + return $this->pushToRedis('data:misp_json_warninglist', $warninglist); } /** @@ -149,13 +149,14 @@ class PubSubTool * @param string $type * @param string|false $action * @return bool + * @throws JsonException */ public function modified($data, $type, $action = false) { if (!empty($action)) { $data['action'] = $action; } - return $this->pushToRedis(':data:misp_json_' . $type, $data); + return $this->pushToRedis('data:misp_json_' . $type, $data); } public function publish($data, $type, $action = false) @@ -163,7 +164,7 @@ class PubSubTool if (!empty($action)) { $data['action'] = $action; } - return $this->pushToRedis(':data:misp_json_' . $type, $data); + return $this->pushToRedis('data:misp_json_' . $type, $data); } public function killService() @@ -171,7 +172,7 @@ class PubSubTool if ($this->checkIfRunning()) { $settings = $this->getSetSettings(); $redis = $this->createRedisConnection($settings); - $redis->rPush($settings['redis_namespace'] . ':command', 'kill'); + $redis->rPush('command', 'kill'); sleep(1); if ($this->checkIfRunning()) { // Still running. @@ -194,7 +195,7 @@ class PubSubTool if ($this->checkIfRunning()) { $redis = $this->createRedisConnection($settings); - $redis->rPush($settings['redis_namespace'] . ':command', 'reload'); + $redis->rPush( 'command', 'reload'); } else { return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.'; } @@ -226,13 +227,12 @@ class PubSubTool if ($this->checkIfRunning(self::OLD_PID_LOCATION)) { // Old version is running, kill it and start again new one. $redis = $this->createRedisConnection($settings); - $redis->rPush($settings['redis_namespace'] . ':command', 'kill'); + $redis->rPush('command', 'kill'); sleep(1); } $this->saveSettingToFile($settings); - $server = ClassRegistry::init('Server'); - shell_exec($server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &'); + shell_exec(ProcessTool::pythonBin() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &'); } } @@ -240,14 +240,12 @@ class PubSubTool * @param string $ns * @param string|array $data * @return bool + * @throws JsonException */ private function pushToRedis($ns, $data) { - if (is_array($data)) { - $data = json_encode($data, JSON_UNESCAPED_UNICODE); - } - - $this->redis->rPush($this->settings['redis_namespace'] . $ns, $data); + $data = JsonTool::encode($data); + $this->redis->rPush($ns, $data); return true; } @@ -264,6 +262,7 @@ class PubSubTool $redis->auth($redisPassword); } $redis->select($settings['redis_database']); + $redis->setOption(Redis::OPT_PREFIX, $settings['redis_namespace'] . ':'); return $redis; } @@ -274,27 +273,21 @@ class PubSubTool private function saveSettingToFile(array $settings) { $settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json'; - $settingsFile = new File($settingFilePath, true, 0644); - if (!$settingsFile->exists()) { - throw new Exception("Could not create zmq config file '$settingFilePath'."); - } + // Because setting file contains secrets, it should be readable just by owner. But because in Travis test, // config file is created under one user and then changed under other user, file must be readable and writable // also by group. - @chmod($settingsFile->pwd(), 0660); // hide error if current user is not file owner - if (!$settingsFile->write(json_encode($settings))) { - throw new Exception("Could not write zmq config file '$settingFilePath'."); - } - $settingsFile->close(); + FileAccessTool::createFile($settingFilePath, 0660); + FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings)); } private function getSetSettings() { $settings = array( 'redis_host' => 'localhost', - 'redis_port' => '6379', + 'redis_port' => 6379, 'redis_password' => '', - 'redis_database' => '1', + 'redis_database' => 1, 'redis_namespace' => 'mispq', 'host' => '127.0.0.1', 'port' => '50000', @@ -302,8 +295,9 @@ class PubSubTool 'password' => null, ); + $pluginConfig = Configure::read('Plugin'); foreach ($settings as $key => $setting) { - $temp = Configure::read('Plugin.ZeroMQ_' . $key); + $temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null; if ($temp) { $settings[$key] = $temp; } diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index 6eceb8282..6d9080e40 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -1,5 +1,6 @@ execute(['uptime', '-s']); + $since = ProcessTool::execute(['uptime', '-s']); $since = new DateTime($since); $diff = (new DateTime())->diff($since); $diffDays = $diff->format('a'); @@ -399,7 +400,7 @@ class SecurityAudit // Python version try { - $pythonVersion = $this->execute([$server->getPythonVersion(), '-V']); + $pythonVersion = ProcessTool::execute([ProcessTool::pythonBin(), '-V']); $parts = explode(' ', $pythonVersion); if ($parts[0] !== 'Python') { throw new Exception("Invalid python version response: $pythonVersion"); @@ -499,34 +500,4 @@ class SecurityAudit } throw new RuntimeException("CakePHP version not found in file '$filePath'."); } - - private function execute(array $command) - { - $descriptorspec = [ - 1 => ["pipe", "w"], // stdout - 2 => ["pipe", "w"], // stderr - ]; - - $command = implode(' ', $command); - $process = proc_open($command, $descriptorspec, $pipes); - if (!$process) { - throw new Exception("Command '$command' could not be started."); - } - - $stdout = stream_get_contents($pipes[1]); - if ($stdout === false) { - throw new Exception("Could not get STDOUT of command."); - } - fclose($pipes[1]); - - $stderr = stream_get_contents($pipes[2]); - fclose($pipes[2]); - - $returnCode = proc_close($process); - if ($returnCode !== 0) { - throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'"); - } - - return $stdout; - } } diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index e1f96e8ff..669bb7df5 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -50,7 +50,9 @@ class ServerSyncTool public function eventExists(array $event) { $url = $this->server['Server']['url'] . '/events/view/' . $event['Event']['uuid']; + $start = microtime(true); $exists = $this->socket->head($url, [], $this->request); + $this->log($start, 'HEAD', $url, $exists); if ($exists->code == '404') { return false; } @@ -83,6 +85,17 @@ class ServerSyncTool return $this->get($url); } + /** + * @param array $rules + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function attributeSearch(array $rules) + { + return $this->post('/attributes/restSearch.json', $rules); + } + /** * @param array $params * @return HttpSocketResponseExtended @@ -170,6 +183,16 @@ class ServerSyncTool return $this->get('/users/view/me.json'); } + /** + * @return HttpSocketResponseExtended + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + public function resetAuthKey() + { + return $this->post('/users/resetauthkey/me', []); + } + /** * @param string $testString * @return HttpSocketResponseExtended @@ -235,7 +258,9 @@ class ServerSyncTool private function get($url) { $url = $this->server['Server']['url'] . $url; + $start = microtime(true); $response = $this->socket->get($url, [], $this->request); + $this->log($start, 'GET', $url, $response); if (!$response->isOk()) { throw new HttpSocketHttpException($response, $url); } @@ -261,7 +286,7 @@ class ServerSyncTool $logMessage, $data ); - file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND); + file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND | LOCK_EX); } $request = $this->request; @@ -275,7 +300,9 @@ class ServerSyncTool } } $url = $this->server['Server']['url'] . $url; + $start = microtime(true); $response = $this->socket->post($url, $data, $request); + $this->log($start, 'POST', $url, $response); if (!$response->isOk()) { throw new HttpSocketHttpException($response, $url); } @@ -300,4 +327,19 @@ class ServerSyncTool } return $url; } + + /** + * @param float $start + * @param string $method HTTP method + * @param string $url + * @param HttpSocketResponse $response + */ + private function log($start, $method, $url, HttpSocketResponse $response) + { + $duration = round(microtime(true) - $start, 3); + $responseSize = strlen($response->body); + $ce = $response->getHeader('Content-Encoding'); + $logEntry = '[' . date("Y-m-d H:i:s") . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n"; + file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX); + } } diff --git a/app/Model/AdminSetting.php b/app/Model/AdminSetting.php index 08c6be720..41a2d8a32 100644 --- a/app/Model/AdminSetting.php +++ b/app/Model/AdminSetting.php @@ -18,16 +18,27 @@ class AdminSetting extends AppModel public function changeSetting($setting, $value = false) { - $setting_object = $this->find('first', array( - 'conditions' => array('setting' => $setting) + $existing = $this->find('first', array( + 'conditions' => array('setting' => $setting), + 'fields' => ['id'], )); - $this->deleteAll(array('setting' => $setting)); - $this->create(); - $setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value); - if ($this->save($setting_object)) { - return true; + if ($existing) { + if ($this->save([ + 'id' => $existing['AdminSetting']['id'], + 'value' => $value, + ])) { + return true; + } else { + return $this->validationErrors; + } } else { - return $this->validationErrors; + $this->create(); + $existing['AdminSetting'] = array('setting' => $setting, 'value' => $value); + if ($this->save($existing)) { + return true; + } else { + return $this->validationErrors; + } } } diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 81b4835e0..effa54a82 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -23,23 +23,26 @@ App::uses('Model', 'Model'); App::uses('LogableBehavior', 'Assets.models/behaviors'); App::uses('RandomTool', 'Tools'); +App::uses('FileAccessTool', 'Tools'); +App::uses('JsonTool', 'Tools'); class AppModel extends Model { - public $name; - /** @var PubSubTool */ private static $loadedPubSubTool; /** @var KafkaPubTool */ private $loadedKafkaPubTool; + /** @var BackgroundJobsTool */ + private static $loadedBackgroundJobsTool; + /** @var null|Redis */ private static $__redisConnection; private $__profiler = array(); - public $elasticSearchClient = false; + public $elasticSearchClient; /** @var AttachmentTool|null */ private $attachmentTool; @@ -47,8 +50,6 @@ class AppModel extends Model public function __construct($id = false, $table = null, $ds = null) { parent::__construct($id, $table, $ds); - - $this->name = get_class($this); $this->findMethods['column'] = true; } @@ -83,7 +84,8 @@ class AppModel extends Model 51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false, 57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false, 63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false, - 69 => false, 70 => false, 71 => true, 72 => true, + 69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false, + 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, ); public $advanced_updates_description = array( @@ -246,13 +248,13 @@ class AppModel extends Model $result = $this->Feed->addDefaultFeeds($feeds); $this->Log->create(); $entry = array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_database', - 'user_id' => 0, - 'title' => 'Added new default feeds.' + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => 'Added new default feeds.' ); if ($result) { $entry['change'] = 'Feeds added: ' . $feedNames; @@ -1533,7 +1535,7 @@ class AppModel extends Model `value` text NOT NULL, `from_json` tinyint(1) default 0, PRIMARY KEY (`id`), - UNIQUE INDEX `value` (`value`(255)) + UNIQUE INDEX `value` (`value`(191)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; break; case 66: @@ -1578,6 +1580,40 @@ class AppModel extends Model case 72: $sqlArray[] = "ALTER TABLE `auth_keys` ADD `read_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `expiration`;"; break; + case 73: + $this->__dropIndex('user_settings', 'timestamp'); // index is not used + $sqlArray[] = "ALTER TABLE `user_settings` ADD UNIQUE INDEX `unique_setting` (`user_id`, `setting`)"; + break; + case 74: + $sqlArray[] = "ALTER TABLE `users` MODIFY COLUMN `change_pw` tinyint(1) NOT NULL DEFAULT 0;"; + break; + case 75: + $this->__addIndex('object_references', 'event_id'); + $this->__dropIndex('object_references', 'timestamp'); + $this->__dropIndex('object_references', 'source_uuid'); + $this->__dropIndex('object_references', 'relationship_type'); + $this->__dropIndex('object_references', 'referenced_uuid'); + break; + case 76: + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `system_settings` ( + `setting` varchar(255) NOT NULL, + `value` blob NOT NULL, + PRIMARY KEY (`setting`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + $sqlArray[] = "ALTER TABLE `servers` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL;"; + $sqlArray[] = "ALTER TABLE `cerebrates` MODIFY COLUMN `authkey` VARBINARY(255) NOT NULL;"; + break; + case 77: + $sqlArray[] = "ALTER TABLE `tags` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `is_custom_galaxy`;"; + $sqlArray[] = "ALTER TABLE `galaxies` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `enabled`;"; + break; + case 78: + $sqlArray[] = "ALTER TABLE `jobs` MODIFY COLUMN `process_id` varchar(36) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL;"; + break; + case 79: + $sqlArray[] = "ALTER TABLE `users` ADD `sub` varchar(255) NULL DEFAULT NULL;"; + $sqlArray[] = "ALTER TABLE `users` ADD UNIQUE INDEX `sub` (`sub`);"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -1767,14 +1803,14 @@ class AppModel extends Model if ($flagStop && $errorCount > 0) { $this->Log->create(); $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_database', - 'user_id' => 0, - 'title' => __('Issues executing the SQL query for %s', $command), - 'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.') + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => __('Issues executing the SQL query for %s', $command), + 'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.') )); return false; } @@ -1840,14 +1876,14 @@ class AppModel extends Model } $this->Log->create(); $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_database', - 'user_id' => 0, - 'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table, - 'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table, + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table, + 'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table, )); } } @@ -1879,14 +1915,14 @@ class AppModel extends Model } $this->Log->create(); $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_database', - 'user_id' => 0, - 'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage), - 'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage), + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage), + 'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage), )); $additionResult = array('success' => $result || $duplicate); if (!$result) { @@ -1914,11 +1950,7 @@ class AppModel extends Model public function getPythonVersion() { - if (!empty(Configure::read('MISP.python_bin'))) { - return Configure::read('MISP.python_bin'); - } else { - return 'python3'; - } + return Configure::read('MISP.python_bin') ?: 'python3'; } public function validateAuthkey($value) @@ -1935,10 +1967,9 @@ class AppModel extends Model // alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7 public function valueNotEmpty($value) { - $field = array_keys($value); - $field = $field[0]; - $value[$field] = trim($value[$field]); - if (!empty($value[$field])) { + $field = array_keys($value)[0]; + $value = trim($value[$field]); + if (!empty($value)) { return true; } return ucfirst($field) . ' cannot be empty.'; @@ -1946,32 +1977,17 @@ class AppModel extends Model public function valueIsJson($value) { - $field = array_keys($value); - $field = $field[0]; - $json_decoded = json_decode($value[$field]); + $value = array_values($value)[0]; + $json_decoded = json_decode($value); if ($json_decoded === null) { return __('Invalid JSON.'); } return true; } - public function valueIsJsonOrNull($value) - { - $field = array_keys($value); - $field = $field[0]; - if (!is_null($value[$field])) { - $json_decoded = json_decode($value[$field]); - if ($json_decoded === null) { - return __('Invalid JSON.'); - } - } - return true; - } - public function valueIsID($value) { - $field = array_keys($value); - $field = $field[0]; + $field = array_keys($value)[0]; if (!is_numeric($value[$field]) || $value[$field] < 0) { return 'Invalid ' . ucfirst($field) . ' ID'; } @@ -1980,17 +1996,17 @@ class AppModel extends Model public function stringNotEmpty($value) { - $field = array_keys($value); - $field = $field[0]; - $value[$field] = trim($value[$field]); - if (!isset($value[$field]) || ($value[$field] == false && $value[$field] !== "0")) { + $field = array_keys($value)[0]; + $value = trim($value[$field]); + if (!isset($value) || ($value == false && $value !== "0")) { return ucfirst($field) . ' cannot be empty.'; } return true; } // Try to create a table with a BIGINT(20) - public function seenOnAttributeAndObjectPreUpdate() { + public function seenOnAttributeAndObjectPreUpdate() + { $sqlArray[] = "CREATE TABLE IF NOT EXISTS testtable ( `testfield` BIGINT(6) NULL DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"; @@ -2008,16 +2024,11 @@ class AppModel extends Model } } - public function failingPreUpdate() { - throw new Exception('Yolo fail'); - } - public function runUpdates($verbose = false, $useWorker = true, $processId = false) { $this->AdminSetting = ClassRegistry::init('AdminSetting'); $this->Job = ClassRegistry::init('Job'); - $this->Log = ClassRegistry::init('Log'); - $this->Server = ClassRegistry::init('Server'); + $db = ConnectionManager::getDataSource('default'); $tables = $db->listSources(); $requiresLogout = false; @@ -2027,7 +2038,10 @@ class AppModel extends Model $requiresLogout = true; } else { $this->__runCleanDB(); - $db_version = $this->AdminSetting->find('all', array('conditions' => array('setting' => 'db_version'))); + $db_version = $this->AdminSetting->find('all', [ + 'conditions' => array('setting' => 'db_version'), + 'fields' => ['id', 'value'], + ]); if (count($db_version) > 1) { // we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57 foreach ($db_version as $k => $v) { @@ -2046,6 +2060,8 @@ class AppModel extends Model $job = null; } if (!empty($updates)) { + $this->Log = ClassRegistry::init('Log'); + $this->Server = ClassRegistry::init('Server'); // Exit if updates are locked. // This is not as reliable as a real lock implementation // However, as all updates are re-playable, there is no harm if they @@ -2054,14 +2070,14 @@ class AppModel extends Model if ($this->isUpdateLocked()) { // prevent creation of useless workers $this->Log->create(); $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_db_worker', - 'user_id' => 0, - 'title' => __('Issues executing run_updates'), - 'change' => __('Database updates are locked. Worker not spawned') + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_db_worker', + 'user_id' => 0, + 'title' => __('Issues executing run_updates'), + 'change' => __('Database updates are locked. Worker not spawned') )); if (!empty($job)) { // if multiple prio worker is enabled, want to mark them as done $job['Job']['progress'] = 100; @@ -2080,26 +2096,28 @@ class AppModel extends Model } else { // update worker not running, doing the update inline return $this->runUpdates($verbose, false); } - $this->Job->create(); - $data = array( - 'worker' => $workerType, - 'job_type' => 'run_updates', - 'job_input' => 'command: ' . implode(',', $updates), - 'status' => 0, - 'retries' => 0, - 'org_id' => 0, - 'org' => '', - 'message' => 'Updating.', + + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_PRIO, + 'run_updates', + 'command: ' . implode(',', $updates), + 'Updating.' + ); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'runUpdates', + $jobId + ], + true, + $jobId ); - $this->Job->save($data); - $jobId = $this->Job->id; - $processId = CakeResque::enqueue( - 'prio', - 'AdminShell', - array('runUpdates', $jobId), - true - ); - $this->Job->saveField('process_id', $processId); + return true; } @@ -2109,14 +2127,14 @@ class AppModel extends Model if ($this->isUpdateLocked()) { $this->Log->create(); $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_db_worker', - 'user_id' => 0, - 'title' => __('Issues executing run_updates'), - 'change' => __('Updates are locked. Stopping worker gracefully') + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_db_worker', + 'user_id' => 0, + 'title' => __('Issues executing run_updates'), + 'change' => __('Updates are locked. Stopping worker gracefully') )); if (!empty($job)) { $job['Job']['progress'] = 100; @@ -2367,19 +2385,17 @@ class AppModel extends Model private function __runCleanDB() { - $cleanDB = $this->AdminSetting->find('first', array('conditions' => array('setting' => 'clean_db'))); - if (empty($cleanDB) || $cleanDB['AdminSetting']['value'] == 1) { + $cleanDB = $this->AdminSetting->getSetting('clean_db'); + if ($cleanDB === false || $cleanDB == 1) { $this->cleanCacheFiles(); - if (empty($cleanDB)) { - $this->AdminSetting->create(); - $cleanDB = array('AdminSetting' => array('setting' => 'clean_db', 'value' => 0)); - } else { - $cleanDB['AdminSetting']['value'] = 0; - } - $this->AdminSetting->save($cleanDB); + $this->AdminSetting->changeSetting('clean_db', 0); } } + /** + * @param string $db_version + * @return array + */ protected function findUpgrades($db_version) { $updates = array(); @@ -2419,84 +2435,31 @@ class AppModel extends Model private function __generateCorrelations() { if (Configure::read('MISP.background_jobs')) { - $Job = ClassRegistry::init('Job'); - $Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generate correlation', - 'job_input' => 'All attributes', - 'status' => 0, - 'retries' => 0, - 'org' => 'ADMIN', - 'message' => 'Job created.', + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate correlation', + 'All attributes', + 'Job created.' ); - $Job->save($data); - $jobId = $Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('jobGenerateCorrelation', $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateCorrelation', + $jobId + ], + true, + $jobId ); - $Job->saveField('process_id', $process_id); + } return true; } - public function populateNotifications($user, $mode = 'full') - { - $notifications = array(); - list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user, $mode); - $notifications['total'] = $notifications['proposalCount']; - if (Configure::read('MISP.delegation')) { - $notifications['delegationCount'] = $this->_getDelegationCount($user); - $notifications['total'] += $notifications['delegationCount']; - } - return $notifications; - } - - // if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu. - private function _getProposalCount($user, $mode = 'full') - { - $this->ShadowAttribute = ClassRegistry::init('ShadowAttribute'); - $results[0] = $this->ShadowAttribute->find( - 'count', - array( - 'recursive' => -1, - 'conditions' => array( - 'ShadowAttribute.event_org_id' => $user['org_id'], - 'ShadowAttribute.deleted' => 0, - ) - ) - ); - if ($mode === 'full') { - $results[1] = $this->ShadowAttribute->find( - 'count', - array( - 'recursive' => -1, - 'conditions' => array( - 'ShadowAttribute.event_org_id' => $user['org_id'], - 'ShadowAttribute.deleted' => 0, - ), - 'fields' => 'distinct event_id' - ) - ); - } else { - $results[1] = $results[0]; - } - return $results; - } - - private function _getDelegationCount($user) - { - $this->EventDelegation = ClassRegistry::init('EventDelegation'); - $delegations = $this->EventDelegation->find('count', array( - 'recursive' => -1, - 'conditions' => array('EventDelegation.org_id' => $user['org_id']) - )); - return $delegations; - } - public function checkFilename($filename) { return preg_match('@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i', $filename); @@ -2557,26 +2520,20 @@ class AppModel extends Model public function getKafkaPubTool() { if (!$this->loadedKafkaPubTool) { - $this->loadKafkaPubTool(); + App::uses('KafkaPubTool', 'Tools'); + $kafkaPubTool = new KafkaPubTool(); + $rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config'); + $kafkaConf = array(); + if (!empty($rdkafkaIni)) { + $kafkaConf = parse_ini_file($rdkafkaIni); + } + $brokers = Configure::read('Plugin.Kafka_brokers'); + $kafkaPubTool->initTool($brokers, $kafkaConf); + $this->loadedKafkaPubTool = $kafkaPubTool; } return $this->loadedKafkaPubTool; } - public function loadKafkaPubTool() - { - App::uses('KafkaPubTool', 'Tools'); - $kafkaPubTool = new KafkaPubTool(); - $rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config'); - $kafkaConf = array(); - if (!empty($rdkafkaIni)) { - $kafkaConf = parse_ini_file($rdkafkaIni); - } - $brokers = Configure::read('Plugin.Kafka_brokers'); - $kafkaPubTool->initTool($brokers, $kafkaConf); - $this->loadedKafkaPubTool = $kafkaPubTool; - return true; - } - public function publishKafkaNotification($topicName, $data, $action = false) { $kafkaTopic = $this->kafkaTopic($topicName); @@ -2599,26 +2556,40 @@ class AppModel extends Model return self::$loadedPubSubTool; } - public function getElasticSearchTool() + protected function getElasticSearchTool() { if (!$this->elasticSearchClient) { - $this->loadElasticSearchTool(); + App::uses('ElasticSearchClient', 'Tools'); + $client = new ElasticSearchClient(); + $client->initTool(); + $this->elasticSearchClient = $client; } return $this->elasticSearchClient; } - public function loadElasticSearchTool() + /** + * @return BackgroundJobsTool + */ + public function getBackgroundJobsTool(): BackgroundJobsTool { - App::uses('ElasticSearchClient', 'Tools'); - $client = new ElasticSearchClient(); - $client->initTool(); - $this->elasticSearchClient = $client; + if (!self::$loadedBackgroundJobsTool) { + App::uses('BackgroundJobsTool', 'Tools'); + + // TODO: remove after CakeResque is deprecated + $settings = ['enabled' => false]; + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $settings = Configure::read('SimpleBackgroundJobs'); + } + + $backgroundJobsTool = new BackgroundJobsTool($settings); + self::$loadedBackgroundJobsTool = $backgroundJobsTool; + } + return self::$loadedBackgroundJobsTool; } // generate a generic subquery - options needs to include conditions - public function subQueryGenerator($model, $options, $lookupKey, $negation = false) + protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false) { - $db = $model->getDataSource(); $defaults = array( 'fields' => array('*'), 'table' => $model->table, @@ -2631,17 +2602,15 @@ class AppModel extends Model 'recursive' => -1 ); $params = array(); - foreach (array_keys($defaults) as $key) { + foreach ($defaults as $key => $defaultValue) { if (isset($options[$key])) { $params[$key] = $options[$key]; } else { - $params[$key] = $defaults[$key]; + $params[$key] = $defaultValue; } } - $subQuery = $db->buildStatement( - $params, - $model - ); + $db = $model->getDataSource(); + $subQuery = $db->buildStatement($params, $model); if ($negation) { $subQuery = $lookupKey . ' NOT IN (' . $subQuery . ') '; } else { @@ -2712,15 +2681,6 @@ class AppModel extends Model } } - public function getRowCount($table = false) - { - if (empty($table)) { - $table = $this->table; - } - $table_data = $this->query("show table status like '" . $table . "'"); - return $table_data[0]['TABLES']['Rows']; - } - public function benchmarkCustomAdd($valueToAdd = 0, $name = 'default', $customName = 'custom') { if (empty($this->__profiler[$name]['custom'][$customName])) { @@ -2762,15 +2722,21 @@ class AppModel extends Model { $version = implode('.', $this->checkMISPVersion()); $commit = $this->checkMIPSCommit(); - $request = array( + + $authkey = $server[$model]['authkey']; + App::uses('EncryptedValue', 'Tools'); + if (EncryptedValue::isEncrypted($authkey)) { + $authkey = (string)new EncryptedValue($authkey); + } + + return array( 'header' => array( - 'Authorization' => $server[$model]['authkey'], + 'Authorization' => $authkey, 'Accept' => 'application/json', 'Content-Type' => 'application/json', 'User-Agent' => 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit), ) ); - return $request; } /** @@ -2783,9 +2749,8 @@ class AppModel extends Model { static $versionArray; if ($versionArray === null) { - $file = new File(ROOT . DS . 'VERSION.json'); - $versionArray = $this->jsonDecode($file->read()); - $file->close(); + $content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json'); + $versionArray = JsonTool::decode($content); } return $versionArray; } @@ -2799,10 +2764,11 @@ class AppModel extends Model { static $commit; if ($commit === null) { - $commit = shell_exec('git log --pretty="%H" -n1 HEAD'); - if ($commit) { - $commit = trim($commit); - } else { + App::uses('GitTool', 'Tools'); + try { + $commit = GitTool::currentCommit(); + } catch (Exception $e) { + $this->logException('Could not get current git commit', $e, LOG_NOTICE); $commit = false; } } @@ -2929,11 +2895,6 @@ class AppModel extends Model return $val / (1024 * 1024); } - public function getDefaultAttachments_dir() - { - return APP . 'files'; - } - private function __bumpReferences() { $this->Event = ClassRegistry::init('Event'); @@ -2991,14 +2952,14 @@ class AppModel extends Model $this->Log = ClassRegistry::init('Log'); $this->Log->create(); $entry = array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'update_database', - 'user_id' => 0, - 'title' => 'Bumped the timestamps of locked events containing object references.', - 'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids)) + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'update_database', + 'user_id' => 0, + 'title' => 'Bumped the timestamps of locked events containing object references.', + 'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids)) ); $this->Log->save($entry); } @@ -3017,10 +2978,10 @@ class AppModel extends Model } $multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1); $lastChar = strtolower(substr($delta, -1)); - if (!is_numeric($lastChar) && array_key_exists($lastChar, $multiplierArray)) { + if (!is_numeric($lastChar) && isset($multiplierArray[$lastChar])) { $multiplier = $multiplierArray[$lastChar]; $delta = substr($delta, 0, -1); - } else if(strtotime($delta) !== false) { + } else if (strtotime($delta) !== false) { return strtotime($delta); } else { // invalid filter, make sure we don't return anything @@ -3156,10 +3117,7 @@ class AppModel extends Model $message .= "\n"; do { - $message .= sprintf("[%s] %s", - get_class($exception), - $exception->getMessage() - ); + $message .= sprintf("[%s] %s", get_class($exception), $exception->getMessage()); $message .= "\nStack Trace:\n" . $exception->getTraceAsString(); $exception = $exception->getPrevious(); } while ($exception !== null); @@ -3167,23 +3125,6 @@ class AppModel extends Model return $this->log($message, $type); } - /** - * Generates random file name in tmp dir. - * @return string - */ - protected function tempFileName() - { - return $this->tempDir() . DS . $this->generateRandomFileName(); - } - - /** - * @return string - */ - protected function tempDir() - { - return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir(); - } - /** * Decodes JSON string and throws exception if string is not valid JSON or if is not array. * @@ -3210,23 +3151,6 @@ class AppModel extends Model return $decoded; } - /* - * Temporary solution for utf8 columns until we migrate to utf8mb4 - * via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars - */ - public function handle4ByteUnicode($input) - { - return preg_replace( - '%(?: - \xF0[\x90-\xBF][\x80-\xBF]{2} - | [\xF1-\xF3][\x80-\xBF]{3} - | \xF4[\x80-\x8F][\x80-\xBF]{2} - )%xs', - '?', - $input - ); - } - /** * Faster version of default `hasAny` method * @param array|null $conditions @@ -3239,9 +3163,22 @@ class AppModel extends Model 'conditions' => $conditions, 'recursive' => -1, 'callbacks' => false, + 'order' => [], // disable order )); } + /** + * @param int $value Timestamp in microseconds + * @return string + */ + protected function microTimestampToIso($value) + { + $sec = (int)($value / 1000000); + $micro = $value % 1000000; + $micro = str_pad($micro, 6, "0", STR_PAD_LEFT); + return DateTime::createFromFormat('U.u', "$sec.$micro")->format('Y-m-d\TH:i:s.uP'); + } + /** * @return AttachmentTool */ @@ -3295,4 +3232,20 @@ class AppModel extends Model } return null; } + + /** + * @param string $name + * @return bool + */ + protected function pubToZmq($name) + { + static $zmqEnabled; + if ($zmqEnabled === null) { + $zmqEnabled = (bool)Configure::read('Plugin.ZeroMQ_enable'); + } + if ($zmqEnabled) { + return Configure::read("Plugin.ZeroMQ_{$name}_notifications_enable"); + } + return false; + } } diff --git a/app/Model/AttachmentScan.php b/app/Model/AttachmentScan.php index c996c586d..36b3508e0 100644 --- a/app/Model/AttachmentScan.php +++ b/app/Model/AttachmentScan.php @@ -295,14 +295,26 @@ class AttachmentScan extends AppModel if ($canScan) { $job = ClassRegistry::init('Job'); - $jobId = $job->createJob('SYSTEM', Job::WORKER_DEFAULT, 'virus_scan', ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'], 'Scanning...'); - $processId = CakeResque::enqueue( + $jobId = $job->createJob( + 'SYSTEM', Job::WORKER_DEFAULT, - 'AdminShell', - array('scanAttachment', $type, $attribute['id'], $jobId), - true + 'virus_scan', + ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'], + 'Scanning...' + ); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'scanAttachment', + $type, + $attribute['id'], + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $processId); } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 23374516e..ba3282133 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -8,6 +8,7 @@ App::uses('RandomTool', 'Tools'); App::uses('AttachmentTool', 'Tools'); App::uses('TmpFileTool', 'Tools'); App::uses('ComplexTypeTool', 'Tools'); +App::uses('AttributeValidationTool', 'Tools'); /** * @property Event $Event @@ -351,21 +352,28 @@ class Attribute extends AppModel public function afterFind($results, $primary = false) { - foreach ($results as $k => $v) { - if (isset($v['Attribute']['object_relation']) && $v['Attribute']['object_relation'] === null) { - $results[$k]['Attribute']['object_relation'] = ''; + foreach ($results as &$v) { + $attribute = &$v['Attribute']; + if (!empty($attribute['first_seen'])) { + $attribute['first_seen'] = $this->microTimestampToIso($attribute['first_seen']); + } + if (!empty($attribute['last_seen'])) { + $attribute['last_seen'] = $this->microTimestampToIso($attribute['last_seen']); } - $results[$k] = $this->UTCToISODatetime($results[$k], $this->alias); } return $results; } public function beforeSave($options = array()) { - if (!empty($this->data['Attribute']['id'])) { + $attribute = &$this->data['Attribute']; + if (empty($attribute['uuid'])) { + $attribute['uuid'] = CakeText::uuid(); + } + if (!empty($attribute['id'])) { $this->old = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('Attribute.id' => $this->data['Attribute']['id']), + 'conditions' => array('Attribute.id' => $attribute['id']), 'fields' => ['value', 'disable_correlation', 'type', 'distribution', 'sharing_group_id'], )); } else { @@ -373,18 +381,18 @@ class Attribute extends AppModel } // explode value of composite type in value1 and value2 // or copy value to value1 if not composite type - if (!empty($this->data['Attribute']['type'])) { + if (!empty($attribute['type'])) { // explode composite types in value1 and value2 - if (in_array($this->data['Attribute']['type'], $this->getCompositeTypes(), true)) { - $pieces = explode('|', $this->data['Attribute']['value']); + if (in_array($attribute['type'], $this->getCompositeTypes(), true)) { + $pieces = explode('|', $attribute['value']); if (2 !== count($pieces)) { throw new InternalErrorException(__('Composite type, but value not explodable')); } - $this->data['Attribute']['value1'] = $pieces[0]; - $this->data['Attribute']['value2'] = $pieces[1]; + $attribute['value1'] = $pieces[0]; + $attribute['value2'] = $pieces[1]; } else { - $this->data['Attribute']['value1'] = $this->data['Attribute']['value']; - $this->data['Attribute']['value2'] = ''; + $attribute['value1'] = $attribute['value']; + $attribute['value2'] = ''; } } @@ -393,12 +401,28 @@ class Attribute extends AppModel return true; } + /** + * @param int $event_id + * @param bool $increment True for increment, false for decrement, + * @return bool + */ private function __alterAttributeCount($event_id, $increment = true) { - return $this->Event->updateAll( - array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count, 1) - 1'), - array('Event.id' => $event_id) - ); + // Temporary unbind models that we don't need to prevent deadlocks + $this->Event->unbindModel([ + 'belongsTo' => array_keys($this->Event->belongsTo), + ]); + try { + return $this->Event->updateAll( + array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count, 1) - 1'), + array('Event.id' => $event_id) + ); + } catch (Exception $e) { + $this->logException('Exception when updating event attribute count', $e); + return false; + } finally { + $this->Event->resetAssociations(); + } } public function afterSave($created, $options = array()) @@ -406,17 +430,19 @@ class Attribute extends AppModel // Passing event in `parentEvent` field will speed up correlation $passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false; + $attribute = $this->data['Attribute']; + // add attributeTags via the shorthand ID list - if (!empty($this->data['Attribute']['tag_ids'])) { - foreach ($this->data['Attribute']['tag_ids'] as $tag_id) { - $this->AttributeTag->attachTagToAttribute($this->id, $this->data['Attribute']['event_id'], $tag_id); + if (!empty($attribute['tag_ids'])) { + foreach ($attribute['tag_ids'] as $tagId) { + $this->AttributeTag->attachTagToAttribute($this->id, $attribute['event_id'], $tagId); } } // update correlation... - if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) { - $this->Correlation->beforeSaveCorrelation($this->data['Attribute']); - if (isset($this->data['Attribute']['event_id'])) { - $this->__alterAttributeCount($this->data['Attribute']['event_id'], false); + if (isset($attribute['deleted']) && $attribute['deleted']) { + $this->Correlation->beforeSaveCorrelation($attribute); + if (isset($attribute['event_id'])) { + $this->__alterAttributeCount($attribute['event_id'], false); } } else { /* @@ -429,71 +455,69 @@ class Attribute extends AppModel if (!$created) { if ( empty($this->old) || - $this->data['Attribute']['value'] != $this->old['Attribute']['value'] || - $this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] || - $this->data['Attribute']['type'] != $this->old['Attribute']['type'] || - $this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] || - $this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id'] + $attribute['value'] != $this->old['Attribute']['value'] || + $attribute['disable_correlation'] != $this->old['Attribute']['disable_correlation'] || + $attribute['type'] != $this->old['Attribute']['type'] || + $attribute['distribution'] != $this->old['Attribute']['distribution'] || + $attribute['sharing_group_id'] != $this->old['Attribute']['sharing_group_id'] ) { - $this->Correlation->beforeSaveCorrelation($this->data['Attribute']); - $this->Correlation->afterSaveCorrelation($this->data['Attribute'], false, $passedEvent); + $this->Correlation->beforeSaveCorrelation($attribute); + $this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent); } } else { - $this->Correlation->afterSaveCorrelation($this->data['Attribute'], false, $passedEvent); + $this->Correlation->afterSaveCorrelation($attribute, false, $passedEvent); } } $result = true; - // if the 'data' field is set on the $this->data then save the data to the correct file - if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type'])) { - if (isset($this->data['Attribute']['data_raw'])) { - $this->data['Attribute']['data'] = $this->data['Attribute']['data_raw']; - unset($this->data['Attribute']['data_raw']); - } elseif (isset($this->data['Attribute']['data'])) { - $this->data['Attribute']['data'] = base64_decode($this->data['Attribute']['data']); + // if the 'data' field is set on the $attribute then save the data to the correct file + if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) { + if (isset($attribute['data_raw'])) { + $attribute['data'] = $attribute['data_raw']; + unset($attribute['data_raw']); + $result = $this->saveAttachment($attribute); + } elseif (isset($attribute['data'])) { + $attribute['data'] = base64_decode($attribute['data']); + $result = $this->saveAttachment($attribute); } - $result = $this->saveAttachment($this->data['Attribute']); } $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable'); $kafkaTopic = $this->kafkaTopic('attribute'); if ($pubToZmq || $kafkaTopic) { - $attribute = $this->fetchAttribute($this->id); - if (!empty($attribute)) { + $attributeForPublish = $this->fetchAttribute($this->id); + if (!empty($attributeForPublish)) { $user = array( 'org_id' => -1, 'Role' => array( 'perm_site_admin' => 1 ) ); - $attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($attribute, $user, $attribute); - if (empty($attribute['Object']['id'])) { - unset($attribute['Object']); - } + $attributeForPublish['Attribute']['Sighting'] = $this->Sighting->attachToEvent($attributeForPublish, $user, $attributeForPublish); $action = $created ? 'add' : 'edit'; - if (!empty($this->data['Attribute']['deleted'])) { + if (!empty($attribute['deleted'])) { $action = 'soft-delete'; } if ($pubToZmq) { - if (Configure::read('Plugin.ZeroMQ_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) { - $attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']); + if (Configure::read('Plugin.ZeroMQ_include_attachments') && $this->typeIsAttachment($attributeForPublish['Attribute']['type'])) { + $attributeForPublish['Attribute']['data'] = $this->base64EncodeAttachment($attributeForPublish['Attribute']); } $pubSubTool = $this->getPubSubTool(); - $pubSubTool->attribute_save($attribute, $action); - unset($attribute['Attribute']['data']); + $pubSubTool->attribute_save($attributeForPublish, $action); + unset($attributeForPublish['Attribute']['data']); } if ($kafkaTopic) { - if (Configure::read('Plugin.Kafka_include_attachments') && $this->typeIsAttachment($attribute['Attribute']['type'])) { - $attribute['Attribute']['data'] = $this->base64EncodeAttachment($attribute['Attribute']); + if (Configure::read('Plugin.Kafka_include_attachments') && $this->typeIsAttachment($attributeForPublish['Attribute']['type'])) { + $attributeForPublish['Attribute']['data'] = $this->base64EncodeAttachment($attributeForPublish['Attribute']); } $kafkaPubTool = $this->getKafkaPubTool(); - $kafkaPubTool->publishJson($kafkaTopic, $attribute, $action); + $kafkaPubTool->publishJson($kafkaTopic, $attributeForPublish, $action); } } } - if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], ['ip-src', 'ip-dst'], true) && strpos($this->data['Attribute']['value'], '/')) { + if (Configure::read('MISP.enable_advanced_correlations') && in_array($attribute['type'], ['ip-src', 'ip-dst'], true) && strpos($attribute['value'], '/')) { $this->setCIDRList(); } - if ($created && isset($this->data['Attribute']['event_id']) && empty($this->data['Attribute']['skip_auto_increment'])) { - $this->__alterAttributeCount($this->data['Attribute']['event_id']); + if ($created && isset($attribute['event_id']) && empty($attribute['skip_auto_increment'])) { + $this->__alterAttributeCount($attribute['event_id']); } return $result; } @@ -547,81 +571,78 @@ class Attribute extends AppModel public function beforeValidate($options = array()) { - if (empty($this->data['Attribute']['type'])) { + $attribute = &$this->data['Attribute']; + if (empty($attribute['type'])) { $this->validationErrors['type'] = ['No type set.']; return false; } - $type = $this->data['Attribute']['type']; - if (is_array($this->data['Attribute']['value'])) { + $type = $attribute['type']; + if (is_array($attribute['value'])) { $this->validationErrors['value'] = ['Value is an array.']; return false; } - if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) { + if (!empty($attribute['object_id']) && empty($attribute['object_relation'])) { $this->validationErrors['object_relation'] = ['Object attribute sent, but no object_relation set.']; return false; } // If `value1` or `value2` provided and `value` is empty, merge them into `value` because of validation - if (empty($this->data['Attribute']['value'])) { - if (!empty($this->data['Attribute']['value1']) && !empty($this->data['Attribute']['value2'])) { - $this->data['Attribute']['value'] = "{$this->data['Attribute']['value1']}|{$this->data['Attribute']['value2']}"; - } else if (!empty($this->data['Attribute']['value1'])) { - $this->data['Attribute']['value'] = $this->data['Attribute']['value1']; + if (empty($attribute['value'])) { + if (!empty($attribute['value1']) && !empty($attribute['value2'])) { + $attribute['value'] = "{$attribute['value1']}|{$attribute['value2']}"; + } else if (!empty($attribute['value1'])) { + $attribute['value'] = $attribute['value1']; } } // remove leading and trailing blanks and refang value and - $this->data['Attribute']['value'] = ComplexTypeTool::refangValue(trim($this->data['Attribute']['value']), $type); + $attribute['value'] = ComplexTypeTool::refangValue(trim($attribute['value']), $type); // make some changes to the inserted value - $this->data['Attribute']['value'] = $this->modifyBeforeValidation($type, $this->data['Attribute']['value']); + $attribute['value'] = AttributeValidationTool::modifyBeforeValidation($type, $attribute['value']); // Run user defined regexp to attribute value - $result = $this->runRegexp($type, $this->data['Attribute']['value']); + $result = $this->runRegexp($type, $attribute['value']); if ($result === false) { $this->invalidate('value', 'This value is blocked by a regular expression in the import filters.'); } else { - $this->data['Attribute']['value'] = $result; + $attribute['value'] = $result; } - if (empty($this->data['Attribute']['comment'])) { - $this->data['Attribute']['comment'] = ""; + if (empty($attribute['comment'])) { + $attribute['comment'] = ""; } - // generate UUID if it doesn't exist - if (empty($this->data['Attribute']['uuid'])) { - $this->data['Attribute']['uuid'] = CakeText::uuid(); - } else { - $this->data['Attribute']['uuid'] = strtolower($this->data['Attribute']['uuid']); + if (!empty($attribute['uuid'])) { + $attribute['uuid'] = strtolower($attribute['uuid']); } // generate timestamp if it doesn't exist - if (empty($this->data['Attribute']['timestamp'])) { - $this->data['Attribute']['timestamp'] = time(); + if (empty($attribute['timestamp'])) { + $attribute['timestamp'] = time(); } // parse first_seen different formats - if (isset($this->data['Attribute']['first_seen'])) { - $this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen']; + if (isset($attribute['first_seen'])) { + $attribute['first_seen'] = $attribute['first_seen'] === '' ? null : $attribute['first_seen']; } // parse last_seen different formats - if (isset($this->data['Attribute']['last_seen'])) { - $this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen']; + if (isset($attribute['last_seen'])) { + $attribute['last_seen'] = $attribute['last_seen'] === '' ? null : $attribute['last_seen']; } // Set defaults for when some of the mandatory fields don't have defaults // These fields all have sane defaults either based on another field, or due to server settings - if (!isset($this->data['Attribute']['distribution'])) { - $this->data['Attribute']['distribution'] = $this->defaultDistribution(); + if (!isset($attribute['distribution'])) { + $attribute['distribution'] = $this->defaultDistribution(); + } + if ($attribute['distribution'] != 4) { + $attribute['sharing_group_id'] = 0; } // If category is not provided, assign default category by type - if (empty($this->data['Attribute']['category'])) { - $this->data['Attribute']['category'] = $this->typeDefinitions[$type]['default_category']; + if (empty($attribute['category'])) { + $attribute['category'] = $this->typeDefinitions[$type]['default_category']; } - if (!isset($this->data['Attribute']['to_ids'])) { - $this->data['Attribute']['to_ids'] = $this->typeDefinitions[$type]['to_ids']; - } - - if ($this->data['Attribute']['distribution'] != 4) { - $this->data['Attribute']['sharing_group_id'] = 0; + if (!isset($attribute['to_ids'])) { + $attribute['to_ids'] = $this->typeDefinitions[$type]['to_ids']; } // return true, otherwise the object cannot be saved return true; @@ -720,27 +741,23 @@ class Attribute extends AppModel public function validateAttributeValue($fields) { $value = $fields['value']; - return $this->runValidation($value, $this->data['Attribute']['type']); + return AttributeValidationTool::validate($this->data['Attribute']['type'], $value); } // check whether the variable is null or datetime public function datetimeOrNull($fields) { - $k = array_keys($fields)[0]; - $seen = $fields[$k]; - try { - new DateTime($seen); - $returnValue = true; - } catch (Exception $e) { - $returnValue = false; + $seen = array_values($fields)[0]; + if ($seen === null) { + return true; } - return $returnValue || is_null($seen); + return strtotime($seen) !== false; } public function validateLastSeenValue($fields) { $ls = $fields['last_seen']; - if (!isset($this->data['Attribute']['first_seen']) || is_null($ls)) { + if (!isset($this->data['Attribute']['first_seen']) || $ls === null) { return true; } $converted = $this->ISODatetimeToUTC(['Attribute' => [ @@ -753,668 +770,6 @@ class Attribute extends AppModel return true; } - const HEX_HAS_LENGTHS = array( - 'authentihash' => 64, - 'md5' => 32, - 'imphash' => 32, - 'telfhash' => 70, - 'sha1' => 40, - 'git-commit-id' => 40, - 'x509-fingerprint-md5' => 32, - 'x509-fingerprint-sha1' => 40, - 'x509-fingerprint-sha256' => 64, - 'ja3-fingerprint-md5' => 32, - 'jarm-fingerprint' => 62, - 'hassh-md5' => 32, - 'hasshserver-md5' => 32, - 'pehash' => 40, - 'sha224' => 56, - 'sha256' => 64, - 'sha384' => 96, - 'sha512' => 128, - 'sha512/224' => 56, - 'sha512/256' => 64, - 'sha3-224' => 56, - 'sha3-256' => 64, - 'sha3-384' => 96, - 'sha3-512' => 128 - ); - - public function runValidation($value, $type) - { - $returnValue = false; - // check data validation - switch ($type) { - case 'md5': - case 'imphash': - case 'telfhash': - case 'sha1': - case 'sha224': - case 'sha256': - case 'sha384': - case 'sha512': - case 'sha512/224': - case 'sha512/256': - case 'sha3-224': - case 'sha3-256': - case 'sha3-384': - case 'sha3-512': - case 'authentihash': - case 'ja3-fingerprint-md5': - case 'jarm-fingerprint': - case 'hassh-md5': - case 'hasshserver-md5': - case 'x509-fingerprint-md5': - case 'x509-fingerprint-sha256': - case 'x509-fingerprint-sha1': - case 'git-commit-id': - if ($this->isHashValid($type, $value)) { - return true; - } else { - $length = self::HEX_HAS_LENGTHS[$type]; - return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length); - } - case 'tlsh': - if (preg_match("#^t?[0-9a-f]{35,}$#i", $value)) { - $returnValue = true; - } else { - $returnValue = __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".'); - } - break; - case 'pehash': - if ($this->isHashValid('pehash', $value)) { - $returnValue = true; - } else { - $returnValue = __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); - } - break; - case 'ssdeep': - if (substr_count($value, ':') === 2) { - $parts = explode(':', $value); - if ($this->isPositiveInteger($parts[0])) { - return true; - } - } - return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash'); - case 'impfuzzy': - if (substr_count($value, ':') === 2) { - $parts = explode(':', $value); - if ($this->isPositiveInteger($parts[0])) { - $returnValue = true; - } - } - if (!$returnValue) { - $returnValue = __('Invalid impfuzzy format. The format has to be imports:hash:hash'); - } - break; - case 'cdhash': - if (preg_match("#^[0-9a-f]{40,}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)'); - } - break; - case 'http-method': - if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) { - $returnValue = true; - } else { - $returnValue = __('Unknown HTTP method.'); - } - break; - case 'filename|pehash': - // no newline - if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!'); - } - break; - case 'filename|md5': - case 'filename|sha1': - case 'filename|imphash': - case 'filename|sha224': - case 'filename|sha256': - case 'filename|sha384': - case 'filename|sha512': - case 'filename|sha512/224': - case 'filename|sha512/256': - case 'filename|sha3-224': - case 'filename|sha3-256': - case 'filename|sha3-384': - case 'filename|sha3-512': - case 'filename|authentihash': - $parts = explode('|', $type); - $length = self::HEX_HAS_LENGTHS[$parts[1]]; - if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length); - } - break; - case 'filename|ssdeep': - if (substr_count($value, '|') != 1 || !preg_match("#^.+\|.+$#", $value)) { - $returnValue = __('Invalid composite type. The format has to be %s.', $type); - } else { - $composite = explode('|', $value); - $value = $composite[1]; - if (substr_count($value, ':') == 2) { - $parts = explode(':', $value); - if ($this->isPositiveInteger($parts[0])) { - $returnValue = true; - } - } - if (!$returnValue) { - $returnValue = __('Invalid SSDeep hash (expected: blocksize:hash:hash).'); - } - } - break; - case 'filename|tlsh': - if (preg_match("#^.+\|[0-9a-f]{35,}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".'); - } - break; - case 'filename|vhash': - if (preg_match('#^.+\|.+$#', $value)) { - $returnValue = true; - } else { - $returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".'); - } - break; - case 'ip-src': - case 'ip-dst': - if (strpos($value, '/') !== false) { - $parts = explode("/", $value); - if (count($parts) !== 2 || !$this->isPositiveInteger($parts[1])) { - return __('Invalid CIDR notation value found.'); - } - - if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { - if ($parts[1] > 32) { - return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.'); - } - } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - if ($parts[1] > 128) { - return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.'); - } - } else { - return __('IP address has an invalid format.'); - } - } else if (!filter_var($value, FILTER_VALIDATE_IP)) { - return __('IP address has an invalid format.'); - } - return true; - - case 'port': - if (!$this->isPortValid($value)) { - $returnValue = __('Port numbers have to be integers between 1 and 65535.'); - } else { - $returnValue = true; - } - break; - case 'ip-dst|port': - case 'ip-src|port': - $parts = explode('|', $value); - if (!filter_var($parts[0], FILTER_VALIDATE_IP)) { - return __('IP address has an invalid format.'); - } - if (!$this->isPortValid($parts[1])) { - return __('Port numbers have to be integers between 1 and 65535.'); - } - return true; - case 'mac-address': - if (preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value)) { - $returnValue = true; - } - break; - case 'mac-eui-64': - if (preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value)) { - $returnValue = true; - } - break; - case 'hostname': - case 'domain': - if ($this->isDomainValid($value)) { - $returnValue = true; - } else { - $returnValue = __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type)); - } - break; - case 'hostname|port': - $parts = explode('|', $value); - if (!$this->isDomainValid($parts[0])) { - return __('Hostname has an invalid format.'); - } - if (!$this->isPortValid($parts[1])) { - return __('Port numbers have to be integers between 1 and 65535.'); - } - return true; - case 'domain|ip': - if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}\|.*$#i", $value)) { - $parts = explode('|', $value); - if (filter_var($parts[1], FILTER_VALIDATE_IP)) { - $returnValue = true; - } else { - $returnValue = __('IP address has an invalid format.'); - } - } else { - $returnValue = __('Domain name has an invalid format.'); - } - break; - case 'email': - case 'email-src': - case 'eppn': - case 'email-dst': - case 'target-email': - case 'whois-registrant-email': - case 'dns-soa-email': - case 'jabber-id': - // we don't use the native function to prevent issues with partial email addresses - if (preg_match("#^.*\@.*\..*$#i", $value)) { - $returnValue = true; - } else { - $returnValue = __('Email address has an invalid format. Please double check the value or select type "other".'); - } - break; - case 'vulnerability': - if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('Invalid format. Expected: CVE-xxxx-xxxx...'); - } - break; - case 'weakness': - if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) { - $returnValue = true; - } else { - $returnValue = __('Invalid format. Expected: CWE-x...'); - } - break; - case 'named pipe': - if (!preg_match("#\n#", $value)) { - $returnValue = true; - } - break; - case 'windows-service-name': - case 'windows-service-displayname': - if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) { - $returnValue = __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.'); - } else { - $returnValue = true; - } - break; - case 'mutex': - case 'process-state': - case 'snort': - case 'bro': - case 'zeek': - case 'community-id': - case 'anonymised': - case 'pattern-in-file': - case 'pattern-in-traffic': - case 'pattern-in-memory': - case 'filename-pattern': - case 'pgp-public-key': - case 'pgp-private-key': - case 'ssh-fingerprint': - case 'yara': - case 'stix2-pattern': - case 'sigma': - case 'gene': - case 'kusto-query': - case 'mime-type': - case 'identity-card-number': - case 'cookie': - case 'attachment': - case 'malware-sample': - case 'comment': - case 'text': - case 'other': - case 'cpe': - case 'email-attachment': - case 'email-body': - case 'email-header': - case 'first-name': - case 'middle-name': - case 'last-name': - case 'full-name': - $returnValue = true; - break; - case 'link': - // Moved to a native function whilst still enforcing the scheme as a requirement - if (filter_var($value, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED) && !preg_match("#\n#", $value)) { - $returnValue = true; - } - break; - case 'hex': - return ctype_xdigit($value); - case 'target-user': - case 'campaign-name': - case 'campaign-id': - case 'threat-actor': - case 'target-machine': - case 'target-org': - case 'target-location': - case 'target-external': - case 'email-subject': - case 'malware-type': - // TODO: review url/uri validation - case 'url': - case 'uri': - case 'user-agent': - case 'regkey': - case 'regkey|value': - case 'filename': - case 'pdb': - case 'windows-scheduled-task': - case 'whois-registrant-name': - case 'whois-registrant-org': - case 'whois-registrar': - case 'whois-creation-date': - case 'date-of-birth': - case 'place-of-birth': - case 'gender': - case 'passport-number': - case 'passport-country': - case 'passport-expiration': - case 'redress-number': - case 'nationality': - case 'visa-number': - case 'issue-date-of-the-visa': - case 'primary-residence': - case 'country-of-residence': - case 'special-service-request': - case 'frequent-flyer-number': - case 'travel-details': - case 'payment-details': - case 'place-port-of-original-embarkation': - case 'place-port-of-clearance': - case 'place-port-of-onward-foreign-destination': - case 'passenger-name-record-locator-number': - case 'email-dst-display-name': - case 'email-src-display-name': - case 'email-reply-to': - case 'email-x-mailer': - case 'email-mime-boundary': - case 'email-thread-index': - case 'email-message-id': - case 'github-username': - case 'github-repository': - case 'github-organisation': - case 'twitter-id': - case 'dkim': - case 'dkim-signature': - case 'favicon-mmh3': - case 'chrome-extension-id': - case 'mobile-application-id': - if (strpos($value, "\n") !== false) { - return __('Value must not contain new line character.'); - } - return true; - case 'datetime': - try { - new DateTime($value); - $returnValue = true; - } catch (Exception $e) { - $returnValue = __('Datetime has to be in the ISO 8601 format.'); - } - break; - case 'size-in-bytes': - case 'counter': - if ($this->isPositiveInteger($value)) { - return true; - } - return __('The value has to be a whole number greater or equal 0.'); - case 'targeted-threat-index': - if (!is_numeric($value) || $value < 0 || $value > 10) { - $returnValue = __('The value has to be a number between 0 and 10.'); - } else { - $returnValue = true; - } - break; - case 'iban': - case 'bic': - case 'btc': - case 'dash': - case 'xmr': - if (preg_match('/^[a-zA-Z0-9]+$/', $value)) { - $returnValue = true; - } - break; - case 'vhash': - if (preg_match('/^.+$/', $value)) { - $returnValue = true; - } - break; - case 'bin': - case 'cc-number': - case 'bank-account-nr': - case 'aba-rtn': - case 'prtn': - case 'phone-number': - case 'whois-registrant-phone': - if (is_numeric($value)) { - $returnValue = true; - } - break; - case 'cortex': - json_decode($value); - $returnValue = (json_last_error() == JSON_ERROR_NONE); - break; - case 'float': - return is_numeric($value); - case 'boolean': - if ($value == 1 || $value == 0) { - $returnValue = true; - } - break; - case 'AS': - if ($this->isPositiveInteger($value) && $value <= 4294967295) { - return true; - } - return __('AS number have to be integers between 1 and 4294967295'); - } - return $returnValue; - } - - // do some last second modifications before the validation - public function modifyBeforeValidation($type, $value) - { - $value = $this->handle4ByteUnicode($value); - switch ($type) { - case 'md5': - case 'sha1': - case 'sha224': - case 'sha256': - case 'sha384': - case 'sha512': - case 'sha512/224': - case 'sha512/256': - case 'sha3-224': - case 'sha3-256': - case 'sha3-384': - case 'sha3-512': - case 'ja3-fingerprint-md5': - case 'jarm-fingerprint': - case 'hassh-md5': - case 'hasshserver-md5': - case 'hostname': - case 'pehash': - case 'authentihash': - case 'vhash': - case 'imphash': - case 'telfhash': - case 'tlsh': - case 'anonymised': - case 'cdhash': - case 'email': - case 'email-src': - case 'email-dst': - case 'target-email': - case 'whois-registrant-email': - $value = strtolower($value); - break; - case 'domain': - $value = strtolower($value); - $value = trim($value, '.'); - // Domain is not valid, try to convert to punycode - if (!$this->isDomainValid($value) && function_exists('idn_to_ascii')) { - $punyCode = idn_to_ascii($value); - if ($punyCode !== false) { - $value = $punyCode; - } - } - break; - case 'domain|ip': - $value = strtolower($value); - $parts = explode('|', $value); - if (!isset($parts[1])) { - return $value; // not a composite - } - $parts[0] = trim($parts[0], '.'); - // Domain is not valid, try to convert to punycode - if (!$this->isDomainValid($parts[0]) && function_exists('idn_to_ascii')) { - $punyCode = idn_to_ascii($parts[0]); - if ($punyCode !== false) { - $parts[0] = $punyCode; - } - } - if (filter_var($parts[1], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - // convert IPv6 address to compressed format - $parts[1] = inet_ntop(inet_pton($parts[1])); - } - return "$parts[0]|$parts[1]"; - case 'filename|md5': - case 'filename|sha1': - case 'filename|imphash': - case 'filename|sha224': - case 'filename|sha256': - case 'filename|sha384': - case 'filename|sha512': - case 'filename|sha512/224': - case 'filename|sha512/256': - case 'filename|sha3-224': - case 'filename|sha3-256': - case 'filename|sha3-384': - case 'filename|sha3-512': - case 'filename|authentihash': - case 'filename|vhash': - case 'filename|pehash': - case 'filename|tlsh': - $pieces = explode('|', $value); - $value = $pieces[0] . '|' . strtolower($pieces[1]); - break; - case 'http-method': - case 'hex': - return strtoupper($value); - case 'vulnerability': - case 'weakness': - $value = str_replace('–', '-', $value); - return strtoupper($value); - case 'cc-number': - case 'bin': - $value = preg_replace('/[^0-9]+/', '', $value); - break; - case 'iban': - case 'bic': - $value = strtoupper($value); - $value = preg_replace('/[^0-9A-Z]+/', '', $value); - break; - case 'prtn': - case 'whois-registrant-phone': - case 'phone-number': - if (substr($value, 0, 2) == '00') { - $value = '+' . substr($value, 2); - } - $value = preg_replace('/\(0\)/', '', $value); - $value = preg_replace('/[^\+0-9]+/', '', $value); - break; - case 'x509-fingerprint-md5': - case 'x509-fingerprint-sha256': - case 'x509-fingerprint-sha1': - $value = str_replace(':', '', $value); - $value = strtolower($value); - break; - case 'ip-src': - case 'ip-dst': - if (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - // convert IPv6 address to compressed format - $value = inet_ntop(inet_pton($value)); - } - break; - case 'ip-dst|port': - case 'ip-src|port': - if (substr_count($value, ':') >= 2) { // (ipv6|port) - tokenize ip and port - if (strpos($value, '|')) { // 2001:db8::1|80 - $parts = explode('|', $value); - } elseif (strpos($value, '[') === 0 && strpos($value, ']') !== false) { // [2001:db8::1]:80 - $ipv6 = substr($value, 1, strpos($value, ']')-1); - $port = explode(':', substr($value, strpos($value, ']')))[1]; - $parts = array($ipv6, $port); - } elseif (strpos($value, '.')) { // 2001:db8::1.80 - $parts = explode('.', $value); - } elseif (strpos($value, ' port ')) { // 2001:db8::1 port 80 - $parts = explode(' port ', $value); - } elseif (strpos($value, 'p')) { // 2001:db8::1p80 - $parts = explode('p', $value); - } elseif (strpos($value, '#')) { // 2001:db8::1#80 - $parts = explode('#', $value); - } else { // 2001:db8::1:80 this one is ambiguous - $temp = explode(':', $value); - $parts = array(implode(':', array_slice($temp, 0, count($temp)-1)), end($temp)); - } - } elseif (strpos($value, ':')) { // (ipv4:port) - $parts = explode(':', $value); - } elseif (strpos($value, '|')) { // (ipv4|port) - $parts = explode('|', $value); - } else { - return $value; - } - if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - // convert IPv6 address to compressed format - $parts[0] = inet_ntop(inet_pton($parts[0])); - } - return $parts[0] . '|' . $parts[1]; - case 'mac-address': - case 'mac-eui-64': - $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value)); - $value = wordwrap($value, 2, ':', true); - break; - case 'hostname|port': - $value = strtolower($value); - return str_replace(':', '|', $value); - case 'boolean': - if ('true' == trim(strtolower($value))) { - $value = 1; - } - if ('false' == trim(strtolower($value))) { - $value = 0; - } - $value = ($value) ? '1' : '0'; - break; - case 'datetime': - try { - $value = (new DateTime($value))->setTimezone(new DateTimeZone('GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds - } catch (Exception $e) { - // silently skip. Rejection will be done in runValidation() - } - break; - case 'AS': - if (strtoupper(substr($value, 0, 2)) === 'AS') { - $value = substr($value, 2); // remove 'AS' - } - if (strpos($value, '.') !== false) { // maybe value is in asdot notation - $parts = explode('.', $value); - if ($this->isPositiveInteger($parts[0]) && $this->isPositiveInteger($parts[1])) { - return $parts[0] * 65536 + $parts[1]; - } - } - break; - } - return $value; - } - public function getCompositeTypes() { static $compositeTypes; @@ -1683,43 +1038,29 @@ class Attribute extends AppModel { // convert into utc and micro sec if (!empty($data[$alias]['first_seen'])) { - $d = new DateTime($data[$alias]['first_seen']); - $d->setTimezone(new DateTimeZone('GMT')); - $fs_sec = $d->format('U'); - $fs_micro = $d->format('u'); - $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); - $fs = $fs_sec . $fs_micro; - $data[$alias]['first_seen'] = $fs; + $d = new DateTime($data[$alias]['first_seen'], new DateTimeZone('GMT')); + $data[$alias]['first_seen'] = $d->format('Uu'); } if (!empty($data[$alias]['last_seen'])) { - $d = new DateTime($data[$alias]['last_seen']); - $d->setTimezone(new DateTimeZone('GMT')); - $ls_sec = $d->format('U'); - $ls_micro = $d->format('u'); - $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); - $ls = $ls_sec . $ls_micro; - $data[$alias]['last_seen'] = $ls; + $d = new DateTime($data[$alias]['last_seen'], new DateTimeZone('GMT')); + $data[$alias]['last_seen'] = $d->format('Uu'); } return $data; } + /** + * @param $data + * @param $alias + * @return array + * @deprecated + */ public function UTCToISODatetime($data, $alias) { if (!empty($data[$alias]['first_seen'])) { - $fs = $data[$alias]['first_seen']; - $fs_sec = intval($fs / 1000000); // $fs is in micro (10^6) - $fs_micro = $fs % 1000000; - $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); - $fs = $fs_sec . '.' . $fs_micro; - $data[$alias]['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP'); + $data[$alias]['first_seen'] = $this->microTimestampToIso($data[$alias]['first_seen']); } if (!empty($data[$alias]['last_seen'])) { - $ls = $data[$alias]['last_seen']; - $ls_sec = intval($ls / 1000000); // $ls is in micro (10^6) - $ls_micro = $ls % 1000000; - $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); - $ls = $ls_sec . '.' . $ls_micro; - $data[$alias]['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP'); + $data[$alias]['last_seen'] = $this->microTimestampToIso($data[$alias]['last_seen']); } return $data; } @@ -1886,6 +1227,7 @@ class Attribute extends AppModel if (empty($params['tags'])) { return $conditions; } + /** @var Tag $tag */ $tag = ClassRegistry::init('Tag'); $params['tags'] = $this->dissectArgs($params['tags']); foreach (array(0, 1, 2) as $tag_operator) { @@ -2340,13 +1682,13 @@ class Attribute extends AppModel // array 1 will have all of the non negated terms and array 2 all the negated terms public function dissectArgs($args) { + $result = array(0 => array(), 1 => array(), 2 => array()); if (empty($args)) { - return array(0 => array(), 1 => array(), 2 => array()); + return $result; } if (!is_array($args)) { $args = explode('&&', $args); } - $result = array(0 => array(), 1 => array(), 2 => array()); if (isset($args['OR']) || isset($args['NOT']) || isset($args['AND'])) { if (!empty($args['OR'])) { $result[0] = $args['OR']; @@ -2359,7 +1701,7 @@ class Attribute extends AppModel } } else { foreach ($args as $arg) { - if (substr($arg, 0, 1) == '!') { + if ($arg[0] === '!') { $result[1][] = substr($arg, 1); } else { $result[0][] = $arg; @@ -2573,15 +1915,18 @@ class Attribute extends AppModel return $conditions; } - /* + /** * Unlike the other fetchers, this one foregoes any ACL checks. * the objective is simple: Fetch the given attribute with all related objects needed for the ZMQ output, * standardising on this function for fetching the attribute to be passed to Attribute->save() + * @param int $id + * @returns array */ public function fetchAttribute($id) { $attribute = $this->find('first', array( 'recursive' => -1, + 'order' => [], 'conditions' => array('Attribute.id' => $id), 'contain' => array( 'Event' => array( @@ -2591,6 +1936,7 @@ class Attribute extends AppModel 'fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.uuid', 'Event.published', 'Event.analysis', 'Event.threat_level_id', 'Event.org_id', 'Event.orgc_id', 'Event.distribution', 'Event.sharing_group_id') ), 'AttributeTag' => array( + 'fields' => ['AttributeTag.tag_id'], 'Tag' => array('fields' => array('Tag.id', 'Tag.name', 'Tag.colour', 'Tag.exportable')) ), 'Object' @@ -2603,6 +1949,10 @@ class Attribute extends AppModel } } unset($attribute['AttributeTag']); + + if (empty($attribute['Object']['id'])) { + unset($attribute['Object']); + } } return $attribute; } @@ -2664,15 +2014,23 @@ class Attribute extends AppModel )); } - // Method that fetches all attributes for the various exports - // very flexible, it's basically a replacement for find, with the addition that it restricts access based on user - // options: - // fields - // contain - // conditions - // order - // group - public function fetchAttributes($user, $options = array(), &$continue = true) + /** + * Method that fetches all attributes for the various exports + * very flexible, it's basically a replacement for find, with the addition that it restricts access based on user + * options: + * - fields + * - contain + * - conditions + * - order + * - group + * + * @param array $user + * @param array $options + * @param int|false $result_count If false, count is not fetched + * @return array + * @throws Exception + */ + public function fetchAttributes(array $user, array $options = [], &$result_count = false) { $params = array( 'conditions' => $this->buildConditions($user), @@ -2681,7 +2039,7 @@ class Attribute extends AppModel 'Event' => array( 'fields' => array('id', 'info', 'org_id', 'orgc_id', 'uuid'), ), - 'AttributeTag', // tags are fetched separately, @see Attribute::__attachTagsToAttributes + 'AttributeTag', // tags are fetched separately, @see Attribute::attachTagsToAttributes 'Object' => array( 'fields' => array('id', 'distribution', 'sharing_group_id') ) @@ -2689,16 +2047,13 @@ class Attribute extends AppModel ); if (!empty($options['includeProposals'])) { - $this->bindModel( - array('hasMany' => array( - 'ShadowAttribute' => array( - 'className' => 'ShadowAttribute', - 'foreignKey' => 'old_id', - 'conditions' => array('ShadowAttribute.deleted' => 0) - ) - ) + $this->bindModel(['hasMany' => array( + 'ShadowAttribute' => array( + 'className' => 'ShadowAttribute', + 'foreignKey' => 'old_id', + 'conditions' => array('ShadowAttribute.deleted' => 0) ) - ); + )]); $params['contain']['ShadowAttribute'] = array('fields' => array( "id", "old_id", @@ -2747,24 +2102,21 @@ class Attribute extends AppModel if (isset($options['limit'])) { $params['limit'] = $options['limit']; } - if ( - !empty($options['allow_proposal_blocking']) && - Configure::read('MISP.proposals_block_attributes') - ) { + if (!empty($options['allow_proposal_blocking']) && Configure::read('MISP.proposals_block_attributes')) { $this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id')))); $proposalRestriction = array( - 'ShadowAttribute' => array( - 'conditions' => array( - 'AND' => array( - 'ShadowAttribute.deleted' => 0, - 'OR' => array( - 'ShadowAttribute.proposal_to_delete' => 1, - 'ShadowAttribute.to_ids' => 0 - ) - ) - ), - 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.value', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.to_ids') - ) + 'ShadowAttribute' => array( + 'conditions' => array( + 'AND' => array( + 'ShadowAttribute.deleted' => 0, + 'OR' => array( + 'ShadowAttribute.proposal_to_delete' => 1, + 'ShadowAttribute.to_ids' => 0 + ) + ) + ), + 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.value', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.to_ids') + ) ); $params['contain'] = array_merge($params['contain'], $proposalRestriction); } @@ -2777,12 +2129,10 @@ class Attribute extends AppModel if (empty($options['flatten'])) { $params['conditions']['AND'][] = array('Attribute.object_id' => 0); } - if (isset($options['order'])) { - $params['order'] = $options['order']; - } + $params['order'] = isset($options['order']) ? $options['order'] : []; if (!isset($options['withAttachments'])) { $options['withAttachments'] = false; - } else ($params['order'] = array()); + } if (!isset($options['enforceWarninglist'])) { $options['enforceWarninglist'] = false; } @@ -2806,7 +2156,7 @@ class Attribute extends AppModel } else { $options['includeDecayScore'] = true; } - //Add EventTags to attributes to take them into account when calculating decay score + // Add EventTags to attributes to take them into account when calculating decay score if ($options['includeDecayScore']) { $options['includeEventTags'] = true; } @@ -2821,10 +2171,6 @@ class Attribute extends AppModel if (isset($options['group'])) { $params['group'] = !empty($options['group']) ? $options['group'] : false; } - // Site admin can access even unpublished event attributes if `unpublishedprivate` option is enabled - if (!$user['Role']['perm_site_admin'] && Configure::read('MISP.unpublishedprivate')) { - $params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id'], 'Event.org_id' => $user['org_id'])); - } if (!empty($options['list'])) { if (!empty($options['event_ids'])) { return $this->find('column', [ @@ -2837,7 +2183,6 @@ class Attribute extends AppModel } else { return $this->find('list', array( 'conditions' => $params['conditions'], - 'recursive' => -1, 'contain' => array('Event', 'Object'), 'fields' => array('Attribute.event_id'), 'order' => false @@ -2848,122 +2193,118 @@ class Attribute extends AppModel if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) { $this->Warninglist = ClassRegistry::init('Warninglist'); } + // If no limit is provided, fetch attributes in bulk if (empty($params['limit'])) { $loopLimit = 50000; $loop = true; $params['limit'] = $loopLimit; - $params['page'] = 0; + $params['page'] = 1; } else { $loop = false; } - $attributes = array(); - if (!empty($options['includeEventTags'])) { - $eventTags = array(); - } - while ($continue) { - if ($loop) { - $params['page'] = $params['page'] + 1; - if (isset($results) && count($results) < $loopLimit) { - $continue = false; - continue; - } - } - $results = $this->find('all', $params); - if (!empty($options['includeContext']) && !empty($results)) { + // Do not fetch result count when `$result_count` is false + if ($result_count !== false) { + $find_params = $params; + unset($find_params['limit']); + $result_count = $this->find('count', $find_params); + if ($result_count === 0) { // skip early + return []; + } + } + + $eventTags = []; // tag cache + $attributes = []; + do { + $results = $this->find('all', $params); + if (empty($results)) { + break; + } + + if (!empty($options['includeContext'])) { $eventIds = []; foreach ($results as $result) { $eventIds[$result['Attribute']['event_id']] = true; // deduplicate } $eventsById = $this->__fetchEventsForAttributeContext($user, array_keys($eventIds), !empty($options['includeAllTags'])); - foreach ($results as &$result) { - $result['Event'] = $eventsById[$result['Attribute']['event_id']]; - } - unset($eventsById, $result); // unset result is important, because it is reference + unset($eventIds); } - $this->__attachTagsToAttributes($results, $options); + $this->attachTagsToAttributes($results, $options); + $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); - foreach ($results as $k => $result) { + foreach ($results as &$attribute) { + if (!empty($options['includeContext'])) { + $attribute['Event'] = $eventsById[$attribute['Attribute']['event_id']]; + } if (!empty($options['includeSightings'])) { - $temp = $result['Attribute']; - $temp['Event'] = $result['Event']; - $results[$k]['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']); + $temp = $attribute['Attribute']; + $temp['Event'] = $attribute['Event']; + $attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']); } if (!empty($options['includeCorrelations'])) { $attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment'); - $results[$k]['Attribute']['RelatedAttribute'] = ($this->getRelatedAttributes($user, $results[$k]['Attribute'], $attributeFields, true)); + $attribute['Attribute']['RelatedAttribute'] = $this->getRelatedAttributes($user, $attribute['Attribute'], $attributeFields, true); } - } - if (!$loop) { - if (!empty($params['limit']) && count($results) < $params['limit']) { - $continue = false; - } - $break = true; - } - // return false if we're paginating - if (isset($options['limit']) && empty($results)) { - return array(); - } - $results = array_values($results); - $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); - foreach ($results as $key => $attribute) { if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) { - unset($results[$key]); // Remove attribute that match any enabled warninglists continue; } if (!empty($options['includeEventTags'])) { - $results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options); + $attribute = $this->__attachEventTagsToAttributes($eventTags, $attribute, $options); } if ($options['includeWarninglistHits']) { - $results[$key]['Attribute'] = $this->Warninglist->checkForWarning($results[$key]['Attribute']); + $attribute['Attribute'] = $this->Warninglist->checkForWarning($attribute['Attribute']); } if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) { - $results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid']; + $attribute['Attribute']['event_uuid'] = $attribute['Event']['uuid']; } if ($proposals_block_attributes) { - $this->__blockAttributeViaProposal($results, $key); - } - if ($options['withAttachments']) { - if ($this->typeIsAttachment($attribute['Attribute']['type'])) { - $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']); - $results[$key]['Attribute']['data'] = $encodedFile; + if ($this->__blockAttributeViaProposal($attribute)) { + continue; } + unset($attribute['ShadowAttribute']); + } + if ($options['withAttachments'] && $this->typeIsAttachment($attribute['Attribute']['type'])) { + $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']); + $attribute['Attribute']['data'] = $encodedFile; } if ($options['includeDecayScore']) { $this->DecayingModel = ClassRegistry::init('DecayingModel'); $include_full_model = isset($options['includeFullModel']) && $options['includeFullModel'] ? 1 : 0; - if (empty($results[$key]['Attribute']['AttributeTag'])) { - $results[$key]['Attribute']['AttributeTag'] = isset($results[$key]['AttributeTag']) ? $results[$key]['AttributeTag'] : array(); - $results[$key]['Attribute']['EventTag'] = isset($results[$key]['EventTag']) ? $results[$key]['EventTag'] : array(); + if (empty($attribute['Attribute']['AttributeTag'])) { + $attribute['Attribute']['AttributeTag'] = isset($attribute['AttributeTag']) ? $attribute['AttributeTag'] : array(); + $attribute['Attribute']['EventTag'] = isset($attribute['EventTag']) ? $attribute['EventTag'] : array(); } - $results[$key]['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $results[$key]['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model); - unset($results[$key]['Attribute']['AttributeTag']); - unset($results[$key]['Attribute']['EventTag']); - if ($options['excludeDecayed'] && !empty($results[$key]['Attribute']['decay_score'])) { // filter out decayed attribute + $attribute['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $attribute['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model); + unset($attribute['Attribute']['AttributeTag']); + unset($attribute['Attribute']['EventTag']); + if ($options['excludeDecayed'] && !empty($attribute['Attribute']['decay_score'])) { // filter out decayed attribute $decayed_flag = true; - foreach ($results[$key]['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay + foreach ($attribute['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay $decayed_flag = $decayed_flag && $decayResult['decayed']; } if ($decayed_flag) { - unset($results[$key]); + continue; } } } - if (!empty($results[$key])) { - if (!empty($options['includeGalaxy'])) { - $massaged_attribute = $this->Event->massageTags($user, $results[$key], 'Attribute'); - $massaged_event = $this->Event->massageTags($user, $results[$key], 'Event'); - $massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']); - $results[$key] = $massaged_attribute; - } - $attributes[] = $results[$key]; + if (!empty($options['includeGalaxy'])) { + $massaged_attribute = $this->Event->massageTags($user, $attribute, 'Attribute'); + $massaged_event = $this->Event->massageTags($user, $attribute, 'Event'); + $massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']); + $attribute = $massaged_attribute; } + $attributes[] = $attribute; } - if (!empty($break)) { - break; + unset($attribute); + + if ($loop) { + if (count($results) < $loopLimit) { // we fetched less results than limit, so we can skip next query + break; + } + $params['page']++; } - } + } while ($loop); return $attributes; } @@ -3011,7 +2352,14 @@ class Attribute extends AppModel return $eventsById; } - private function __attachTagsToAttributes(array &$attributes, array $options) + /** + * Options: + * - includeAllTags - if true, include also exportable tags + * + * @param array $attributes + * @param array $options + */ + public function attachTagsToAttributes(array &$attributes, array $options) { $tagIdsToFetch = []; foreach ($attributes as $attribute) { @@ -3032,15 +2380,12 @@ class Attribute extends AppModel $conditions['Tag.exportable'] = 1; } - $tagsToModify = $this->AttributeTag->Tag->find('all', [ + $tags = $this->AttributeTag->Tag->find('all', [ 'conditions' => $conditions, 'fields' => ['id', 'name', 'colour', 'numerical_value'], 'recursive' => -1, ]); - $tags = []; - foreach ($tagsToModify as $tag) { - $tags[$tag['Tag']['id']] = $tag['Tag']; - } + $tags = array_column(array_column($tags, 'Tag'), null, 'id'); foreach ($attributes as $k => $attribute) { $tagCulled = false; @@ -3060,48 +2405,58 @@ class Attribute extends AppModel } } - private function __attachEventTagsToAttributes($eventTags, &$results, $key, $options) + /** + * @param array $eventTags + * @param array $attribute + * @param array $options + * @return array + */ + private function __attachEventTagsToAttributes(&$eventTags, $attribute, $options) { - if (!isset($eventTags[$results[$key]['Event']['id']])) { - $tagConditions = array('EventTag.event_id' => $results[$key]['Event']['id']); + $eventId = $attribute['Event']['id']; + if (!isset($eventTags[$eventId])) { + $tagConditions = array('EventTag.event_id' => $eventId); if (empty($options['includeAllTags'])) { $tagConditions['Tag.exportable'] = 1; } $temp = $this->Event->EventTag->find('all', array( 'recursive' => -1, 'contain' => array('Tag'), - 'conditions' => $tagConditions + 'conditions' => $tagConditions, )); - foreach ($temp as $tag) { - $tag['EventTag']['Tag'] = $tag['Tag']; - unset($tag['Tag']); - $eventTags[$results[$key]['Event']['id']][] = $tag; + if (empty($temp)) { + $eventTags[$eventId] = []; + } else { + foreach ($temp as $tag) { + $tag['EventTag']['Tag'] = $tag['Tag']; + unset($tag['Tag']); + $eventTags[$eventId][] = $tag['EventTag']; + } } } if (!empty($eventTags)) { - foreach ($eventTags[$results[$key]['Event']['id']] as $eventTag) { - $results[$key]['EventTag'][] = $eventTag['EventTag']; + foreach ($eventTags[$eventId] as $eventTag) { + $attribute['EventTag'][] = $eventTag; } } - return $results; + return $attribute; } - private function __blockAttributeViaProposal(&$attributes, $k) + private function __blockAttributeViaProposal($attribute) { - if (!empty($attributes[$k]['ShadowAttribute'])) { - foreach ($attributes[$k]['ShadowAttribute'] as $sa) { - if ($sa['value'] === $attributes[$k]['Attribute']['value'] && - $sa['type'] === $attributes[$k]['Attribute']['type'] && - $sa['category'] === $attributes[$k]['Attribute']['category'] && + if (!empty($attribute['ShadowAttribute'])) { + foreach ($attribute['ShadowAttribute'] as $sa) { + if ($sa['value'] === $attribute['Attribute']['value'] && + $sa['type'] === $attribute['Attribute']['type'] && + $sa['category'] === $attribute['Attribute']['category'] && ($sa['to_ids'] == 0 || $sa['to_ids'] == '') && - $attributes[$k]['Attribute']['to_ids'] == 1 + $attribute['Attribute']['to_ids'] == 1 ) { - unset($attributes[$k]); + return true; } } - } else { - unset($attributes[$k]['ShadowAttribute']); } + return false; } // Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive @@ -3157,7 +2512,7 @@ class Attribute extends AppModel public function isAdvancedExtractionAvailable() { try { - $types = $this->loadAttachmentTool()->checkAdvancedExtractionStatus($this->getPythonVersion()); + $types = $this->loadAttachmentTool()->checkAdvancedExtractionStatus(); } catch (Exception $e) { return false; } @@ -3279,51 +2634,6 @@ class Attribute extends AppModel return $attribute; } - private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event') - { - $conditionKey = $blocked ? array('NOT' => array('EventTag.tag_id' => $tag_id)) : array('EventTag.tag_id' => $tag_id); - $db = $this->getDataSource(); - $subQuery = $db->buildStatement( - array( - 'fields' => array($scope . 'Tag.' . $limitAttributeHitsTo . '_id'), - 'table' => strtolower($scope) . '_tags', - 'alias' => $scope . 'Tag', - 'limit' => null, - 'offset' => null, - 'joins' => array(), - 'conditions' => array( - $scope . 'Tag.tag_id' => $tag_id - ), - 'group' => array($scope . 'Tag.' . $limitAttributeHitsTo . '_id') - ), - $this - ); - $subQuery = ucfirst($limitAttributeHitsTo) . '.id IN (' . $subQuery . ') '; - $conditions = array( - $db->expression($subQuery)->value - ); - return $conditions; - } - - public function setTagConditions($tags, $conditions, $limitAttributeHitsTo = 'event') - { - $args = $this->dissectArgs($tags); - $tagArray = $this->AttributeTag->Tag->fetchTagIdsFromFilter($args[0], $args[1]); - $temp = array(); - if (!empty($tagArray[0])) { - $temp['OR'][] = $this->__createTagSubQuery($tagArray[0]); - $temp['OR'][] = $this->__createTagSubQuery($tagArray[0], false, 'Attribute', $limitAttributeHitsTo); - } - if (!empty($tagArray[1])) { - $temp['AND']['NOT'] = $this->__createTagSubQuery($tagArray[1], true); - if ($limitAttributeHitsTo == 'attribute') { - $temp['AND']['NOT'] = $this->__createTagSubQuery($tagArray[1], true, 'Attribute', $limitAttributeHitsTo); - } - } - $conditions['AND'][] = $temp; - return $conditions; - } - public function setTimestampConditions($timestamp, $conditions, $scope = 'Event.timestamp', $returnRaw = false) { if (is_array($timestamp)) { @@ -3372,16 +2682,6 @@ class Attribute extends AppModel return $conditions; } - public function setToIDSConditions($to_ids, $conditions) - { - if ($to_ids === 'exclude') { - $conditions['AND'][] = array('Attribute.to_ids' => 0); - } else { - $conditions['AND'][] = array('Attribute.to_ids' => 1); - } - return $conditions; - } - /** * Get list of all CIDR for correlation. * @return array @@ -3526,7 +2826,7 @@ class Attribute extends AppModel public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile) { try { - $result = $this->loadAttachmentTool()->advancedExtraction($this->getPythonVersion(), $tmpfile->path); + $result = $this->loadAttachmentTool()->advancedExtraction($tmpfile->path); } catch (Exception $e) { $this->logException("Could not finish advanced extraction", $e); return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile); @@ -3916,7 +3216,7 @@ class Attribute extends AppModel } App::uses($this->validFormats[$returnFormat][1], 'Export'); $exportTool = new $this->validFormats[$returnFormat][1](); - if (!empty($exportTool->use_default_filters)) { + if (method_exists($exportTool, 'setDefaultFilters')) { $exportTool->setDefaultFilters($filters); } if (empty($exportTool->non_restrictive_export)) { @@ -4045,7 +3345,7 @@ class Attribute extends AppModel * @param TmpFileTool $tmpfile * @param object $exportTool * @param array $exportToolParams - * @return int Number of attributes + * @return int Number of all attributes that matches given conditions * @throws Exception */ private function __iteratedFetch(array $user, array $params, $loop, TmpFileTool $tmpfile, $exportTool, array $exportToolParams) @@ -4053,9 +3353,10 @@ class Attribute extends AppModel $this->Allowedlist = ClassRegistry::init('Allowedlist'); $separator = $exportTool->separator($exportToolParams); $elementCounter = 0; - $continue = true; do { - $results = $this->fetchAttributes($user, $params, $continue); + $results = $this->fetchAttributes($user, $params, $elementCounter); + $totalCount = $elementCounter; + $elementCounter = false; // do not call `count` again if (empty($results)) { break; // nothing found, skip rest } @@ -4064,17 +3365,19 @@ class Attribute extends AppModel $results = $this->Sightingdb->attachToAttributes($results, $user); } $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true); - $elementCounter += count($results); foreach ($results as $attribute) { $handlerResult = $exportTool->handler($attribute, $exportToolParams); if ($handlerResult !== '') { $tmpfile->writeWithSeparator($handlerResult, $separator); } } + if ($loop && count($results) < $params['limit']) { + break; // do not continue if we received less results than limit + } $params['page'] += 1; - } while ($loop && $continue); + } while ($loop); - return $elementCounter; + return $totalCount; } public function set_filter_uuid(&$params, $conditions, $options) @@ -4125,47 +3428,6 @@ class Attribute extends AppModel } } - /** - * @param string $value - * @return bool - */ - private function isDomainValid($value) - { - return preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value) === 1; - } - - /** - * @param string $value - * @return bool - */ - private function isPortValid($value) - { - return $this->isPositiveInteger($value) && $value >= 1 && $value <= 65535; - } - - /** - * @param string $type - * @param string $value - * @return bool - */ - private function isHashValid($type, $value) - { - if (!isset(self::HEX_HAS_LENGTHS[$type])) { - throw new InvalidArgumentException("Invalid hash type '$type'."); - } - return strlen($value) === self::HEX_HAS_LENGTHS[$type] && ctype_xdigit($value); - } - - /** - * Returns true if input value is positive integer or zero. - * @param int|string $value - * @return bool - */ - private function isPositiveInteger($value) - { - return (is_int($value) && $value >= 0) || ctype_digit($value); - } - public function typeToCategoryMapping() { $typeCategoryMapping = array(); diff --git a/app/Model/AttributeTag.php b/app/Model/AttributeTag.php index 63f8f0a9c..4f88bfda1 100644 --- a/app/Model/AttributeTag.php +++ b/app/Model/AttributeTag.php @@ -342,42 +342,40 @@ class AttributeTag extends AppModel return $allClusters; } - public function extractAttributeTagsNameFromEvent(&$event, $to_extract='both') + /** + * @param array $event + * @return array|array[] + */ + public function extractAttributeTagsNameFromEvent(array $event) { - $attribute_tags_name = array('tags' => array(), 'clusters' => array()); - foreach ($event['Attribute'] as $i => $attribute) { - if ($to_extract == 'tags' || $to_extract == 'both') { - foreach ($attribute['AttributeTag'] as $tag) { - $attribute_tags_name['tags'][] = $tag['Tag']['name']; - } + $extractedTags = []; + $extractedClusters = []; + + foreach ($event['Attribute'] as $attribute) { + foreach ($attribute['AttributeTag'] as $tag) { + $extractedTags[$tag['Tag']['id']] = $tag['Tag']['name']; } - if ($to_extract == 'clusters' || $to_extract == 'both') { - foreach ($attribute['Galaxy'] as $galaxy) { - foreach ($galaxy['GalaxyCluster'] as $cluster) { - $attribute_tags_name['clusters'][] = $cluster['tag_name']; - } + foreach ($attribute['Galaxy'] as $galaxy) { + foreach ($galaxy['GalaxyCluster'] as $cluster) { + $extractedClusters[$cluster['tag_id']] = $cluster['tag_name']; } } } - foreach ($event['Object'] as $i => $object) { + foreach ($event['Object'] as $object) { if (!empty($object['Attribute'])) { - foreach ($object['Attribute'] as $j => $object_attribute) { - if ($to_extract == 'tags' || $to_extract == 'both') { - foreach ($object_attribute['AttributeTag'] as $tag) { - $attribute_tags_name['tags'][] = $tag['Tag']['name']; - } + foreach ($object['Attribute'] as $object_attribute) { + foreach ($object_attribute['AttributeTag'] as $tag) { + $extractedTags[$tag['Tag']['id']] = $tag['Tag']['name']; } - if ($to_extract == 'clusters' || $to_extract == 'both') { - foreach ($object_attribute['Galaxy'] as $galaxy) { - foreach ($galaxy['GalaxyCluster'] as $cluster) { - $attribute_tags_name['clusters'][] = $cluster['tag_name']; - } + foreach ($object_attribute['Galaxy'] as $galaxy) { + foreach ($galaxy['GalaxyCluster'] as $cluster) { + $extractedClusters[$cluster['tag_id']] = $cluster['tag_name']; } } } } } - $attribute_tags_name['tags'] = array_diff_key($attribute_tags_name['tags'], $attribute_tags_name['clusters']); // de-dup if needed. - return $attribute_tags_name; + $extractedTags = array_diff_key($extractedTags, $extractedClusters); // de-dup if needed. + return ['tags' => $extractedTags, 'clusters' => $extractedClusters]; } } diff --git a/app/Model/AuditLog.php b/app/Model/AuditLog.php index 5c4b2d4f0..a1415c20f 100644 --- a/app/Model/AuditLog.php +++ b/app/Model/AuditLog.php @@ -41,6 +41,15 @@ class AuditLog extends AppModel /** @var bool */ private $compressionEnabled; + /** @var bool */ + private $pubToZmq; + + /** @var bool */ + private $elasticLogging; + + /** @var bool */ + private $logClientIp; + /** * Null when not defined, false when not enabled * @var Syslog|null|false @@ -72,19 +81,22 @@ class AuditLog extends AppModel { parent::__construct($id, $table, $ds); $this->compressionEnabled = Configure::read('MISP.log_new_audit_compress') && function_exists('brotli_compress'); + $this->pubToZmq = $this->pubToZmq('audit'); + $this->elasticLogging = Configure::read('Plugin.ElasticSearch_logging_enable'); + $this->logClientIp = Configure::read('MISP.log_client_ip'); } public function afterFind($results, $primary = false) { - foreach ($results as $key => $result) { + foreach ($results as &$result) { if (isset($result['AuditLog']['ip'])) { - $results[$key]['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']); + $result['AuditLog']['ip'] = inet_ntop($result['AuditLog']['ip']); } if (isset($result['AuditLog']['change']) && $result['AuditLog']['change']) { - $results[$key]['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']); + $result['AuditLog']['change'] = $this->decodeChange($result['AuditLog']['change']); } if (isset($result['AuditLog']['action']) && isset($result['AuditLog']['model']) && isset($result['AuditLog']['model_id'])) { - $results[$key]['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']); + $result['AuditLog']['title'] = $this->generateUserFriendlyTitle($result['AuditLog']); } } return $results; @@ -119,13 +131,17 @@ class AuditLog extends AppModel if (in_array($auditLog['model'], ['Attribute', 'Object', 'ShadowAttribute'], true)) { $modelName = $auditLog['model'] === 'ShadowAttribute' ? 'Proposal' : $auditLog['model']; $title = __('%s from Event #%s', $modelName, $auditLog['event_id']); - } else { - $title = "{$auditLog['model']} #{$auditLog['model_id']}"; } + if (isset($auditLog['model_title']) && $auditLog['model_title']) { - $title .= ": {$auditLog['model_title']}"; + if (isset($title)) { + $title .= ": {$auditLog['model_title']}"; + return $title; + } else { + return $auditLog['model_title']; + } } - return $title; + return ''; } /** @@ -160,51 +176,52 @@ class AuditLog extends AppModel public function beforeSave($options = array()) { - if (!isset($this->data['AuditLog']['ip']) && Configure::read('MISP.log_client_ip')) { + $auditLog = &$this->data['AuditLog']; + if (!isset($auditLog['ip']) && $this->logClientIp) { $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR'; if (isset($_SERVER[$ipHeader])) { - $this->data['AuditLog']['ip'] = inet_pton($_SERVER[$ipHeader]); + $auditLog['ip'] = inet_pton($_SERVER[$ipHeader]); // convert to binary form } } - if (!isset($this->data['AuditLog']['user_id'])) { - $this->data['AuditLog']['user_id'] = $this->userInfo()['id']; + if (!isset($auditLog['user_id'])) { + $auditLog['user_id'] = $this->userInfo()['id']; } - if (!isset($this->data['AuditLog']['org_id'])) { - $this->data['AuditLog']['org_id'] = $this->userInfo()['org_id']; + if (!isset($auditLog['org_id'])) { + $auditLog['org_id'] = $this->userInfo()['org_id']; } - if (!isset($this->data['AuditLog']['request_type'])) { - $this->data['AuditLog']['request_type'] = $this->userInfo()['request_type']; + if (!isset($auditLog['request_type'])) { + $auditLog['request_type'] = $this->userInfo()['request_type']; } - if (!isset($this->data['AuditLog']['authkey_id'])) { - $this->data['AuditLog']['authkey_id'] = $this->userInfo()['authkey_id']; + if (!isset($auditLog['authkey_id'])) { + $auditLog['authkey_id'] = $this->userInfo()['authkey_id']; } - if (!isset($this->data['AuditLog']['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) { - $this->data['AuditLog']['request_id'] = $_SERVER['HTTP_X_REQUEST_ID']; + if (!isset($auditLog['request_id'] ) && isset($_SERVER['HTTP_X_REQUEST_ID'])) { + $auditLog['request_id'] = $_SERVER['HTTP_X_REQUEST_ID']; } // Truncate request_id - if (isset($this->data['AuditLog']['request_id']) && strlen($this->data['AuditLog']['request_id']) > 255) { - $this->data['AuditLog']['request_id'] = substr($this->data['AuditLog']['request_id'], 0, 255); + if (isset($auditLog['request_id']) && strlen($auditLog['request_id']) > 255) { + $auditLog['request_id'] = substr($auditLog['request_id'], 0, 255); } // Truncate model title - if (isset($this->data['AuditLog']['model_title']) && mb_strlen($this->data['AuditLog']['model_title']) > 255) { - $this->data['AuditLog']['model_title'] = mb_substr($this->data['AuditLog']['model_title'], 0, 252) . '...'; + if (isset($auditLog['model_title']) && mb_strlen($auditLog['model_title']) > 255) { + $auditLog['model_title'] = mb_substr($auditLog['model_title'], 0, 252) . '...'; } $this->logData($this->data); - if (isset($this->data['AuditLog']['change'])) { - $change = json_encode($this->data['AuditLog']['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if (isset($auditLog['change'])) { + $change = json_encode($auditLog['change'], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); if ($this->compressionEnabled && strlen($change) >= self::BROTLI_MIN_LENGTH) { $change = self::BROTLI_HEADER . brotli_compress($change, 4, BROTLI_TEXT); } - $this->data['AuditLog']['change'] = $change; + $auditLog['change'] = $change; } } @@ -214,14 +231,14 @@ class AuditLog extends AppModel */ private function logData(array $data) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_audit_notifications_enable')) { + if ($this->pubToZmq) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->publish($data, 'audit', 'log'); } $this->publishKafkaNotification('audit', $data, 'log'); - if (Configure::read('Plugin.ElasticSearch_logging_enable')) { + if ($this->elasticLogging) { // send off our logs to distributed /dev/null $logIndex = Configure::read("Plugin.ElasticSearch_log_index"); $elasticSearchClient = $this->getElasticSearchTool(); @@ -344,6 +361,7 @@ class AuditLog extends AppModel 'conditions' => $conditions, 'group' => ['Date'], 'order' => ['Date'], + 'callbacks' => false, ]); } elseif ($dataSource === 'Database/Postgres') { if (!empty($conditions['org_id'])) { diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index 6e14c97d4..b05b6f2ee 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -3,9 +3,6 @@ App::uses('AuditLog', 'Model'); class AuditLogBehavior extends ModelBehavior { - /** @var array */ - private $config; - /** @var array|null */ private $old; @@ -42,6 +39,7 @@ class AuditLogBehavior extends ModelBehavior 'TagCollection' => 'name', 'Taxonomy' => 'namespace', 'Organisation' => 'name', + 'SystemSetting' => 'setting', 'AdminSetting' => 'setting', 'UserSetting' => 'setting', 'Galaxy' => 'name', @@ -57,7 +55,6 @@ class AuditLogBehavior extends ModelBehavior public function setup(Model $model, $config = []) { - $this->config = $config; // Generate model info for attribute and proposals $attributeInfo = function (array $new, array $old) { $category = isset($new['category']) ? $new['category'] : $old['category']; @@ -81,11 +78,33 @@ class AuditLogBehavior extends ModelBehavior if (!$this->enabled) { return true; } + + // Do not fetch old version when just few fields will be fetched + $fieldToFetch = []; + if (!empty($options['fieldList'])) { + foreach ($options['fieldList'] as $field) { + if (!isset($this->skipFields[$field])) { + $fieldToFetch[] = $field; + } + } + + // For objects, that are assigned to event, we need to know event ID. So if data to save doesn't contain + // that ID, we need to fetch it from database. + if (isset($model->schema()['event_id']) && empty($model->data[$model->alias]['event_id']) && !in_array('event_id', $fieldToFetch, true)) { + $fieldToFetch[] = 'event_id'; + } + + if (empty($fieldToFetch)) { + $this->old = null; + return true; + } + } if ($model->id) { $this->old = $model->find('first', [ 'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id], 'recursive' => -1, 'callbacks' => false, + 'fields' => $fieldToFetch, ]); } else { $this->old = null; @@ -99,36 +118,31 @@ class AuditLogBehavior extends ModelBehavior return; } - if ($model->id) { - $id = $model->id; - } else if ($model->insertId) { - $id = $model->insertId; - } else { - $id = null; - } + $id = $model->id ?: 0; + $data = $model->data[$model->alias]; if ($created) { $action = AuditLog::ACTION_ADD; } else { $action = AuditLog::ACTION_EDIT; - if (isset($model->data[$model->alias]['deleted'])) { - if ($model->data[$model->alias]['deleted']) { + if (isset($data['deleted'])) { + if ($data['deleted']) { $action = AuditLog::ACTION_SOFT_DELETE; - } else if (!$model->data[$model->alias]['deleted'] && $this->old[$model->alias]['deleted']) { + } else if (isset($this->old[$model->alias]['deleted']) && $this->old[$model->alias]['deleted']) { $action = AuditLog::ACTION_UNDELETE; } } } - $changedFields = $this->changedFields($model, isset($options['fieldList']) ? $options['fieldList'] : null); + $changedFields = $this->changedFields($model, $options['fieldList']); if (empty($changedFields)) { return; } if ($model->name === 'Event') { $eventId = $id; - } else if (isset($model->data[$model->alias]['event_id'])) { - $eventId = $model->data[$model->alias]['event_id']; + } else if (isset($data['event_id'])) { + $eventId = $data['event_id']; } else if (isset($this->old[$model->alias]['event_id'])) { $eventId = $this->old[$model->alias]['event_id']; } else { @@ -139,9 +153,9 @@ class AuditLogBehavior extends ModelBehavior if (isset($this->modelInfo[$model->name])) { $modelTitleField = $this->modelInfo[$model->name]; if (is_callable($modelTitleField)) { - $modelTitle = $modelTitleField($model->data[$model->alias], isset($this->old[$model->alias]) ? $this->old[$model->alias] : []); - } else if (isset($model->data[$model->alias][$modelTitleField])) { - $modelTitle = $model->data[$model->alias][$modelTitleField]; + $modelTitle = $modelTitleField($data, isset($this->old[$model->alias]) ? $this->old[$model->alias] : []); + } else if (isset($data[$modelTitleField])) { + $modelTitle = $data[$modelTitleField]; } else if ($this->old[$model->alias][$modelTitleField]) { $modelTitle = $this->old[$model->alias][$modelTitleField]; } @@ -150,9 +164,9 @@ class AuditLogBehavior extends ModelBehavior $modelName = $model->name === 'MispObject' ? 'Object' : $model->name; if ($modelName === 'AttributeTag' || $modelName === 'EventTag') { - $isLocal = isset($model->data[$model->alias]['local']) ? $model->data[$model->alias]['local'] : false; + $isLocal = isset($data['local']) ? $data['local'] : false; $action = $isLocal ? AuditLog::ACTION_TAG_LOCAL : AuditLog::ACTION_TAG; - $tagInfo = $this->getTagInfo($model, $model->data[$model->alias]['tag_id']); + $tagInfo = $this->getTagInfo($model, $data['tag_id']); if ($tagInfo) { $modelTitle = $tagInfo['tag_name']; if ($tagInfo['is_galaxy']) { @@ -162,26 +176,26 @@ class AuditLogBehavior extends ModelBehavior } } } - $id = $modelName === 'AttributeTag' ? $model->data[$model->alias]['attribute_id'] : $model->data[$model->alias]['event_id']; + $id = $modelName === 'AttributeTag' ? $data['attribute_id'] : $data['event_id']; $modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event'; - } - - if ($modelName === 'Event') { + } else if ($modelName === 'Event') { if (isset($changedFields['published'][1]) && $changedFields['published'][1]) { $action = AuditLog::ACTION_PUBLISH; } else if (isset($changedFields['sighting_timestamp'][1]) && $changedFields['sighting_timestamp'][1]) { $action = AuditLog::ACTION_PUBLISH_SIGHTINGS; } + } else if ($modelName === 'SystemSetting') { + $id = 0; } - $this->auditLog()->insert([ + $this->auditLog()->insert(['AuditLog' => [ 'action' => $action, 'model' => $modelName, 'model_id' => $id, 'model_title' => $modelTitle, 'event_id' => $eventId, 'change' => $changedFields, - ]); + ]]); } public function beforeDelete(Model $model, $cascade = true) @@ -240,16 +254,18 @@ class AuditLogBehavior extends ModelBehavior } $id = $modelName === 'AttributeTag' ? $model->data[$model->alias]['attribute_id'] : $model->data[$model->alias]['event_id']; $modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event'; + } else if ($modelName === 'SystemSetting') { + $id = 0; } - $this->auditLog()->insert([ + $this->auditLog()->insert(['AuditLog' => [ 'action' => $action, 'model' => $modelName, 'model_id' => $id, 'model_title' => $modelTitle, 'event_id' => $eventId, 'change' => $this->changedFields($model), - ]); + ]]); } /** @@ -263,6 +279,7 @@ class AuditLogBehavior extends ModelBehavior 'conditions' => ['Tag.id' => $tagId], 'recursive' => -1, 'fields' => ['Tag.name', 'Tag.is_galaxy'], + 'callbacks' => false, // disable Tag::afterFind callback ]); if (empty($tag)) { return null; @@ -299,6 +316,7 @@ class AuditLogBehavior extends ModelBehavior { $dbFields = $model->schema(); $changedFields = []; + $hasPrimaryField = isset($model->data[$model->alias][$model->primaryKey]); foreach ($model->data[$model->alias] as $key => $value) { if (isset($this->skipFields[$key])) { continue; @@ -310,7 +328,7 @@ class AuditLogBehavior extends ModelBehavior continue; } - if (isset($model->data[$model->alias][$model->primaryKey]) && isset($this->old[$model->alias][$key])) { + if ($hasPrimaryField && isset($this->old[$model->alias][$key])) { $old = $this->old[$model->alias][$key]; } else { $old = null; @@ -335,7 +353,7 @@ class AuditLogBehavior extends ModelBehavior continue; } - if ($key === 'password' || $key === 'authkey') { + if ($key === 'password' || $key === 'authkey' || ($key === 'value' && $model->name === 'SystemSetting' && SystemSetting::isSensitive($model->data[$model->alias]['setting']))) { $value = '*****'; if ($old !== null) { $old = $value; diff --git a/app/Model/Cerebrate.php b/app/Model/Cerebrate.php index 125bd450d..f61be6d4d 100644 --- a/app/Model/Cerebrate.php +++ b/app/Model/Cerebrate.php @@ -1,6 +1,6 @@ array( - 'className' => 'Organisation', - 'foreignKey' => 'org_id' + 'className' => 'Organisation', + 'foreignKey' => 'org_id' ) ); - public function queryInstance($options) { + public function beforeSave($options = array()) + { + $cerebrate = &$this->data['Server']; + // Encrypt authkey if plain key provided and encryption is enabled + if (!empty($cerebrate['authkey']) && strlen($cerebrate['authkey']) === 40) { + $cerebrate['authkey'] = EncryptedValue::encryptIfEnabled($cerebrate['authkey']); + } + return true; + } + + public function queryInstance($options) + { $url = $options['cerebrate']['Cerebrate']['url'] . $options['path']; $url_params = []; @@ -413,4 +424,37 @@ class Cerebrate extends AppModel } return __('The retrieved data isn\'t a valid sharing group.'); } + + /** + * @param string|null $old Old (or current) encryption key. + * @param string|null $new New encryption key. If empty, encrypted values will be decrypted. + * @throws Exception + */ + public function reencryptAuthKeys($old, $new) + { + $cerebrates = $this->find('list', [ + 'fields' => ['Cerebrate.id', 'Cerebrate.authkey'], + ]); + $toSave = []; + foreach ($cerebrates as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + $authkey = BetterSecurity::decrypt(substr($authkey, 2), $old); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for Cerebrate #$id", 0, $e); + } + } + if (!empty($new)) { + $authkey = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($authkey, $new); + } + $toSave[] = ['Cerebrate' => [ + 'id' => $id, + 'authkey' => $authkey, + ]]; + } + if (empty($toSave)) { + return true; + } + return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); + } } diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 7ae14ff15..391a31d82 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -43,29 +43,29 @@ class Correlation extends AppModel public function correlateValueRouter($value) { if (Configure::read('MISP.background_jobs')) { - if (empty($this->Job)) { - $this->Job = ClassRegistry::init('Job'); - } - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'correlateValue', - 'job_input' => $value, - 'status' => 0, - 'retries' => 0, - 'org_id' => 0, - 'org' => 0, - 'message' => 'Recorrelating', + + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'correlateValue', + $value, + 'Recorrelating' ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'EventShell', - ['correlateValue', $value, $jobId], - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'correlateValue', + $value, + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); + return true; } else { return $this->correlateValue($value); @@ -590,29 +590,27 @@ class Correlation extends AppModel public function generateTopCorrelationsRouter() { if (Configure::read('MISP.background_jobs')) { - if (empty($this->Job)) { - $this->Job = ClassRegistry::init('Job'); - } - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generateTopCorrelations', - 'job_input' => '', - 'status' => 0, - 'retries' => 0, - 'org_id' => 0, - 'org' => 0, - 'message' => 'Starting generation of top correlations.', + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generateTopCorrelations', + '', + 'Starting generation of top correlations.' ); - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'EventShell', - ['generateTopCorrelations', $jobId], - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'generateTopCorrelations', + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); + return $jobId; } else { return $this->generateTopCorrelations(); diff --git a/app/Model/CorrelationExclusion.php b/app/Model/CorrelationExclusion.php index 3fcb54575..3a819da1c 100644 --- a/app/Model/CorrelationExclusion.php +++ b/app/Model/CorrelationExclusion.php @@ -67,27 +67,26 @@ class CorrelationExclusion extends AppModel public function cleanRouter($user) { if (Configure::read('MISP.background_jobs')) { - $this->Job = ClassRegistry::init('Job'); - $this->Job->create(); - $data = [ - 'worker' => 'default', - 'job_type' => 'clean_correlation_exclusions', - 'job_input' => '', - 'status' => 0, - 'retries' => 0, - 'org' => $user['Organisation']['name'], - 'message' => __('Cleaning up excluded correlations.'), - ]; - $this->Job->save($data); - $jobId = $this->Job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - ['cleanExcludedCorrelations', $jobId], - true + /** @var Job $job */ + $job = ClassRegistry::init('Job'); + $jobId = $job->createJob( + $user, + Job::WORKER_DEFAULT, + 'clean_correlation_exclusions', + '', + __('Cleaning up excluded correlations.') + ); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'cleanExcludedCorrelations', + $jobId + ], + true, + $jobId ); - $this->Job->saveField('process_id', $process_id); - $message = __('Cleanup queued for background execution.'); } else { $this->clean(); } diff --git a/app/Model/Event.php b/app/Model/Event.php index 07f74a4a1..923a4dc40 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1,10 +1,10 @@ array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - 'numeric' => array( - 'rule' => array('numeric'), - ), + 'rule' => 'numeric', + 'required' => true, + 'allowEmpty' => false, ), 'orgc_id' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty'), - ), - 'numeric' => array( - 'rule' => array('numeric'), - ), + 'rule' => 'numeric', + 'required' => true, + 'allowEmpty' => false, ), 'date' => array( 'date' => array( @@ -132,22 +127,21 @@ class Event extends AppModel 'required' => true, //'last' => false, // Stop validation after this rule //'on' => 'create', // Limit validation to 'create' or 'update' operations - ) + ) ), 'sharing_group_id' => array( 'rule' => array('sharingGroupRequired'), - 'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.', - //'required' => true, - //'allowEmpty' => true + 'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.', + //'required' => true, + //'allowEmpty' => true ), - 'analysis' => array( 'rule' => array('inList', array('0', '1', '2')), - 'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)', - //'allowEmpty' => false, - 'required' => true, - //'last' => false, // Stop validation after this rule - //'on' => 'create', // Limit validation to 'create' or 'update' operations + 'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)', + //'allowEmpty' => false, + 'required' => true, + //'last' => false, // Stop validation after this rule + //'on' => 'create', // Limit validation to 'create' or 'update' operations ), 'info' => array( 'valueNotEmpty' => array( @@ -407,7 +401,7 @@ class Event extends AppModel } if (Configure::read('Plugin.Kafka_enable')) { $kafkaEventTopic = Configure::read('Plugin.Kafka_event_notifications_topic'); - if(Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) { + if (Configure::read('Plugin.Kafka_event_notifications_enable') && !empty($kafkaEventTopic)) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaEventTopic, array('Event' => $this->data['Event']), 'delete'); } @@ -430,9 +424,6 @@ class Event extends AppModel } } - // delete all of the event->tag combinations that involve the deleted event - $this->EventTag->deleteAll(array('event_id' => $this->id)); - try { $this->loadAttachmentTool()->deleteAll($this->id); } catch (Exception $e) { @@ -443,91 +434,103 @@ class Event extends AppModel public function beforeValidate($options = array()) { + $event = &$this->data['Event']; // analysis - setting correct vars - if (isset($this->data['Event']['analysis'])) { - switch ($this->data['Event']['analysis']) { + if (isset($event['analysis'])) { + switch ($event['analysis']) { case 'Initial': - $this->data['Event']['analysis'] = 0; + $event['analysis'] = 0; break; case 'Ongoing': - $this->data['Event']['analysis'] = 1; + $event['analysis'] = 1; break; case 'Completed': - $this->data['Event']['analysis'] = 2; + $event['analysis'] = 2; break; } } else { - $this->data['Event']['analysis'] = 0; + $event['analysis'] = 0; } - if (!isset($this->data['Event']['threat_level_id'])) { - $this->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4; + if (!isset($event['threat_level_id'])) { + $event['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4; } // generate UUID if it doesn't exist - if (empty($this->data['Event']['uuid'])) { - $this->data['Event']['uuid'] = CakeText::uuid(); - } else { - $this->data['Event']['uuid'] = strtolower($this->data['Event']['uuid']); + if (!empty($event['uuid'])) { + $event['uuid'] = strtolower($event['uuid']); } // Convert event ID to uuid if needed - if (!empty($this->data['Event']['extends_uuid']) && is_numeric($this->data['Event']['extends_uuid'])) { - $extended_event = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('Event.id' => $this->data['Event']['extends_uuid']), - 'fields' => array('Event.uuid') - )); - if (empty($extended_event)) { - $this->data['Event']['extends_uuid'] = ''; + if (!empty($event['extends_uuid'])) { + if (is_numeric($event['extends_uuid'])) { + $extended_event = $this->find('first', array( + 'recursive' => -1, + 'conditions' => array('Event.id' => $event['extends_uuid']), + 'fields' => array('Event.uuid') + )); + if (empty($extended_event)) { + $event['extends_uuid'] = ''; + $this->invalidate('extends_uuid', 'Invalid event ID provided.'); + } else { + $event['extends_uuid'] = $extended_event['Event']['uuid']; + } } else { - $this->data['Event']['extends_uuid'] = $extended_event['Event']['uuid']; + $event['extends_uuid'] = strtolower($event['extends_uuid']); } - } else if (!empty($this->data['Event']['extends_uuid'])) { - $this->data['Event']['extends_uuid'] = strtolower($this->data['Event']['extends_uuid']); } // generate timestamp if it doesn't exist - if (empty($this->data['Event']['timestamp'])) { - $this->data['Event']['timestamp'] = time(); + if (empty($event['timestamp'])) { + $event['timestamp'] = time(); } - if (isset($this->data['Event']['publish_timestamp']) && empty($this->data['Event']['publish_timestamp'])) { - $this->data['Event']['publish_timestamp'] = 0; + if (isset($event['publish_timestamp']) && empty($event['publish_timestamp'])) { + $event['publish_timestamp'] = 0; } - if (empty($this->data['Event']['date'])) { - $this->data['Event']['date'] = date('Y-m-d'); + if (empty($event['date'])) { + $event['date'] = date('Y-m-d'); } - if (!isset($this->data['Event']['distribution']) || $this->data['Event']['distribution'] != 4) { - $this->data['Event']['sharing_group_id'] = 0; + if (!isset($event['distribution']) || $event['distribution'] != 4) { + $event['sharing_group_id'] = 0; } } + public function beforeSave($options = []) + { + // generate UUID if not provided + if (empty($this->data['Event']['uuid'])) { + $this->data['Event']['uuid'] = CakeText::uuid(); + } + return true; + } + public function afterSave($created, $options = array()) { + $event = $this->data['Event']; if (!Configure::read('MISP.completely_disable_correlation') && !$created) { $updateCorrelation = []; - if (isset($this->data['Event']['distribution'])) { - $updateCorrelation['Correlation.distribution'] = (int)$this->data['Event']['distribution']; + if (isset($event['distribution']) && (empty($options['fieldList']) || in_array('distribution', $options['fieldList']))) { + $updateCorrelation['Correlation.distribution'] = (int)$event['distribution']; } - if (isset($this->data['Event']['sharing_group_id'])) { - $updateCorrelation['Correlation.sharing_group_id'] = (int)$this->data['Event']['sharing_group_id']; + if (isset($event['sharing_group_id']) && (empty($options['fieldList']) || in_array('sharing_group_id', $options['fieldList']))) { + $updateCorrelation['Correlation.sharing_group_id'] = (int)$event['sharing_group_id']; } if (!empty($updateCorrelation)) { - $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$this->data['Event']['id']]); + $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]); } } - if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { $pubSubTool = $this->getPubSubTool(); - $event = $this->quickFetchEvent($this->data['Event']['id']); + $eventForZmq = $this->quickFetchEvent($event['id']); if (!empty($event)) { - $pubSubTool->event_save($event, $created ? 'add' : 'edit'); + $pubSubTool->event_save($eventForZmq, $created ? 'add' : 'edit'); } } - if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_kafka'])) { - $this->publishKafkaNotification('event', $this->quickFetchEvent($this->data['Event']['id']), $created ? 'add' : 'edit'); + if (empty($event['unpublishAction']) && empty($event['skip_kafka'])) { + $this->publishKafkaNotification('event', $this->quickFetchEvent($event['id']), $created ? 'add' : 'edit'); } } @@ -535,23 +538,23 @@ class Event extends AppModel { $tagsToFetch = array(); foreach ($events as $event) { - if (!empty($event['EventTag'])) { - foreach ($event['EventTag'] as $et) { - $tagsToFetch[$et['tag_id']] = $et['tag_id']; - } + foreach ($event['EventTag'] as $et) { + $tagsToFetch[$et['tag_id']] = $et['tag_id']; } } + if (empty($tagsToFetch)) { + return $events; + } $tags = $this->EventTag->Tag->find('all', array( 'conditions' => array('Tag.id' => $tagsToFetch), 'recursive' => -1, + 'fields' => ['id', 'name', 'colour', 'is_galaxy'], // fetch just necessary columns 'order' => false )); $tags = array_column(array_column($tags, 'Tag'), null, 'id'); - foreach ($events as $k => $event) { - if (!empty($event['EventTag'])) { - foreach ($event['EventTag'] as $k2 => $et) { - $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]; - } + foreach ($events as &$event) { + foreach ($event['EventTag'] as &$et) { + $et['Tag'] = $tags[$et['tag_id']]; } } return $events; @@ -773,51 +776,53 @@ class Event extends AppModel public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event') { if ($shadowAttribute) { - $settings = array('correlationModel' => 'ShadowAttributeCorrelation', 'parentIdField' => '1_shadow_attribute_id'); + $parentIdField = '1_shadow_attribute_id'; + $correlationModelName = 'ShadowAttributeCorrelation'; } else { - $settings = array('correlationModel' => 'Correlation', 'parentIdField' => '1_attribute_id'); + $parentIdField = '1_attribute_id'; + $correlationModelName = 'Correlation'; } - if (!isset($this->{$settings['correlationModel']})) { - $this->{$settings['correlationModel']} = ClassRegistry::init($settings['correlationModel']); + if (!isset($this->{$correlationModelName})) { + $this->{$correlationModelName} = ClassRegistry::init($correlationModelName); } if (!$user['Role']['perm_site_admin']) { $sgids = $this->cacheSgids($user, true); $conditionsCorrelation = array( 'AND' => array( - $settings['correlationModel'] . '.1_' . $scope . '_id' => $id, + $correlationModelName . '.1_' . $scope . '_id' => $id, array( 'OR' => array( - $settings['correlationModel'] . '.org_id' => $user['org_id'], + $correlationModelName . '.org_id' => $user['org_id'], 'AND' => array( array( 'OR' => array( array( 'AND' => array( - $settings['correlationModel'] . '.distribution >' => 0, - $settings['correlationModel'] . '.distribution <' => 4, + $correlationModelName . '.distribution >' => 0, + $correlationModelName . '.distribution <' => 4, ), ), array( 'AND' => array( - $settings['correlationModel'] . '.distribution' => 4, - $settings['correlationModel'] . '.sharing_group_id' => $sgids + $correlationModelName . '.distribution' => 4, + $correlationModelName . '.sharing_group_id' => $sgids ), ), ), ), array( 'OR' => array( - $settings['correlationModel'] . '.a_distribution' => 5, + $correlationModelName . '.a_distribution' => 5, array( 'AND' => array( - $settings['correlationModel'] . '.a_distribution >' => 0, - $settings['correlationModel'] . '.a_distribution <' => 4, + $correlationModelName . '.a_distribution >' => 0, + $correlationModelName . '.a_distribution <' => 4, ), ), array( 'AND' => array( - $settings['correlationModel'] . '.a_distribution' => 4, - $settings['correlationModel'] . '.a_sharing_group_id' => $sgids + $correlationModelName . '.a_distribution' => 4, + $correlationModelName . '.a_sharing_group_id' => $sgids ), ), ), @@ -829,11 +834,11 @@ class Event extends AppModel ) ); } else { - $conditionsCorrelation = array($settings['correlationModel'] . '.1_' . $scope . '_id' => $id); + $conditionsCorrelation = array($correlationModelName . '.1_' . $scope . '_id' => $id); } $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; - $correlations = $this->{$settings['correlationModel']}->find('all', array( - 'fields' => ['event_id', 'attribute_id', 'value', $settings['parentIdField']], + $correlations = $this->{$correlationModelName}->find('all', array( + 'fields' => ['event_id', 'attribute_id', 'value', $parentIdField], 'conditions' => $conditionsCorrelation, 'recursive' => -1, 'order' => false, @@ -843,11 +848,11 @@ class Event extends AppModel return array(); } - $correlations = array_column($correlations, $settings['correlationModel']); + $correlations = array_column($correlations, $correlationModelName); $eventIds = array_unique(array_column($correlations, 'event_id')); $conditions = $this->createEventConditions($user); - $conditions['AND']['Event.id'] = $eventIds; + $conditions['Event.id'] = $eventIds; $events = $this->find('all', array( 'recursive' => -1, 'conditions' => $conditions, @@ -859,20 +864,21 @@ class Event extends AppModel $relatedAttributes = []; foreach ($correlations as $correlation) { // User don't have access to correlated attribute event, skip. - if (!isset($events[$correlation['event_id']])) { + $eventId = $correlation['event_id']; + if (!isset($events[$eventId])) { continue; } - $event = $events[$correlation['event_id']]; + $event = $events[$eventId]; $current = array( - 'id' => $correlation['event_id'], + 'id' => $eventId, 'attribute_id' => $correlation['attribute_id'], 'value' => $correlation['value'], 'org_id' => $event['orgc_id'], 'info' => $event['info'], 'date' => $event['date'], ); - $parentId = $correlation[$settings['parentIdField']]; + $parentId = $correlation[$parentIdField]; $relatedAttributes[$parentId][] = $current; } return $relatedAttributes; @@ -924,7 +930,7 @@ class Event extends AppModel return 'The distribution level of this event blocks it from being pushed.'; } - $push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket); + $push = $this->Server->checkVersionCompatibility($server, false); if (empty($push['canPush'])) { return 'The remote user is not a sync user - the upload of the event has been blocked.'; } @@ -1344,9 +1350,9 @@ class Event extends AppModel return $data; } - public function quickDelete($event) + public function quickDelete(array $event) { - $id = $event['Event']['id']; + $id = (int)$event['Event']['id']; $this->Thread = ClassRegistry::init('Thread'); $thread = $this->Thread->find('first', array( 'conditions' => array('Thread.event_id' => $id), @@ -1433,7 +1439,8 @@ class Event extends AppModel foreach ($relations as $relation) { $queryTool->quickDelete($relation['table'], $relation['foreign_key'], $relation['value'], $this); } - return $this->delete($id, false); + $this->set($event); + return $this->delete(null, false); } public function createEventConditions($user) @@ -1770,6 +1777,9 @@ class Event extends AppModel // includeAttachments: true will attach the attachments to the attributes in the data field public function fetchEvent($user, $options = array(), $useCache = false) { + if (!isset($user['org_id'])) { + throw new InvalidArgumentException('There was an error with the user account (missing `org_id` field).'); + } if (isset($options['Event.id'])) { $options['eventid'] = $options['Event.id']; } @@ -1789,14 +1799,14 @@ class Event extends AppModel 'disableSiteAdmin', 'metadata', 'enforceWarninglist', // return just attributes that contains no warnings - 'sgReferenceOnly', + 'sgReferenceOnly', // do not fetch additional information about sharing groups 'flatten', 'blockedAttributeTags', 'eventsExtendingUuid', 'extended', 'extensionList', 'excludeGalaxy', - 'includeCustomGalaxyCluster', // not used + // 'includeCustomGalaxyCluster', // not used 'includeRelatedTags', 'excludeLocalTags', 'includeDecayScore', @@ -1855,9 +1865,6 @@ class Event extends AppModel $conditions['AND'][] = array('Event.id' => -1); } } - if (!isset($user['org_id'])) { - throw new Exception('There was an error with the user account.'); - } $isSiteAdmin = $user['Role']['perm_site_admin']; if (isset($options['disableSiteAdmin']) && $options['disableSiteAdmin']) { $isSiteAdmin = false; @@ -1866,13 +1873,10 @@ class Event extends AppModel $conditionsObjects = array(); $conditionsEventReport = array(); - if ($options['flatten']) { - $flatten = true; - } else { - $flatten = false; - } - $sgids = $this->cacheSgids($user, $useCache); + $flatten = (bool)$options['flatten']; + // restricting to non-private or same org if the user is not a site-admin. + $sgids = $this->cacheSgids($user, $useCache); if (!$isSiteAdmin) { // if delegations are enabled, check if there is an event that the current user might see because of the request itself if (Configure::read('MISP.delegation')) { @@ -1882,7 +1886,7 @@ class Event extends AppModel $attributeCondSelect = '(SELECT events.org_id FROM events WHERE events.id = Attribute.event_id)'; $objectCondSelect = '(SELECT events.org_id FROM events WHERE events.id = Object.event_id)'; $eventReportCondSelect = '(SELECT events.org_id FROM events WHERE events.id = EventReport.event_id)'; - if ($this->getDataSource()->config['datasource'] == 'Database/Postgres') { + if ($this->getDataSource()->config['datasource'] === 'Database/Postgres') { $schemaName = $this->getDataSource()->config['schema']; $attributeCondSelect = sprintf('(SELECT "%s"."events"."org_id" FROM "%s"."events" WHERE "%s"."events"."id" = "Attribute"."event_id")', $schemaName, $schemaName, $schemaName); $objectCondSelect = sprintf('(SELECT "%s"."events"."org_id" FROM "%s"."events" WHERE "%s"."events"."id" = "Object"."event_id")', $schemaName, $schemaName, $schemaName); @@ -1955,7 +1959,7 @@ class Event extends AppModel $conditions['AND'][] = array('Event.published' => 1); $conditionsAttributes['AND'][] = array('Attribute.to_ids' => 1); } - $softDeletables = array('Attribute', 'Object', 'ObjectReference', 'EventReport'); + $softDeletables = array('Attribute', 'Object', 'EventReport'); if (isset($options['deleted'])) { if (!is_array($options['deleted'])) { $options['deleted'] = array($options['deleted']); @@ -1964,7 +1968,7 @@ class Event extends AppModel if ($deleted_value === 'only') { $deleted_value = 1; } - $options['deleted'][$deleted_key] = intval($deleted_value); + $options['deleted'][$deleted_key] = (int)$deleted_value; } if (!$user['Role']['perm_sync']) { foreach ($softDeletables as $softDeletable) { @@ -1981,17 +1985,21 @@ class Event extends AppModel 'OR' => array( 'AND' => array( sprintf('(SELECT events.org_id FROM events WHERE events.id = %s.event_id)', $softDeletable) => $user['org_id'], - sprintf('%s.deleted', $softDeletable) => $options['deleted'] + "$softDeletable.deleted" => $options['deleted'], ), $deletion_subconditions ) ); } } else { - foreach ($softDeletables as $softDeletable) { - ${'conditions' . $softDeletable . 's'}['AND'][] = array( - sprintf('%s.deleted', $softDeletable) => $options['deleted'] - ); + // MySQL couldn't optimise query, so it is better just skip this condition + $both = in_array(0, $options['deleted']) && in_array(1, $options['deleted']); + if (!$both) { + foreach ($softDeletables as $softDeletable) { + ${'conditions' . $softDeletable . 's'}['AND'][] = [ + "$softDeletable.deleted" => $options['deleted'], + ]; + } } } } else { @@ -2029,10 +2037,8 @@ class Event extends AppModel // do not expose all the data ... $fields = array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.user_id', 'Event.locked', 'Event.publish_timestamp', 'Event.sharing_group_id', 'Event.disable_correlation', 'Event.extends_uuid'); $fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.first_seen', 'Attribute.last_seen'); - $fieldsObj = array('*'); $fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'); $fieldsOrg = array('id', 'name', 'uuid', 'local'); - $fieldsEventReport = array('*'); $params = array( 'conditions' => $conditions, 'recursive' => 0, @@ -2044,18 +2050,11 @@ class Event extends AppModel 'Attribute' => array( 'fields' => $fieldsAtt, 'conditions' => $conditionsAttributes, - 'AttributeTag' => array( - 'order' => false - ), 'order' => false ), 'Object' => array( - 'fields' => $fieldsObj, 'conditions' => $conditionsObjects, 'order' => false, - 'ObjectReference' => array( - 'order' => false - ) ), 'ShadowAttribute' => array( 'fields' => $fieldsShadowAtt, @@ -2067,7 +2066,6 @@ class Event extends AppModel 'order' => false ), 'EventReport' => array( - 'fields' => $fieldsEventReport, 'conditions' => $conditionsEventReport, 'order' => false ) @@ -2077,9 +2075,6 @@ class Event extends AppModel $params['contain']['EventTag']['conditions'] = array( 'EventTag.local' => 0 ); - $params['contain']['Attribute']['AttributeTag']['conditions'] = array( - 'AttributeTag.local' => 0 - ); } if ($flatten) { unset($params['contain']['Object']); @@ -2110,12 +2105,16 @@ class Event extends AppModel return array(); } - $sharingGroupData = $options['sgReferenceOnly'] ? [] : $this->__cacheSharingGroupData($user, $useCache); + $sharingGroupReferenceOnly = (bool)$options['sgReferenceOnly']; + $sharingGroupData = $sharingGroupReferenceOnly ? [] : $this->__cacheSharingGroupData($user, $useCache); // Initialize classes that will be necessary during event fetching if (!empty($options['includeDecayScore']) && !isset($this->DecayingModel)) { $this->DecayingModel = ClassRegistry::init('DecayingModel'); } + if ($options['includeServerCorrelations'] && !$isSiteAdmin && $user['org_id'] != Configure::read('MISP.host_org_id')) { + $options['includeServerCorrelations'] = false; // not permission to see server correlations + } if (($options['includeFeedCorrelations'] || $options['includeServerCorrelations']) && !isset($this->Feed)) { $this->Feed = ClassRegistry::init('Feed'); } @@ -2125,20 +2124,27 @@ class Event extends AppModel // Precache current user email $userEmails = empty($user['id']) ? [] : [$user['id'] => $user['email']]; - // Do some refactoring with the event - $fields = array( - 'common' => array('distribution', 'sharing_group_id', 'uuid'), - 'Attribute' => array('value', 'type', 'category', 'to_ids'), - 'Object' => array('name', 'meta-category') - ); - if (!$options['includeAllTags']) { $justExportableTags = true; } else { $justExportableTags = false; } - foreach ($results as $eventKey => &$event) { + $overrideLimit = !empty($options['overrideLimit']); + + if (!empty($options['allow_proposal_blocking']) && !Configure::read('MISP.proposals_block_attributes')) { + $options['allow_proposal_blocking'] = false; // proposal blocking is not enabled + } + + if (!$options['metadata']) { + $this->__attachAttributeTags($results, $options['excludeLocalTags']); + } + + if (!$options['metadata'] && !$flatten) { + $this->__attachReferences($results); + } + + foreach ($results as &$event) { /* // REMOVING THIS FOR NOW - users should see data they own, even if they're not in the sharing group. if ($event['Event']['distribution'] == 4 && !in_array($event['Event']['sharing_group_id'], $sgids)) { @@ -2163,13 +2169,12 @@ class Event extends AppModel $this->Warninglist->attachWarninglistToAttributes($event['ShadowAttribute']); $event['warnings'] = $eventWarnings; } - $this->__attachReferences($event, $fields); $this->__attachTags($event, $justExportableTags); $this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']); $event = $this->Orgc->attachOrgs($event, $fieldsOrg); - if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) { - if (!empty($sharingGroupData[$event['Event']['sharing_group_id']]['SharingGroup'])) { - $event['SharingGroup'] = $sharingGroupData[$event['Event']['sharing_group_id']]['SharingGroup']; + if (!$sharingGroupReferenceOnly && $event['Event']['sharing_group_id']) { + if (isset($sharingGroupData[$event['Event']['sharing_group_id']])) { + $event['SharingGroup'] = $sharingGroupData[$event['Event']['sharing_group_id']]; } } @@ -2191,126 +2196,107 @@ class Event extends AppModel } // Let's find all the related events and attach it to the event itself if ($options['includeEventCorrelations']) { - $results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids); + $event['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids); } // Let's also find all the relations for the attributes - this won't be in the xml export though if (!empty($options['includeGranularCorrelations'])) { - $results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id']); + $event['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id']); if (!empty($options['includeRelatedTags'])) { - $results[$eventKey] = $this->includeRelatedTags($results[$eventKey], $options); + $event = $this->includeRelatedTags($event, $options); } - $results[$eventKey]['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true); + $event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true); } - if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute']) && isset($options['includeAttachments']) && $options['includeAttachments']) { - foreach ($event['ShadowAttribute'] as $k => $sa) { - if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) { - $encodedFile = $this->ShadowAttribute->base64EncodeAttachment($sa); - $event['ShadowAttribute'][$k]['data'] = $encodedFile; + $shadowAttributeByOldId = []; + if (!empty($event['ShadowAttribute'])) { + if ($isSiteAdmin && $options['includeFeedCorrelations']) { + $event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit); + } + if ($options['includeServerCorrelations']) { + $event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server'); + } + + if ($options['includeAttachments']) { + foreach ($event['ShadowAttribute'] as &$sa) { + if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) { + $encodedFile = $this->ShadowAttribute->base64EncodeAttachment($sa); + $sa['data'] = $encodedFile; + } } + unset($sa); } + + foreach ($event['ShadowAttribute'] as $sa) { + $shadowAttributeByOldId[$sa['old_id']][] = $sa; + } + // Assign just shadow attributes that are linked to event (that means they have old_id set to `0`) + $event['ShadowAttribute'] = $shadowAttributeByOldId[0] ?? []; } - if (isset($event['Attribute'])) { + if (!empty($event['Attribute'])) { if ($options['includeFeedCorrelations']) { - if (!empty($options['overrideLimit'])) { - $overrideLimit = true; - } else { - $overrideLimit = false; - } $event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit); } - if (!empty($options['includeServerCorrelations']) && ($user['Role']['perm_site_admin'] || $user['org_id'] == Configure::read('MISP.host_org_id'))) { - if (!empty($options['overrideLimit'])) { - $overrideLimit = true; - } else { - $overrideLimit = false; - } + if ($options['includeServerCorrelations']) { $event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit, 'Server'); } $event = $this->__filterBlockedAttributesByTags($event, $options, $user); - $event['Attribute'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Attribute'], $sharingGroupData); + if (!$sharingGroupReferenceOnly) { + $event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData); + } - $proposalBlockAttributes = Configure::read('MISP.proposals_block_attributes'); // move all object attributes to a temporary container $tempObjectAttributeContainer = array(); - foreach ($event['Attribute'] as $key => $attribute) { + foreach ($event['Attribute'] as $key => &$attribute) { if ($options['enforceWarninglist'] && !empty($attribute['warnings'])) { unset($event['Attribute'][$key]); continue; } if ($attribute['category'] === 'Financial fraud') { - $event['Attribute'][$key] = $this->Attribute->attachValidationWarnings($event['Attribute'][$key]); + $attribute = $this->Attribute->attachValidationWarnings($attribute); } if ($options['includeAttachments'] && $this->Attribute->typeIsAttachment($attribute['type'])) { $encodedFile = $this->Attribute->base64EncodeAttachment($attribute); - $event['Attribute'][$key]['data'] = $encodedFile; + $attribute['data'] = $encodedFile; } if (!empty($options['includeDecayScore'])) { if (isset($event['EventTag'])) { // include EventTags for score computation - $event['Attribute'][$key]['EventTag'] = $event['EventTag']; + $attribute['EventTag'] = $event['EventTag']; } - $event['Attribute'][$key] = $this->DecayingModel->attachScoresToAttribute($user, $event['Attribute'][$key]); + $attribute = $this->DecayingModel->attachScoresToAttribute($user, $attribute); if (isset($event['EventTag'])) { // remove included EventTags - unset($event['Attribute'][$key]['EventTag']); + unset($attribute['EventTag']); } } - $event['Attribute'][$key]['ShadowAttribute'] = array(); - // If a shadowattribute can be linked to an attribute, link it to it then remove it from the event + // If a shadowattribute can be linked to an attribute, link it to it // This is to differentiate between proposals that were made to an attribute for modification and between proposals for new attributes - - if (isset($event['ShadowAttribute'])) { - foreach ($event['ShadowAttribute'] as $k => $sa) { - if (!empty($sa['old_id'])) { - if ($event['ShadowAttribute'][$k]['old_id'] == $attribute['id']) { - $results[$eventKey]['Attribute'][$key]['ShadowAttribute'][] = $sa; - unset($results[$eventKey]['ShadowAttribute'][$k]); - } - } - } - } - if ($proposalBlockAttributes && !empty($options['allow_proposal_blocking'])) { - foreach ($results[$eventKey]['Attribute'][$key]['ShadowAttribute'] as $sa) { + $attribute['ShadowAttribute'] = $shadowAttributeByOldId[$attribute['id']] ?? []; + if (!empty($options['allow_proposal_blocking'])) { + foreach ($attribute['ShadowAttribute'] as $sa) { if ($sa['proposal_to_delete'] || $sa['to_ids'] == 0) { - unset($results[$eventKey]['Attribute'][$key]); - continue; + unset($event['Attribute'][$key]); + continue 2; } } } if (!$flatten && $attribute['object_id'] != 0) { - $tempObjectAttributeContainer[$attribute['object_id']][] = $event['Attribute'][$key]; + $tempObjectAttributeContainer[$attribute['object_id']][] = $attribute; unset($event['Attribute'][$key]); } } $event['Attribute'] = array_values($event['Attribute']); } if (!empty($event['Object'])) { - $event['Object'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Object'], $sharingGroupData); - foreach ($event['Object'] as $objectKey => $objectValue) { + if (!$sharingGroupReferenceOnly) { + $event['Object'] = $this->__attachSharingGroups($event['Object'], $sharingGroupData); + } + foreach ($event['Object'] as &$objectValue) { if (isset($tempObjectAttributeContainer[$objectValue['id']])) { - $event['Object'][$objectKey]['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']]; + $objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']]; } } unset($tempObjectAttributeContainer); } - if (!empty($event['EventReport'])) { - $event['EventReport'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['EventReport'], $sharingGroupData); - } - if (!empty($event['ShadowAttribute'])) { - if ($isSiteAdmin && $options['includeFeedCorrelations']) { - if (!empty($options['overrideLimit'])) { - $overrideLimit = true; - } else { - $overrideLimit = false; - } - $event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit); - } - if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) { - if (!empty($options['overrideLimit'])) { - $overrideLimit = true; - } else { - $overrideLimit = false; - } - $event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server'); - } + if (!$sharingGroupReferenceOnly && !empty($event['EventReport'])) { + $event['EventReport'] = $this->__attachSharingGroups($event['EventReport'], $sharingGroupData); } if (empty($options['metadata']) && empty($options['noSightings'])) { $event['Sighting'] = $this->Sighting->attachToEvent($event, $user); @@ -2319,16 +2305,6 @@ class Event extends AppModel $this->Sightingdb = ClassRegistry::init('Sightingdb'); $event = $this->Sightingdb->attachToEvent($event, $user); } - // remove proposals to attributes that we cannot see - // if the shadow attribute wasn't moved within an attribute before, this is the case - if (isset($event['ShadowAttribute'])) { - foreach ($event['ShadowAttribute'] as $k => $sa) { - if (!empty($sa['old_id'])) { - unset($event['ShadowAttribute'][$k]); - } - } - $event['ShadowAttribute'] = array_values($event['ShadowAttribute']); - } } if ($options['extended']) { foreach ($results as $k => $result) { @@ -2385,10 +2361,7 @@ class Event extends AppModel return; } - $clustersByTagNames = []; - foreach ($clusters as $cluster) { - $clustersByTagNames[mb_strtolower($cluster['GalaxyCluster']['tag_name'])] = $cluster['GalaxyCluster']; - } + $clustersByTagIds = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id'); unset($clusters); if (isset($event['EventTag'])) { @@ -2396,9 +2369,9 @@ class Event extends AppModel if (!$eventTag['Tag']['is_galaxy']) { continue; } - $tagName = mb_strtolower($eventTag['Tag']['name']); - if (isset($clustersByTagNames[$tagName])) { - $cluster = $clustersByTagNames[$tagName]; + $tagId = $eventTag['Tag']['id']; + if (isset($clustersByTagIds[$tagId])) { + $cluster = $clustersByTagIds[$tagId]; $galaxyId = $cluster['Galaxy']['id']; $cluster['local'] = isset($eventTag['local']) ? $eventTag['local'] : false; if (isset($event['Galaxy'][$galaxyId])) { @@ -2420,9 +2393,9 @@ class Event extends AppModel if (!$attributeTag['Tag']['is_galaxy']) { continue; } - $tagName = mb_strtolower($attributeTag['Tag']['name']); - if (isset($clustersByTagNames[$tagName])) { - $cluster = $clustersByTagNames[$tagName]; + $tagId = $attributeTag['Tag']['id']; + if (isset($clustersByTagIds[$tagId])) { + $cluster = $clustersByTagIds[$tagId]; $galaxyId = $cluster['Galaxy']['id']; $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false; if (isset($attribute['Galaxy'][$galaxyId])) { @@ -2623,15 +2596,12 @@ class Event extends AppModel return $event; } - private function __attachSharingGroups($doAttach, $data, $sharingGroupData) + private function __attachSharingGroups($data, $sharingGroupData) { - if (!$doAttach) { - return $data; - } foreach ($data as $k => $v) { if ($v['distribution'] == 4) { if (isset($sharingGroupData[$v['sharing_group_id']])) { - $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup']; + $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]; } else { unset($data[$k]); // current user could not fetch the sharing_group } @@ -3151,15 +3121,24 @@ class Event extends AppModel return !$banError; } if (Configure::read('MISP.background_jobs')) { + /** @var Job $job */ $job = ClassRegistry::init('Job'); $jobId = $job->createJob($user, Job::WORKER_EMAIL, 'publish_alert_email', "Event: $id", 'Sending...'); - $process_id = CakeResque::enqueue( - Job::WORKER_EMAIL, - 'EventShell', - array('alertemail', $user['id'], $jobId, $id, $oldpublish), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::EMAIL_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'alertemail', + $user['id'], + $jobId, + $id, + $oldpublish + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return true; } else { return $this->sendAlertEmail($id, $user, $oldpublish); @@ -3209,7 +3188,6 @@ class Event extends AppModel ); $userCount = count($usersWithAccess); - $this->UserSetting = ClassRegistry::init('UserSetting'); $metadataOnly = Configure::read('MISP.event_alert_metadata_only') || Configure::read('MISP.publish_alerts_summary_only'); foreach ($usersWithAccess as $k => $user) { // Fetch event for user that will receive alert e-mail to respect all ACLs @@ -3222,7 +3200,7 @@ class Event extends AppModel 'metadata' => $metadataOnly, ])[0]; - if ($this->UserSetting->checkPublishFilter($user, $eventForUser)) { + if ($this->User->UserSetting->checkPublishFilter($user, $eventForUser)) { $body = $this->prepareAlertEmail($eventForUser, $user, $oldpublish); $this->User->sendEmail(['User' => $user], $body, false, null); } @@ -3879,9 +3857,12 @@ class Event extends AppModel foreach ($referencesToCapture as $referenceToCapture) { $result = $this->Object->ObjectReference->captureReference( $referenceToCapture, - $this->id, - $user + $this->id ); + if ($result !== true) { + $title = "Could not save object reference when capturing event with ID {$this->id}"; + $this->loadLog()->validationError($user, 'add', 'ObjectReference', $title, $result, $referenceToCapture); + } } } @@ -4098,7 +4079,11 @@ class Event extends AppModel foreach ($object['ObjectReference'] as $objectRef) { $nothingToChange = false; $objectRef['source_uuid'] = $object['uuid']; - $result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user); + $result = $this->Object->ObjectReference->captureReference($objectRef, $this->id); + if ($result !== true) { + $title = "Could not save object reference when capturing event with ID {$this->id}"; + $this->loadLog()->validationError($user, 'edit', 'ObjectReference', $title, $result, $objectRef); + } if ($result && !$nothingToChange) { $changed = true; } @@ -4544,25 +4529,24 @@ class Event extends AppModel public function publishSightingsRouter($id, array $user, $passAlong = null, array $sightingUuids = []) { if (Configure::read('MISP.background_jobs')) { + /** @var Job $job */ $job = ClassRegistry::init('Job'); $message = empty($sightingUuids) ? __('Publishing sightings.') : __('Publishing %s sightings.', count($sightingUuids)); $jobId = $job->createJob($user, Job::WORKER_PRIO, 'publish_event', "Event ID: $id", $message); - $command = ['publish_sightings', $id, $passAlong, $jobId, $user['id']]; + $args = ['publish_sightings', $id, $passAlong, $jobId, $user['id']]; if (!empty($sightingUuids)) { - $randomFileName = $this->generateRandomFileName() . '.json'; - App::uses('File', 'Utility'); - $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $randomFileName, true, 0644); - $writeResult = $tempFile->write(json_encode($sightingUuids)); - if (!$writeResult) { - throw new Exception("Could not write file content"); - } - $command[] = $randomFileName; + $filePath = FileAccessTool::writeToTempFile(json_encode($sightingUuids)); + $args[] = $filePath; } - $processId = CakeResque::enqueue(Job::WORKER_PRIO, 'EventShell', $command, true); - $job->saveField('process_id', $processId); - return $processId; + return $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + $args, + true, + $jobId + ); } return $this->publish_sightings($id, $passAlong, $sightingUuids); @@ -4571,17 +4555,26 @@ class Event extends AppModel public function publishRouter($id, $passAlong = null, $user) { if (Configure::read('MISP.background_jobs')) { + + /** @var Job $job */ $job = ClassRegistry::init('Job'); $jobId = $job->createJob($user, Job::WORKER_PRIO, 'publish_event', "Event ID: $id", 'Publishing.'); - $process_id = CakeResque::enqueue( - Job::WORKER_PRIO, - 'EventShell', - array('publish', $id, $passAlong, $jobId, $user['id']), - true + + return $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'publish', + $id, + $passAlong, + $jobId, + $user['id'] + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); - return $process_id; } + return $this->publish($id, $passAlong); } @@ -4689,26 +4682,31 @@ class Event extends AppModel public function sendContactEmailRouter($id, $message, $creator_only, $user) { if (Configure::read('MISP.background_jobs')) { + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'email', - 'job_type' => 'contact_alert', - 'job_input' => 'Owner ' . ($creator_only ? 'user' : 'org') . ' of event #' . $id, - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'message' => 'Contacting.', + $jobId = $job->createJob( + $user, + Job::WORKER_EMAIL, + 'contact_alert', + 'Owner ' . ($creator_only ? 'user' : 'org') . ' of event #' . $id, + 'Contacting.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'email', - 'EventShell', - array('contactemail', $id, $message, $creator_only, $user['id'], $jobId), - true + + return $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::EMAIL_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'contactemail', + $id, + $message, + $creator_only, + $user['id'], + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return true; } else { return $this->sendContactEmail($id, $message, $creator_only, $user); @@ -5445,22 +5443,18 @@ class Event extends AppModel unset($event['Object']); } - $event['objects'] = $objects; - $referencedByArray = array(); - foreach ($event['objects'] as $object) { - if (!in_array($object['objectType'], array('attribute', 'object'))) { - continue; - } - if (!empty($object['ObjectReference'])) { + foreach ($objects as $object) { + $objectType = $object['objectType']; + if (($objectType === 'attribute' || $objectType === 'object') && !empty($object['ObjectReference'])) { foreach ($object['ObjectReference'] as $reference) { if (isset($reference['referenced_uuid'])) { - $referencedByArray[$reference['referenced_uuid']][$object['objectType']][] = array( + $referencedByArray[$reference['referenced_uuid']][$objectType][] = array( 'meta-category' => $object['meta-category'], 'name' => $object['name'], 'uuid' => $object['uuid'], 'id' => isset($object['id']) ? $object['id'] : 0, - 'object_type' => $object['objectType'], + 'object_type' => $objectType, 'relationship_type' => $reference['relationship_type'] ); } @@ -5472,15 +5466,16 @@ class Event extends AppModel if ($all) { $passedArgs['page'] = 0; } - $params = $customPagination->applyRulesOnArray($event['objects'], $passedArgs, 'events', 'category'); - foreach ($event['objects'] as $k => $object) { + $params = $customPagination->applyRulesOnArray($objects, $passedArgs, 'events', 'category'); + foreach ($objects as $k => $object) { if (isset($referencedByArray[$object['uuid']])) { foreach ($referencedByArray[$object['uuid']] as $objectType => $references) { - $event['objects'][$k]['referenced_by'][$objectType] = $references; + $objects[$k]['referenced_by'][$objectType] = $references; } } } - $params['total_elements'] = count($event['objects']); + $event['objects'] = $objects; + $params['total_elements'] = count($objects); return $params; } @@ -5784,7 +5779,7 @@ class Event extends AppModel } else { $sharingGroupDataTemp = $this->SharingGroup->fetchAllAuthorised($user, 'simplified'); $sharingGroupData = array(); - foreach ($sharingGroupDataTemp as $k => $v) { + foreach ($sharingGroupDataTemp as $v) { if (isset($v['Organisation'])) { $v['SharingGroup']['Organisation'] = $v['Organisation']; } @@ -5803,7 +5798,7 @@ class Event extends AppModel } } } - $sharingGroupData[$v['SharingGroup']['id']] = array('SharingGroup' => $v['SharingGroup']); + $sharingGroupData[$v['SharingGroup']['id']] = $v['SharingGroup']; } if ($useCache) { $this->assetCache['sharingGroupData'] = $sharingGroupData; @@ -5836,7 +5831,7 @@ class Event extends AppModel } else { $filters = array(); $args = $this->Attribute->dissectArgs($tagRules); - $tagArray = $this->EventTag->Tag->fetchEventTagIds($args[0], $args[1]); + $tagArray = $this->EventTag->fetchEventTagIds($args[0], $args[1]); if (!empty($tagArray[0])) { $filters[] = ['OR' => ['Event.id' => $tagArray[0]]]; } else { @@ -5849,72 +5844,80 @@ class Event extends AppModel } } - public function unpublishEvent($id, $proposalLock = false) + /** + * @param int $id + * @param bool $proposalLock + * @param int|null $timestamp If not provided, current time will be used + * @return array|bool|mixed|null + * @throws Exception + */ + public function unpublishEvent($id, $proposalLock = false, $timestamp = null) { $event = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('Event.id' => $id) + 'conditions' => array('Event.id' => $id), + 'fields' => ['id', 'info'], // info is required because of SysLogLogableBehavior )); if (empty($event)) { return false; } + $fields = ['published', 'timestamp']; $event['Event']['published'] = 0; - $date = new DateTime(); - $event['Event']['timestamp'] = $date->getTimestamp(); + $event['Event']['timestamp'] = $timestamp ?: time(); if ($proposalLock) { $event['Event']['proposal_email_lock'] = 0; + $fields[] = 'proposal_email_lock'; } $event['Event']['unpublishAction'] = true; - return $this->save($event); + return $this->save($event, true, $fields); } /** * @param array $user - * @param string $scriptDir - * @param string $filename + * @param string $file Path * @param string $stix_version * @param string $original_file * @param bool $publish * @return int|string|array * @throws JsonException + * @throws InvalidArgumentException + * @throws Exception */ - public function upload_stix(array $user, $scriptDir, $filename, $stix_version, $original_file, $publish) + public function upload_stix(array $user, $file, $stix_version, $original_file, $publish) { - $tempFilePath = $scriptDir . DS . 'tmp' . DS . $filename; + $scriptDir = APP . 'files' . DS . 'scripts'; if ($stix_version == '2') { $scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py'; - $shell_command = $this->getPythonVersion() . ' ' . $scriptFile . ' ' . $tempFilePath; - $output_path = $tempFilePath . '.stix2'; + $output_path = $file . '.stix2'; $stix_version = "STIX 2.0"; } elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') { $scriptFile = $scriptDir . DS . 'stix2misp.py'; - $shell_command = $this->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename; - $output_path = $tempFilePath . '.json'; + $output_path = $file . '.json'; $stix_version = "STIX 1.1"; } else { - throw new MethodNotAllowedException('Invalid STIX version'); + throw new InvalidArgumentException('Invalid STIX version'); } - $shell_command .= ' ' . escapeshellarg(Configure::read('MISP.default_event_distribution')) . ' ' . escapeshellarg(Configure::read('MISP.default_attribute_distribution')); - $synonymsToTagNames = $this->__getTagNamesFromSynonyms($scriptDir); - if ($synonymsToTagNames) { - $shell_command .= ' ' . $synonymsToTagNames; - } - $shell_command .= ' 2>' . APP . 'tmp/logs/exec-errors.log'; - $result = shell_exec($shell_command); + + $shell_command = [ + ProcessTool::pythonBin(), + $scriptFile, + $file, + Configure::read('MISP.default_event_distribution'), + Configure::read('MISP.default_attribute_distribution'), + $this->__getTagNamesFromSynonyms($scriptDir), + ]; + + $result = ProcessTool::execute($shell_command, null, true); $result = preg_split("/\r\n|\n|\r/", trim($result)); $result = trim(end($result)); - $tempFile = file_get_contents($tempFilePath); - unlink($tempFilePath); + $tempFile = file_get_contents($file); + unlink($file); if ($result === '1') { - $data = file_get_contents($output_path); - if ($data === false) { - throw new Exception("Could not get content of `$output_path` file."); - } + $data = FileAccessTool::readAndDelete($output_path); $data = $this->jsonDecode($data); if (empty($data['Event'])) { $data = array('Event' => $data); } - unlink($output_path); $created_id = false; $validationIssues = false; $result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues); @@ -5933,26 +5936,31 @@ class Event extends AppModel } return $validationIssues; } else if ($result === '2') { - $response = __('Issues while loading the stix file. '); + $response = __('Issues while loading the stix file.'); } elseif ($result === '3') { - $response = __('Issues with the maec library. '); + $response = __('Issues with the maec library.'); } else { - $response = __('Issues executing the ingestion script or invalid input. '); + $response = __('Issues executing the ingestion script or invalid input.'); } if (!$user['Role']['perm_site_admin']) { - $response .= __('Please ask your administrator to '); + $response .= ' ' . __('Please ask your administrator to'); } else { - $response .= __('Please '); + $response .= ' ' . __('Please'); } $response .= ' ' . __('check whether the dependencies for STIX are met via the diagnostic tool.'); return $response; } + /** + * @param string $scriptDir + * @return string + * @throws Exception + */ private function __getTagNamesFromSynonyms($scriptDir) { $synonymsToTagNames = $scriptDir . DS . 'tmp' . DS . 'synonymsToTagNames.json'; if (!file_exists($synonymsToTagNames) || (time() - filemtime($synonymsToTagNames)) > 600) { - if (empty($this->GalaxyCluster)) { + if (!isset($this->GalaxyCluster)) { $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); } $clusters = $this->GalaxyCluster->find('all', array( @@ -5971,21 +5979,19 @@ class Event extends AppModel 'conditions' => array('key' => 'synonyms') )); $idToSynonyms = array(); - foreach($synonyms as $synonym) { + foreach ($synonyms as $synonym) { $idToSynonyms[$synonym['GalaxyElement']['galaxy_cluster_id']][] = $synonym['GalaxyElement']['value']; } $mapping = array(); - foreach($clusters as $cluster) { + foreach ($clusters as $cluster) { $mapping[$cluster['GalaxyCluster']['value']][] = $cluster['GalaxyCluster']['tag_name']; if (!empty($idToSynonyms[$cluster['GalaxyCluster']['id']])) { - foreach($idToSynonyms[$cluster['GalaxyCluster']['id']] as $synonym) { + foreach ($idToSynonyms[$cluster['GalaxyCluster']['id']] as $synonym) { $mapping[$synonym][] = $cluster['GalaxyCluster']['tag_name']; } } } - $file = new File($synonymsToTagNames, true, 0644); - $file->write(json_encode($mapping)); - $file->close(); + FileAccessTool::writeToFile($synonymsToTagNames, JsonTool::encode($mapping)); } return $synonymsToTagNames; } @@ -5993,27 +5999,31 @@ class Event extends AppModel public function enrichmentRouter($options) { if (Configure::read('MISP.background_jobs')) { + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'prio', - 'job_type' => 'enrichment', - 'job_input' => 'Event ID: ' . $options['event_id'] . ' modules: ' . json_encode($options['modules']), - 'status' => 0, - 'retries' => 0, - 'org_id' => $options['user']['org_id'], - 'org' => $options['user']['Organisation']['name'], - 'message' => 'Enriching event.', + $jobId = $job->createJob( + $options['user'], + Job::WORKER_PRIO, + 'enrichment', + 'Event ID: ' . $options['event_id'] . ' modules: ' . json_encode($options['modules']), + 'Enriching event.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'prio', - 'EventShell', - array('enrichment', $options['user']['id'], $options['event_id'], json_encode($options['modules']), $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'enrichment', + $options['user']['id'], + $options['event_id'], + json_encode($options['modules']), + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return true; } else { $result = $this->enrichment($options); @@ -6775,7 +6785,13 @@ class Event extends AppModel if (Configure::read('MISP.background_jobs') && count($attributes) > 5) { // on background process just big attributes batch /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->createJob($user, Job::WORKER_PRIO, "process_freetext_data", 'Event: ' . $id, 'Processing...'); + $jobId = $job->createJob( + $user, + Job::WORKER_PRIO, + "process_freetext_data", + 'Event: ' . $id, + 'Processing...' + ); $tempData = array( 'user' => $user, @@ -6784,18 +6800,23 @@ class Event extends AppModel 'default_comment' => $default_comment, 'proposals' => $proposals, 'adhereToWarninglists' => $adhereToWarninglists, - 'jobId' => $job->id, + 'jobId' => $jobId, ); try { - $filePath = FileAccessTool::writeToTempFile(json_encode($tempData)); - $process_id = CakeResque::enqueue( - Job::WORKER_PRIO, - 'EventShell', - array('processfreetext', $filePath), - true + $filePath = FileAccessTool::writeToTempFile(JsonTool::encode($tempData)); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'processfreetext', + $filePath + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return 'Freetext ingestion queued for background processing. Attributes will be added to the event as they are being processed.'; } catch (Exception $e) { $this->logException("Could not process freetext in background.", $e, LOG_NOTICE); @@ -6809,25 +6830,30 @@ class Event extends AppModel if (Configure::read('MISP.background_jobs')) { /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->createJob($user, Job::WORKER_PRIO, "process_module_results_data", 'Event: ' . $id, 'Processing...'); + $jobId = $job->createJob($user, Job::WORKER_PRIO, "process_module_results_data", 'Event: ' . $id, 'Processing...'); $tempData = array( 'user' => $user, 'misp_format' => $resolved_data, 'id' => $id, 'default_comment' => $default_comment, - 'jobId' => $job->id + 'jobId' => $jobId ); try { - $filePath = FileAccessTool::writeToTempFile(json_encode($tempData)); - $process_id = CakeResque::enqueue( - Job::WORKER_PRIO, - 'EventShell', - array('processmoduleresult', $filePath), - true + $filePath = FileAccessTool::writeToTempFile(JsonTool::encode($tempData)); + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'processmoduleresult', + $filePath + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return 'Module results ingestion queued for background processing. Related data will be added to the event as it is being processed.'; } catch (Exception $e) { $this->logException("Could not process module results in background.", $e, LOG_NOTICE); @@ -6836,6 +6862,123 @@ class Event extends AppModel return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment); } + /** + * Attach references to objects faster than CakePHP. + * @param array $events + */ + private function __attachReferences(array &$events) + { + $eventIds = []; + foreach ($events as $event) { + if (!empty($event['Object'])) { + $eventIds[] = $event['Event']['id']; // event contains objects + } + } + if (!empty($eventIds)) { + // Do not fetch fields that we already know to reduce memory usage + $schema = $this->Object->ObjectReference->schema(); + unset($schema['event_id']); + unset($schema['source_uuid']); + + $references = $this->Object->ObjectReference->find('all', [ + 'conditions' => ['ObjectReference.event_id' => $eventIds], + 'fields' => array_keys($schema), + 'recursive' => -1, + ]); + } + if (empty($references)) { + // Assign empty object reference object + foreach ($events as &$event) { + foreach ($event['Object'] as &$object) { + $object['ObjectReference'] = []; + } + } + return; + } + $referencesForObject = []; + foreach ($references as $reference) { + $referencesForObject[$reference['ObjectReference']['object_id']][] = $reference['ObjectReference']; + } + $fieldsToCopy = array( + 'common' => array('distribution', 'sharing_group_id', 'uuid'), + 'Attribute' => array('value', 'type', 'category', 'to_ids'), + 'Object' => array('name', 'meta-category') + ); + foreach ($events as &$event) { + $eventIdCache = []; + foreach ($event['Object'] as &$object) { + $objectReferences = $referencesForObject[$object['id']] ?? []; + foreach ($objectReferences as &$reference) { + $reference['event_id'] = $event['Event']['id']; + $reference['source_uuid'] = $object['uuid']; + // find referenced object in current event + $type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object'; + // construct array with ID in key, so we can search attributes and objects by ID faster + if (!isset($eventIdCache[$type])) { + $eventIdCache[$type] = array_column($event[$type], null, 'id'); + } + $found = $eventIdCache[$type][$reference['referenced_id']] ?? null; + + if ($found) { + // copy requested fields + $copied = []; + foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) { + $copied[$field] = $found[$field]; + } + $reference[$type] = $copied; + } else { // object / attribute might be from an extended event + $otherEventText = __('%s from another event', $type); + $reference[$type] = [ + 'name' => '', + 'meta-category' => $otherEventText, + 'category' => $otherEventText, + 'type' => '', + 'value' => '', + 'uuid' => $reference['referenced_uuid'] + ]; + } + } + $object['ObjectReference'] = $objectReferences; + } + } + } + + /** + * Faster way how to attach tags to events that integrated in CakePHP. + * @param array $events + * @param bool $excludeLocalTags + */ + private function __attachAttributeTags(array &$events, $excludeLocalTags = false) + { + $eventIds = array_column(array_column($events, 'Event'), 'id'); + $conditions = ['AttributeTag.event_id' => $eventIds]; + if ($excludeLocalTags) { + $conditions['AttributeTag.local'] = false; + } + $ats = $this->Attribute->AttributeTag->find('all', [ + 'conditions' => $conditions, + 'fields' => ['AttributeTag.attribute_id', 'AttributeTag.tag_id', 'AttributeTag.local'], // we don't need id or event_id + 'recursive' => -1, + ]); + if (empty($ats)) { + foreach ($events as &$event) { + foreach ($event['Attribute'] as &$attribute) { + $attribute['AttributeTag'] = []; + } + } + return; + } + $atForAttributes = []; + foreach ($ats as $at) { + $atForAttributes[$at['AttributeTag']['attribute_id']][] = $at['AttributeTag']; + } + foreach ($events as &$event) { + foreach ($event['Attribute'] as &$attribute) { + $attribute['AttributeTag'] = $atForAttributes[$attribute['id']] ?? []; + } + } + } + /** * Get tag from cache by given ID. * @@ -6872,21 +7015,19 @@ class Event extends AppModel if (!empty($event['Attribute'])) { foreach ($event['Attribute'] as $attribute) { - if (!empty($attribute['AttributeTag'])) { - foreach ($attribute['AttributeTag'] as $attributeTag) { - $tagIds[$attributeTag['tag_id']] = true; - } + foreach ($attribute['AttributeTag'] as $attributeTag) { + $tagIds[$attributeTag['tag_id']] = true; } } } - $notCachedTags = array_diff(array_keys($tagIds), isset($this->assetCache['tags']) ? array_keys($this->assetCache['tags']) : []); + $notCachedTags = array_diff_key($tagIds, isset($this->assetCache['tags']) ? $this->assetCache['tags'] : []); if (empty($notCachedTags)) { return; } - $conditions = ['id' => $notCachedTags]; + $conditions = ['Tag.id' => array_keys($notCachedTags)]; if ($justExportable) { - $conditions['exportable'] = 1; + $conditions['Tag.exportable'] = 1; } $tags = $this->EventTag->Tag->find('all', [ 'recursive' => -1, @@ -6938,52 +7079,17 @@ class Event extends AppModel } /** - * Attach referenced object to ObjectReference. Since reference can be just to attribute or object in the same event, - * we just find proper element in event. - * - * @param array $event - * @param array $fields + * @param array $user + * @param string $returnFormat + * @param array $filters + * @param bool $paramsOnly + * @param int|false $jobId + * @param int $elementCounter + * @param bool $renderView + * @return TmpFileTool + * @throws Exception */ - private function __attachReferences(array &$event, array $fields) - { - if (!isset($event['Object'])) { - return; - } - foreach ($event['Object'] as $k => $object) { - foreach ($object['ObjectReference'] as $k2 => $reference) { - // find referenced object in current event - $type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object'; - $found = null; - foreach ($event[$type] as $o) { - if ($o['id'] == $reference['referenced_id']) { - $found = $o; - break; - } - } - - if ($found) { - // copy requested fields - $reference = []; - foreach (array_merge($fields['common'], $fields[$type]) as $field) { - $reference[$field] = $found[$field]; - } - $event['Object'][$k]['ObjectReference'][$k2][$type] = $reference; - } else { // object / attribute might be from an extended event - $otherEventText = __('%s from another event', $reference['referenced_type'] == 0 ? 'Attribute' : 'Object'); - $event['Object'][$k]['ObjectReference'][$k2][$type] = [ - 'name' => '', - 'meta-category' => $otherEventText, - 'category' => $otherEventText, - 'type' => '', - 'value' => '', - 'uuid' => $reference['referenced_uuid'] - ]; - } - } - } - } - - public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false, &$elementCounter = 0, &$renderView = false) + public function restSearch(array $user, $returnFormat, $filters, $paramsOnly = false, $jobId = false, &$elementCounter = 0, &$renderView = false) { if (!isset($this->validFormats[$returnFormat][1])) { throw new NotFoundException('Invalid output format.'); @@ -6996,7 +7102,7 @@ class Event extends AppModel $this->Job->id = $jobId; } - if (!empty($exportTool->use_default_filters)) { + if (method_exists($exportTool, 'setDefaultFilters')) { $exportTool->setDefaultFilters($filters); } @@ -7095,9 +7201,12 @@ class Event extends AppModel } } } - unset($result); - unset($temp); - $tmpfile->write($exportTool->footer($exportToolParams)); + $footer = $exportTool->footer($exportToolParams); + if ($footer instanceof TmpFileTool) { + return $footer; // Some exports returns TmpFileTool with all data when ends, so we can just pass the file as output + } + + $tmpfile->write($footer); return $tmpfile; } @@ -7207,9 +7316,8 @@ class Event extends AppModel 'disable_correlation' => true ) ); - foreach ($attributes as $attribute) { - $this->Attribute->create(); - $this->Attribute->save($attribute); + if (!$this->Attribute->saveMany($attributes)) { + throw new Exception("Could not save attributes for original file because of validation errors:" . json_encode($this->Attribute->validationErrors)); } return true; } @@ -7326,7 +7434,7 @@ class Event extends AppModel public function addFiltersFromUserSettings($user, $filters) { $this->UserSetting = ClassRegistry::init('UserSetting'); - $defaultParameters = $this->UserSetting->getDefaulRestSearchParameters($user); + $defaultParameters = $this->UserSetting->getDefaultRestSearchParameters($user); $filters = array_replace_recursive($defaultParameters, $filters); return $filters; } diff --git a/app/Model/EventGraph.php b/app/Model/EventGraph.php index 1e6a203ac..b2861c7da 100644 --- a/app/Model/EventGraph.php +++ b/app/Model/EventGraph.php @@ -30,7 +30,7 @@ class EventGraph extends AppModel public $validate = array( 'network_json' => array( - 'rule' => array('isValidJson'), + 'rule' => 'valueIsJson', 'message' => 'The provided eventGraph is not a valid json format', 'required' => true, ), @@ -44,16 +44,6 @@ class EventGraph extends AppModel return true; } - public function isValidJson($fields) - { - $text = $fields['network_json']; - $check = json_decode($text); - if ($check === null) { - return false; - } - return true; - } - public function getPictureData($eventGraph) { $b64 = str_replace('data:image/png;base64,', '', $eventGraph['EventGraph']['preview_img']); diff --git a/app/Model/EventTag.php b/app/Model/EventTag.php index 99b00ebfc..b3a74b9df 100644 --- a/app/Model/EventTag.php +++ b/app/Model/EventTag.php @@ -141,6 +141,50 @@ class EventTag extends AppModel return false; } + /** + * Find all of the event Ids that belong to the accepted tags and the rejected tags + * @param array $accept + * @param array $reject + * @return array[] + */ + public function fetchEventTagIds(array $accept = array(), array $reject = array()) + { + $acceptIds = array(); + $rejectIds = array(); + if (!empty($accept)) { + $acceptIds = $this->findEventIdsByTagNames($accept); + if (empty($acceptIds)) { + $acceptIds = [-1]; + } + } + if (!empty($reject)) { + $rejectIds = $this->findEventIdsByTagNames($reject); + } + return array($acceptIds, $rejectIds); + } + + /** + * @param array $tagIdsOrNames + * @return array|int|null + */ + private function findEventIdsByTagNames(array $tagIdsOrNames) + { + $conditions = []; + foreach ($tagIdsOrNames as $tagIdOrName) { + if (is_numeric($tagIdOrName)) { + $conditions[] = array('Tag.id' => $tagIdOrName); + } else { + $conditions[] = array('LOWER(Tag.name)' => mb_strtolower($tagIdOrName)); + } + } + return $this->find('column', array( + 'recursive' => -1, + 'contain' => 'Tag', + 'conditions' => ['OR' => $conditions], + 'fields' => ['EventTag.event_id'], + )); + } + public function getSortedTagList($context = false) { $conditions = array(); diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 87fd4aa81..fb63bde87 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -2,7 +2,7 @@ App::uses('AppModel', 'Model'); App::uses('RandomTool', 'Tools'); App::uses('TmpFileTool', 'Tools'); -App::uses('FileAccessTool', 'Tools'); +App::uses('AttributeValidationTool', 'Tools'); class Feed extends AppModel { @@ -190,7 +190,7 @@ class Feed extends AppModel /** * @param array $feed * @param HttpSocket|null $HttpSocket Null can be for local feed - * @return Generator + * @return Generator * @throws Exception */ public function getCache(array $feed, HttpSocket $HttpSocket = null) @@ -355,13 +355,10 @@ class Feed extends AppModel * @param array $feed * @param HttpSocket|null $HttpSocket Null can be for local feed * @param string $type - * @param int|string $page - * @param int $limit - * @param array $params * @return array|bool * @throws Exception */ - public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) + public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext') { if ($this->isFeedLocal($feed)) { $feedUrl = $feed['Feed']['url']; @@ -386,18 +383,9 @@ class Feed extends AppModel } $resultArray = $complexTypeTool->checkComplexRouter($data, $type, $settings); $this->Attribute = ClassRegistry::init('Attribute'); - foreach ($resultArray as $key => $value) { - $resultArray[$key]['category'] = $this->Attribute->typeDefinitions[$value['default_type']]['default_category']; - } - App::uses('CustomPaginationTool', 'Tools'); - $customPagination = new CustomPaginationTool(); - $params = $customPagination->createPaginationRules($resultArray, array('page' => $page, 'limit' => $limit), 'Feed', $sort = false); - if (!empty($page) && $page != 'all') { - $start = ($page - 1) * $limit; - if ($start > count($resultArray)) { - return false; - } - $resultArray = array_slice($resultArray, $start, $limit); + $typeDefinitions = $this->Attribute->typeDefinitions; + foreach ($resultArray as &$value) { + $value['category'] = $typeDefinitions[$value['default_type']]['default_category']; } return $resultArray; } @@ -478,7 +466,7 @@ class Feed extends AppModel } $compositeTypes = $this->Attribute->getCompositeTypes(); - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); $hashTable = []; $redisResultToAttributePosition = []; @@ -550,7 +538,7 @@ class Feed extends AppModel foreach ($sources as $source) { $sourceId = $source[$scope]['id']; - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); foreach ($hitIds as $k) { $redis->sismember($cachePrefix . $sourceId, $hashTable[$k]); } @@ -574,7 +562,7 @@ class Feed extends AppModel // Append also exact MISP feed or server event UUID // TODO: This can be optimised in future to do that in one pass if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) { - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); $eventUuidHitPosition = []; foreach ($hitIds as $sourceHitPos => $k) { if ($sourceHits[$sourceHitPos]) { @@ -648,7 +636,7 @@ class Feed extends AppModel try { $redis = $this->setupRedisWithException(); - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); $cachePrefix = 'misp:' . strtolower($scope) . '_cache:'; foreach ($sources as $source) { $pipe->exists($cachePrefix . $source[$scope]['id']); @@ -1055,19 +1043,32 @@ class Feed extends AppModel return $success; } + /** + * @param int $feedId + * @param array $user + * @param int|false $jobId + * @return array|bool + * @throws Exception + */ public function downloadFromFeedInitiator($feedId, $user, $jobId = false) { - $this->id = $feedId; - $this->read(); - if (isset($this->data['Feed']['settings']) && !empty($this->data['Feed']['settings'])) { - $this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true); + $feed = $this->find('first', array( + 'conditions' => ['Feed.id' => $feedId], + 'recursive' => -1, + )); + if (empty($feed)) { + throw new Exception("Feed with ID $feedId not found."); } - $HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket(); - if ($this->data['Feed']['source_format'] === 'misp') { + if (!empty($feed['Feed']['settings'])) { + $feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true); + } + + $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket(); + if ($feed['Feed']['source_format'] === 'misp') { $this->jobProgress($jobId, 'Fetching event manifest.'); try { - $actions = $this->getNewEventUuids($this->data, $HttpSocket); + $actions = $this->getNewEventUuids($feed, $HttpSocket); } catch (Exception $e) { $this->logException("Could not get new event uuids for feed $feedId.", $e); $this->jobProgress($jobId, 'Could not fetch event manifest. See error log for more details.'); @@ -1080,12 +1081,12 @@ class Feed extends AppModel $total = count($actions['add']) + count($actions['edit']); $this->jobProgress($jobId, __("Fetching %s events.", $total)); - $result = $this->downloadFromFeed($actions, $this->data, $HttpSocket, $user, $jobId); - $this->__cleanupFile($this->data, '/manifest.json'); + $result = $this->downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId); + $this->__cleanupFile($feed, '/manifest.json'); } else { $this->jobProgress($jobId, 'Fetching data.'); try { - $temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all'); + $temp = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); } catch (Exception $e) { $this->logException("Could not get freetext feed $feedId", $e); $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.'); @@ -1105,17 +1106,18 @@ class Feed extends AppModel 'to_ids' => $value['to_ids'] ); } + unset($temp); $this->jobProgress($jobId, 'Saving data.', 50); try { - $result = $this->saveFreetextFeedData($this->data, $data, $user, $jobId); + $result = $this->saveFreetextFeedData($feed, $data, $user, $jobId); } catch (Exception $e) { $this->logException("Could not save freetext feed data for feed $feedId.", $e); return false; } - $this->__cleanupFile($this->data, ''); + $this->__cleanupFile($feed, ''); } return $result; } @@ -1201,7 +1203,7 @@ class Feed extends AppModel // Because some types can be saved in modified version (for example, IPv6 address is convert to compressed // format, we should also check if current event contains modified value. - $modifiedValue = $this->Event->Attribute->modifyBeforeValidation($dataPoint['type'], $dataPoint['value']); + $modifiedValue = AttributeValidationTool::modifyBeforeValidation($dataPoint['type'], $dataPoint['value']); if (isset($existsAttributesValueToId[$modifiedValue])) { unset($data[$k]); unset($existsAttributesValueToId[$modifiedValue]); @@ -1318,7 +1320,7 @@ class Feed extends AppModel return $feeds; } - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); foreach ($feeds as $feed) { $pipe->get('misp:feed_cache_timestamp:' . $feed['Feed']['id']); } @@ -1329,14 +1331,28 @@ class Feed extends AppModel return $feeds; } + /** + * @param array $feed + * @param Redis $redis + * @param int|false $jobId + * @return bool + */ private function __cacheFeed($feed, $redis, $jobId = false) { $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket(); if ($feed['Feed']['source_format'] === 'misp') { - return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId); + $result = true; + if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) { + $result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId); + } } else { - return $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId); + $result = $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId); } + + if ($result) { + $redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time()); + } + return $result; } /** @@ -1353,7 +1369,7 @@ class Feed extends AppModel $this->jobProgress($jobId, __("Feed %s: Fetching.", $feedId)); try { - $values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all'); + $values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format']); } catch (Exception $e) { $this->logException("Could not get freetext feed $feedId", $e); $this->jobProgress($jobId, __('Could not fetch freetext feed %s. See error log for more details.', $feedId)); @@ -1365,7 +1381,7 @@ class Feed extends AppModel $redis->del('misp:feed_cache:' . $feedId); foreach (array_chunk($md5Values, 5000) as $k => $chunk) { - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); if (method_exists($redis, 'sAddArray')) { $redis->sAddArray('misp:feed_cache:' . $feedId, $chunk); $redis->sAddArray('misp:feed_cache:combined', $chunk); @@ -1378,10 +1394,16 @@ class Feed extends AppModel $pipe->exec(); $this->jobProgress($jobId, __('Feed %s: %s/%s values cached.', $feedId, $k * 5000, count($md5Values))); } - $redis->set('misp:feed_cache_timestamp:' . $feedId, time()); return true; } + /** + * @param array $feed + * @param Redis $redis + * @param HttpSocket|null $HttpSocket + * @param false $jobId + * @return bool + */ private function __cacheMISPFeedTraditional($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false) { $feedId = $feed['Feed']['id']; @@ -1395,6 +1417,7 @@ class Feed extends AppModel $redis->del('misp:feed_cache:' . $feedId); $k = 0; + $this->Attribute = ClassRegistry::init('Attribute'); foreach ($manifest as $uuid => $event) { try { $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket); @@ -1404,11 +1427,10 @@ class Feed extends AppModel } if (!empty($event['Event']['Attribute'])) { - $this->Attribute = ClassRegistry::init('Attribute'); - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); foreach ($event['Event']['Attribute'] as $attribute) { if (!in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) { - if (in_array($attribute['type'], $this->Attribute->getCompositeTypes())) { + if (in_array($attribute['type'], $this->Attribute->getCompositeTypes(), true)) { $value = explode('|', $attribute['value']); if (in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true)) { unset($value[1]); @@ -1436,6 +1458,13 @@ class Feed extends AppModel return true; } + /** + * @param array $feed + * @param Redis $redis + * @param HttpSocket|null $HttpSocket + * @param false $jobId + * @return bool + */ private function __cacheMISPFeedCache($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false) { $feedId = $feed['Feed']['id']; @@ -1447,7 +1476,7 @@ class Feed extends AppModel return false; } - $pipe = $redis->multi(Redis::PIPELINE); + $pipe = $redis->pipeline(); $redis->del('misp:feed_cache:' . $feedId); foreach ($cache as $v) { list($hash, $eventUuid) = $v; @@ -1460,18 +1489,6 @@ class Feed extends AppModel return true; } - private function __cacheMISPFeed($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false) - { - $result = true; - if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) { - $result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId); - } - if ($result) { - $redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time()); - } - return $result; - } - public function compareFeeds($id = false) { $redis = $this->setupRedis(); @@ -1747,19 +1764,12 @@ class Feed extends AppModel { $hits = array(); $this->Server = ClassRegistry::init('Server'); - $result['Server'] = $this->Server->find('all', array( - 'conditions' => array( - 'caching_enabled' => 1 - ), - 'recursive' => -1, - 'fields' => array('Server.id', 'Server.name', 'Server.url') - )); $redis = $this->setupRedis(); $is_array = true; if (!is_array($value)) { $is_array = false; if (empty($value)) { - // old behaviour allowd for empty values to return all data + // old behaviour allowed for empty values to return all data $value = [false]; } else { $value = [$value]; @@ -1821,7 +1831,6 @@ class Feed extends AppModel } } if ($v === false || $redis->sismember('misp:server_cache:combined', md5($v))) { - $this->Server = ClassRegistry::init('Server'); $servers = $this->Server->find('all', array( 'conditions' => array( 'caching_enabled' => 1 @@ -1932,16 +1941,14 @@ class Feed extends AppModel $contentType = $response->getHeader('Content-Type'); if ($contentType === 'application/zip') { - $zipFile = new File($this->tempFileName()); - $zipFile->write($response->body); - $zipFile->close(); + $zipFilePath = FileAccessTool::writeToTempFile($response->body); try { - $response->body = $this->unzipFirstFile($zipFile); + $response->body = $this->unzipFirstFile($zipFilePath); } catch (Exception $e) { throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}"); } finally { - $zipFile->delete(); + FileAccessTool::deleteFile($zipFilePath); } } @@ -2047,18 +2054,18 @@ class Feed extends AppModel } /** - * @param File $zipFile + * @param string $zipFile * @return string Uncompressed data * @throws Exception */ - private function unzipFirstFile(File $zipFile) + private function unzipFirstFile($zipFile) { if (!class_exists('ZipArchive')) { throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.'); } $zip = new ZipArchive(); - $result = $zip->open($zipFile->pwd()); + $result = $zip->open($zipFile); if ($result !== true) { $errorCodes = [ ZipArchive::ER_EXISTS => 'file already exists', @@ -2086,18 +2093,12 @@ class Feed extends AppModel $zip->close(); - $destinationFile = $this->tempFileName(); - $result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile); + $destinationFile = FileAccessTool::createTempFile(); + $result = copy("zip://$zipFile#$filename", $destinationFile); if ($result === false) { throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted."); } - $unzipped = new File($destinationFile); - $data = $unzipped->read(); - if ($data === false) { - throw new Exception("Couldn't read extracted file content."); - } - $unzipped->delete(); - return $data; + return FileAccessTool::readAndDelete($destinationFile); } } diff --git a/app/Model/Galaxy.php b/app/Model/Galaxy.php index 1b0860c39..a14da5c44 100644 --- a/app/Model/Galaxy.php +++ b/app/Model/Galaxy.php @@ -57,17 +57,14 @@ class Galaxy extends AppModel /** * @param bool $force * @return array Galaxy type => Galaxy ID - * @throws JsonException + * @throws Exception */ private function __load_galaxies($force = false) { - $dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'galaxies'); - $files = $dir->find('.*\.json'); + $files = new GlobIterator(APP . 'files' . DS . 'misp-galaxy' . DS . 'galaxies' . DS . '*.json'); $galaxies = array(); foreach ($files as $file) { - $file = new File($dir->pwd() . DS . $file); - $galaxies[] = $this->jsonDecode($file->read()); - $file->close(); + $galaxies[] = FileAccessTool::readJsonFromFile($file->getPathname()); } $existingGalaxies = $this->find('all', array( 'fields' => array('uuid', 'version', 'id', 'icon'), @@ -157,6 +154,10 @@ class Galaxy extends AppModel $relations = []; $elements = []; $this->GalaxyCluster->bulkEntry = true; + + // Start transaction + $this->getDataSource()->begin(); + foreach ($cluster_package['values'] as $cluster) { if (empty($cluster['version'])) { $cluster['version'] = 1; @@ -182,7 +183,12 @@ class Galaxy extends AppModel $cluster_to_save['published'] = false; $cluster_to_save['org_id'] = 0; $cluster_to_save['orgc_id'] = 0; - $result = $this->GalaxyCluster->save($cluster_to_save, false); + // We are already in transaction + $result = $this->GalaxyCluster->save($cluster_to_save, ['atomic' => false, 'validate' => false]); + if (!$result) { + $this->log("Could not save galaxy cluster with UUID {$cluster_to_save['uuid']}."); + continue; + } $galaxyClusterId = $this->GalaxyCluster->id; if (isset($cluster['meta'])) { foreach ($cluster['meta'] as $key => $value) { @@ -206,7 +212,7 @@ class Galaxy extends AppModel $elements[] = array( $galaxyClusterId, $key, - strval($v) + (string)$v ); } } @@ -221,24 +227,26 @@ class Galaxy extends AppModel 'referenced_galaxy_cluster_type' => $relation['type'], 'default' => true, 'distribution' => 3, - 'tags' => !empty($relation['tags']) ? $relation['tags'] : [] + 'tags' => $relation['tags'] ?? [] ]; } } } + + // Commit transaction + $this->getDataSource()->commit(); + return [$elements, $relations]; } public function update($force = false) { $galaxies = $this->__load_galaxies($force); - $dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'clusters'); - $files = $dir->find('.*\.json'); + $files = new GlobIterator(APP . 'files' . DS . 'misp-galaxy' . DS . 'clusters' . DS . '*.json'); $force = (bool)$force; + $allRelations = []; foreach ($files as $file) { - $file = new File($dir->pwd() . DS . $file); - $cluster_package = $this->jsonDecode($file->read()); - $file->close(); + $cluster_package = FileAccessTool::readJsonFromFile($file->getPathname()); if (!isset($galaxies[$cluster_package['type']])) { continue; } @@ -253,10 +261,13 @@ class Galaxy extends AppModel $fields = array('galaxy_cluster_id', 'key', 'value'); $db->insertMulti('galaxy_elements', $fields, $elements); } - if (!empty($relations)) { - $this->GalaxyCluster->GalaxyClusterRelation->bulkSaveRelations($relations); - } + $allRelations = array_merge($allRelations, $relations); } + // Save relation as last part when all clusters are created + if (!empty($allRelations)) { + $this->GalaxyCluster->GalaxyClusterRelation->bulkSaveRelations($allRelations); + } + // Probably unnecessary anymore $this->GalaxyCluster->generateMissingRelations(); return true; } @@ -348,11 +359,15 @@ class Galaxy extends AppModel } else { $local = 0; } + $cluster_alias = $this->GalaxyCluster->alias; + $galaxy_alias = $this->alias; $cluster = $this->GalaxyCluster->fetchGalaxyClusters($user, array( 'first' => true, - 'conditions' => array('id' => $cluster_id), - 'fields' => array('tag_name', 'id', 'value'), + 'conditions' => array("${cluster_alias}.id" => $cluster_id), + 'contain' => array('Galaxy'), + 'fields' => array('tag_name', 'id', 'value', "${galaxy_alias}.local_only"), ), $full=false); + if (empty($cluster)) { throw new NotFoundException(__('Invalid Galaxy cluster')); } @@ -368,7 +383,11 @@ class Galaxy extends AppModel throw new NotFoundException(__('Invalid %s.', $target_type)); } $target = $target[0]; - $tag_id = $this->Tag->captureTag(array('name' => $cluster['GalaxyCluster']['tag_name'], 'colour' => '#0088cc', 'exportable' => 1), $user, true); + $local_only = $cluster['GalaxyCluster']['Galaxy']['local_only']; + if ($local_only && !$local) { + throw new MethodNotAllowedException(__("This Cluster can only be attached in a local scope")); + } + $tag_id = $this->Tag->captureTag(array('name' => $cluster['GalaxyCluster']['tag_name'], 'colour' => '#0088cc', 'exportable' => 1, 'local_only' => $local_only), $user, true); $existingTag = $this->Tag->$connectorModel->find('first', array('conditions' => array($target_type . '_id' => $target_id, 'tag_id' => $tag_id))); if (!empty($existingTag)) { return 'Cluster already attached.'; diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index f59b779e1..ab074e107 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -77,6 +77,7 @@ class GalaxyCluster extends AppModel private $__clusterCache = array(); private $deletedClusterUUID; + public $bulkEntry = false; public $hasMany = array( 'GalaxyElement' => array('dependent' => true), @@ -97,20 +98,20 @@ class GalaxyCluster extends AppModel public function beforeValidate($options = array()) { - parent::beforeValidate(); - if (!isset($this->data['GalaxyCluster']['description'])) { - $this->data['GalaxyCluster']['description'] = ''; + $cluster = &$this->data['GalaxyCluster']; + if (!isset($cluster['description'])) { + $cluster['description'] = ''; } - if (isset($this->data['GalaxyCluster']['distribution']) && $this->data['GalaxyCluster']['distribution'] != 4) { - $this->data['GalaxyCluster']['sharing_group_id'] = null; + if (isset($cluster['distribution']) && $cluster['distribution'] != 4) { + $cluster['sharing_group_id'] = null; } - if (!isset($this->data['GalaxyCluster']['published'])) { - $this->data['GalaxyCluster']['published'] = false; + if (!isset($cluster['published'])) { + $cluster['published'] = false; } - if (!isset($this->data['GalaxyCluster']['authors']) || is_null($this->data['GalaxyCluster']['authors'])) { - $this->data['GalaxyCluster']['authors'] = ''; - } elseif (is_array($this->data['GalaxyCluster']['authors'])) { - $this->data['GalaxyCluster']['authors'] = json_encode($this->data['GalaxyCluster']['authors']); + if (!isset($cluster['authors']) || $cluster['authors'] === null) { + $cluster['authors'] = ''; + } elseif (is_array($cluster['authors'])) { + $cluster['authors'] = json_encode($cluster['authors']); } return true; } @@ -118,24 +119,24 @@ class GalaxyCluster extends AppModel public function afterFind($results, $primary = false) { foreach ($results as $k => $result) { - if (isset($results[$k][$this->alias]['authors'])) { + if (isset($result[$this->alias]['authors'])) { $results[$k][$this->alias]['authors'] = json_decode($results[$k][$this->alias]['authors'], true); } - if (isset($results[$k][$this->alias]['distribution']) && $results[$k][$this->alias]['distribution'] != 4) { + if (isset($result[$this->alias]['distribution']) && $results[$k][$this->alias]['distribution'] != 4) { unset($results[$k]['SharingGroup']); } - if (isset($results[$k][$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) { + if (isset($result[$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) { if (isset($results[$k]['Org'])) { $results[$k]['Org'] = $this->Org->genericMISPOrganisation; } } - if (isset($results[$k][$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) { + if (isset($result[$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) { if (isset($results[$k]['Orgc'])) { $results[$k]['Orgc'] = $this->Org->genericMISPOrganisation; } } - if (!empty($results[$k]['GalaxyClusterRelation'])) { + if (!empty($result['GalaxyClusterRelation'])) { foreach ($results[$k]['GalaxyClusterRelation'] as $i => $relation) { if (isset($relation['distribution']) && $relation['distribution'] != 4) { unset($results[$k]['GalaxyClusterRelation'][$i]['SharingGroup']); @@ -149,8 +150,7 @@ class GalaxyCluster extends AppModel public function afterSave($created, $options = array()) { // Update all relations IDs that are unknown but saved - parent::afterSave($created, $options); - if (empty($this->bulkEntry)) { + if (!$this->bulkEntry) { $cluster = $this->data[$this->alias]; $cluster = $this->fetchAndSetUUID($cluster); $this->GalaxyClusterRelation->updateAll( @@ -457,32 +457,31 @@ class GalaxyCluster extends AppModel } else { return false; } - $this->Event = ClassRegistry::init('Event'); - $job_type = 'publish_cluster'; - $function = 'publish_galaxy_clusters'; - $message = 'Publishing.'; + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'prio', - 'job_type' => 'publish_galaxy_clusters', - 'job_input' => 'Cluster ID: ' . $clusterId, - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => $message + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_PRIO, + 'publish_galaxy_clusters', + 'Cluster ID: ' . $clusterId, + 'Publishing.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'prio', - 'EventShell', - array($function, $clusterId, $jobId, $user['id'], $passAlong), - true + + return $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'publish_galaxy_clusters', + $clusterId, + $jobId, + $user['id'], + $passAlong + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); - return $process_id; + } else { $result = $this->publish($cluster, $passAlong=$passAlong); return $result; @@ -1089,18 +1088,23 @@ class GalaxyCluster extends AppModel if (isset($options['list']) && $options['list']) { return $this->find('list', $params); } + if (isset($options['first']) && $options['first']) { $clusters = $this->find('first', $params); } else if (isset($options['count']) && $options['count']) { - $clusterCount = $this->find('count', $params); - return $clusterCount; + return $this->find('count', $params); } else { $clusters = $this->find('all', $params); } + if (empty($clusters)) { return $clusters; } + if (isset($options['first']) && $options['first']) { + $clusters = [$clusters]; + } + if ($full) { $clusterIds = array_column(array_column($clusters, 'GalaxyCluster'), 'id'); $targetingClusterRelations = $this->TargetingClusterRelation->fetchRelations($user, array( @@ -1116,11 +1120,15 @@ class GalaxyCluster extends AppModel $tagsToFetch = Hash::extract($clusters, "{n}.GalaxyClusterRelation.{n}.GalaxyClusterRelationTag.{n}.tag_id"); $tagsToFetch = array_merge($tagsToFetch, Hash::extract($targetingClusterRelations, "GalaxyClusterRelationTag.{n}.tag_id")); - $tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [ - 'conditions' => ['id' => array_unique($tagsToFetch)], - 'recursive' => -1, - ]); - $tags = array_column(array_column($tags, 'Tag'), null, 'id'); + if (!empty($tagsToFetch)) { + $tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [ + 'conditions' => ['id' => array_unique($tagsToFetch)], + 'recursive' => -1, + ]); + $tags = array_column(array_column($tags, 'Tag'), null, 'id'); + } else { + $tags = []; + } foreach ($targetingClusterRelations as $k => $targetingClusterRelation) { if (!empty($targetingClusterRelation['GalaxyClusterRelationTag'])) { @@ -1142,12 +1150,12 @@ class GalaxyCluster extends AppModel $sharingGroupData = $this->Event->__cacheSharingGroupData($user, false); foreach ($clusters as $i => $cluster) { if (!empty($cluster['GalaxyCluster']['sharing_group_id']) && isset($sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']])) { - $clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']]['SharingGroup']; + $clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']]; } if (isset($cluster['GalaxyClusterRelation'])) { foreach ($cluster['GalaxyClusterRelation'] as $j => $relation) { if (!empty($relation['sharing_group_id']) && isset($sharingGroupData[$relation['sharing_group_id']])) { - $clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']]['SharingGroup']; + $clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']]; } foreach ($relation['GalaxyClusterRelationTag'] as $relationTag) { if (isset($tags[$relationTag['tag_id']])) { @@ -1166,6 +1174,11 @@ class GalaxyCluster extends AppModel } $clusters[$i] = $this->arrangeData($clusters[$i]); } + + if (isset($options['first']) && $options['first']) { + return $clusters[0]; + } + return $clusters; } @@ -1662,7 +1675,7 @@ class GalaxyCluster extends AppModel { $this->Server = ClassRegistry::init('Server'); $this->Log = ClassRegistry::init('Log'); - $push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket); + $push = $this->Server->checkVersionCompatibility($server, false); if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) { return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); } diff --git a/app/Model/GalaxyClusterRelation.php b/app/Model/GalaxyClusterRelation.php index 1a5a0b646..00b69153b 100644 --- a/app/Model/GalaxyClusterRelation.php +++ b/app/Model/GalaxyClusterRelation.php @@ -3,6 +3,7 @@ App::uses('AppModel', 'Model'); /** * @property GalaxyClusterRelationTag $GalaxyClusterRelationTag + * @property GalaxyCluster $TargetCluster */ class GalaxyClusterRelation extends AppModel { @@ -66,10 +67,10 @@ class GalaxyClusterRelation extends AppModel public function afterFind($results, $primary = false) { foreach ($results as $k => $result) { - if (isset($results[$k]['TargetCluster']) && key_exists('id', $results[$k]['TargetCluster']) && is_null($results[$k]['TargetCluster']['id'])) { + if (isset($result['TargetCluster']) && key_exists('id', $result['TargetCluster']) && is_null($result['TargetCluster']['id'])) { $results[$k]['TargetCluster'] = array(); } - if (isset($results[$k]['GalaxyClusterRelation']['distribution']) && $results[$k]['GalaxyClusterRelation']['distribution'] != 4) { + if (isset($result['GalaxyClusterRelation']['distribution']) && $result['GalaxyClusterRelation']['distribution'] != 4) { unset($results[$k]['SharingGroup']); } } @@ -78,9 +79,9 @@ class GalaxyClusterRelation extends AppModel public function buildConditions($user, $clusterConditions = true) { - $this->Event = ClassRegistry::init('Event'); $conditions = []; if (!$user['Role']['perm_site_admin']) { + $this->Event = ClassRegistry::init('Event'); $alias = $this->alias; $sgids = $this->Event->cacheSgids($user, true); $gcOwnerIds = $this->SourceCluster->cacheGalaxyClusterOwnerIDs($user); @@ -346,44 +347,41 @@ class GalaxyClusterRelation extends AppModel public function bulkSaveRelations(array $relations) { - if (!isset($this->bulkCache)) { - $this->bulkCache = [ - 'tag_ids' => [] - ]; - } + // Fetch existing tags Name => ID mapping + $tagNameToId = $this->GalaxyClusterRelationTag->Tag->find('list', [ + 'fields' => ['Tag.name', 'Tag.id'], + 'callbacks' => false, + ]); + + // Fetch all cluster UUID => ID mapping + $galaxyClusterUuidToId = $this->TargetCluster->find('list', [ + 'fields' => ['uuid', 'id'], + 'callbacks' => false, + ]); + $lookupSavedIds = []; $relationTagsToSave = []; - foreach ($relations as $k => $relation) { - $relations[$k]['referenced_galaxy_cluster_id'] = 0; - $lookupSavedIds[$relation['galaxy_cluster_id']] = true; + foreach ($relations as &$relation) { + if (isset($galaxyClusterUuidToId[$relation['referenced_galaxy_cluster_uuid']])) { + $relation['referenced_galaxy_cluster_id'] = $galaxyClusterUuidToId[$relation['referenced_galaxy_cluster_uuid']]; + } else { + $relation['referenced_galaxy_cluster_id'] = 0; // referenced cluster doesn't exists + } if (!empty($relation['tags'])) { + $lookupSavedIds[$relation['galaxy_cluster_id']] = true; foreach ($relation['tags'] as $tag) { - if (!isset($this->bulkCache['tag_ids'][$tag])) { - $existingTag = $this->GalaxyClusterRelationTag->Tag->find('first', [ - 'recursive' => -1, - 'fields' => ['Tag.id'], - 'conditions' => ['Tag.name' => $tag] - ]); - if (empty($existingTag)) { - $this->GalaxyClusterRelationTag->Tag->create(); - $this->GalaxyClusterRelationTag->Tag->save([ - 'name' => $tag, - 'colour' => $this->GalaxyClusterRelationTag->Tag->random_color(), - 'exportable' => 1, - 'org_id' => 0, - 'user_id' => 0, - 'hide_tag' => Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0 - ]); - $this->bulkCache['tag_ids'][$tag] = $this->GalaxyClusterRelationTag->Tag->id; - } else { - $this->bulkCache['tag_ids'][$tag] = $existingTag['Tag']['id']; - } + if (!isset($tagNameToId[$tag])) { + $tagNameToId[$tag] = $this->GalaxyClusterRelationTag->Tag->quickAdd($tag); } - $relationTagsToSave[$relation['galaxy_cluster_uuid']][$relation['referenced_galaxy_cluster_uuid']][] = $this->bulkCache['tag_ids'][$tag]; + $relationTagsToSave[$relation['galaxy_cluster_uuid']][$relation['referenced_galaxy_cluster_uuid']][] = $tagNameToId[$tag]; } } } - $this->saveAll($relations); + unset($galaxyClusterUuidToId, $tagNameToId); + + $this->saveMany($relations, ['validate' => false]); // Some clusters uses invalid UUID :/ + + // Insert tags $savedRelations = $this->find('all', [ 'recursive' => -1, 'conditions' => ['galaxy_cluster_id' => array_keys($lookupSavedIds)], @@ -393,9 +391,9 @@ class GalaxyClusterRelation extends AppModel foreach ($savedRelations as $savedRelation) { $uuid1 = $savedRelation['GalaxyClusterRelation']['galaxy_cluster_uuid']; $uuid2 = $savedRelation['GalaxyClusterRelation']['referenced_galaxy_cluster_uuid']; - if (!empty($relationTagsToSave[$uuid1][$uuid2])) { - foreach ($relationTagsToSave[$uuid1][$uuid2] as $tag) { - $relation_tags[] = [$savedRelation['GalaxyClusterRelation']['id'], $tag]; + if (isset($relationTagsToSave[$uuid1][$uuid2])) { + foreach ($relationTagsToSave[$uuid1][$uuid2] as $tagId) { + $relation_tags[] = [$savedRelation['GalaxyClusterRelation']['id'], $tagId]; } } } diff --git a/app/Model/GalaxyClusterRelationTag.php b/app/Model/GalaxyClusterRelationTag.php index 0e5c1b226..b36e9b82d 100644 --- a/app/Model/GalaxyClusterRelationTag.php +++ b/app/Model/GalaxyClusterRelationTag.php @@ -1,6 +1,9 @@ array( @@ -32,42 +33,49 @@ class Job extends AppModel public function cache($type, $user) { - $extra = null; - $extra2 = null; - $shell = 'Event'; - $this->create(); - $data = array( - 'worker' => 'cache', - 'job_type' => 'cache_' . $type, - 'job_input' => $user['Role']['perm_site_admin'] ? 'All events.' : 'Events visible to: ' . $user['Organisation']['name'], - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['Role']['perm_site_admin'] ? 0 : $user['org_id'], - 'message' => 'Fetching events.', + $jobId = $this->createJob( + $user, + Job::WORKER_CACHE, + 'cache_' . $type, + $user['Role']['perm_site_admin'] ? 'All events.' : 'Events visible to: ' . $user['Organisation']['name'], + 'Fetching events.' ); - $this->save($data); - $id = $this->id; + $this->Event = ClassRegistry::init('Event'); + if (in_array($type, array_keys($this->Event->export_types)) && $type !== 'bro') { - $process_id = CakeResque::enqueue( + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::CACHE_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ 'cache', - $shell . 'Shell', - array('cache', $user['id'], $id, $type), - true + $user['id'], + $jobId, + $type + ], + true, + $jobId ); } elseif ($type === 'bro') { - $type = 'bro'; - $process_id = CakeResque::enqueue( - 'cache', - $shell . 'Shell', - array('cachebro', $user['id'], $id), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::CACHE_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'cachebro', + $user['id'], + $jobId, + $type + ], + true, + $jobId ); } else { throw new MethodNotAllowedException('Invalid export type.'); } - $this->saveField('process_id', $process_id); - return $id; + + return $jobId; } /** diff --git a/app/Model/Log.php b/app/Model/Log.php index 13a517304..0d4a7b4b9 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -240,6 +240,9 @@ class Log extends AppModel if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) { return null; } + if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) { + return null; + } throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors)); } @@ -247,6 +250,22 @@ class Log extends AppModel return $result; } + /** + * @param array|string $user + * @param string $action + * @param string $model + * @param string $title + * @param array $validationErrors + * @param array $fullObject + * @throws Exception + */ + public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject) + { + $this->log($title, LOG_WARNING); + $change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject); + $this->createLogEntry($user, $action, $model, 0, $title, $change); + } + // to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version // this function remedies a fixed upgrade bug instance by eliminating the massive number of erroneous upgrade log entries public function pruneUpdateLogs($jobId = false, $user) @@ -291,32 +310,31 @@ class Log extends AppModel )); } - public function pruneUpdateLogsRouter($user) { if (Configure::read('MISP.background_jobs')) { + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'prune_update_logs', - 'job_input' => 'All update entries', - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => 'Purging the heretic.', + $jobId = $job->createJob( + $user, + Job::WORKER_DEFAULT, + 'prune_update_logs', + 'All update entries', + 'Purging the heretic.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('prune_update_logs', $jobId, $user['id']), - true + + return $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'prune_update_logs', + $jobId, + $user['id'] + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); - return $process_id; } else { $result = $this->pruneUpdateLogs(false, $user); return $result; diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index bd7856bd5..d50ed1fa7 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -1,6 +1,8 @@ array( 'rule' => 'isUnique', 'message' => 'The UUID provided is not unique', - 'required' => true, 'on' => 'create' ), ), @@ -216,23 +217,19 @@ class MispObject extends AppModel } // check whether the variable is null or datetime - public function datetimeOrNull($fields) - { - $k = array_keys($fields)[0]; - $seen = $fields[$k]; - try { - new DateTime($seen); - $returnValue = true; - } catch (Exception $e) { - $returnValue = false; - } - return $returnValue || is_null($seen); - } + public function datetimeOrNull($fields) + { + $seen = array_values($fields)[0]; + if ($seen === null) { + return true; + } + return strtotime($seen) !== false; + } public function validateLastSeenValue($fields) { $ls = $fields['last_seen']; - if (!isset($this->data['Object']['first_seen']) || is_null($ls)) { + if (!isset($this->data['Object']['first_seen']) || $ls === null) { return true; } $converted = $this->Attribute->ISODatetimeToUTC(['Object' => [ @@ -247,50 +244,56 @@ class MispObject extends AppModel public function afterFind($results, $primary = false) { - foreach ($results as $k => $v) { - $results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias); + foreach ($results as &$v) { + $object = &$v['Object']; + if (!empty($object['first_seen'])) { + $object['first_seen'] = $this->microTimestampToIso($object['first_seen']); + } + if (!empty($object['last_seen'])) { + $object['last_seen'] = $this->microTimestampToIso($object['last_seen']); + } } return $results; } - public function beforeSave($options = array()) { + public function beforeSave($options = array()) + { + // generate UUID if it doesn't exist + if (empty($this->data['Object']['uuid'])) { + $this->data['Object']['uuid'] = CakeText::uuid(); + } $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias); } public function beforeValidate($options = array()) { - parent::beforeValidate(); - if (empty($this->data[$this->alias]['comment'])) { - $this->data[$this->alias]['comment'] = ""; - } - // generate UUID if it doesn't exist - if (empty($this->data[$this->alias]['uuid'])) { - $this->data[$this->alias]['uuid'] = CakeText::uuid(); + $object = &$this->data['Object']; + if (empty($object['comment'])) { + $object['comment'] = ""; } // generate timestamp if it doesn't exist - if (empty($this->data[$this->alias]['timestamp'])) { - $date = new DateTime(); - $this->data[$this->alias]['timestamp'] = $date->getTimestamp(); + if (empty($object['timestamp'])) { + $object['timestamp'] = time(); } // parse first_seen different formats - if (isset($this->data[$this->alias]['first_seen'])) { - $this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen']; + if (isset($object['first_seen'])) { + $object['first_seen'] = $object['first_seen'] === '' ? null : $object['first_seen']; } // parse last_seen different formats - if (isset($this->data[$this->alias]['last_seen'])) { - $this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen']; + if (isset($object['last_seen'])) { + $object['last_seen'] = $object['last_seen'] === '' ? null : $object['last_seen']; } - if (empty($this->data[$this->alias]['template_version'])) { - $this->data[$this->alias]['template_version'] = 1; + if (empty($object['template_version'])) { + $object['template_version'] = 1; } - if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) { - $this->data[$this->alias]['deleted'] = 0; + if (isset($object['deleted']) && empty($object['deleted'])) { + $object['deleted'] = 0; } - if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) { - $this->data['Object']['sharing_group_id'] = 0; + if (!isset($object['distribution']) || $object['distribution'] != 4) { + $object['sharing_group_id'] = 0; } - if (!isset($this->data[$this->alias]['distribution'])) { - $this->data['Object']['distribution'] = 5; + if (!isset($object['distribution'])) { + $object['distribution'] = 5; } return true; } @@ -359,7 +362,7 @@ class MispObject extends AppModel } } - public function checkForDuplicateObjects($object, $eventId, &$duplicatedObjectID) + public function checkForDuplicateObjects($object, $eventId, &$duplicatedObjectID, &$duplicateObjectUuid) { $newObjectAttributes = array(); if (isset($object['Object']['Attribute'])) { @@ -373,7 +376,7 @@ class MispObject extends AppModel $attribute['value'] = $attribute['value'] . '|' . md5(base64_decode($attribute['data'])); } } - $attributeValueAfterModification = $this->Attribute->modifyBeforeValidation($attribute['type'], $attribute['value']); + $attributeValueAfterModification = AttributeValidationTool::modifyBeforeValidation($attribute['type'], $attribute['value']); $attributeValueAfterModification = $this->Attribute->runRegexp($attribute['type'], $attributeValueAfterModification); $newObjectAttributes[] = sha1($attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $attributeValueAfterModification, true); @@ -384,6 +387,7 @@ class MispObject extends AppModel if ($newObjectAttributeCount === count($previousNewObject)) { if (empty(array_diff($previousNewObject, $newObjectAttributes))) { $duplicatedObjectID = $previousNewObject['Object']['id']; + $duplicateObjectUuid = $previousNewObject['Object']['uuid']; return true; } } @@ -442,9 +446,10 @@ class MispObject extends AppModel $object['Object']['event_id'] = $eventId; if ($breakOnDuplicate) { $duplicatedObjectID = null; - $duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID); + $duplicateObjectUuid = null; + $duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID, $dupicateObjectUuid); if ($duplicate) { - return array('value' => array(__('Duplicate object found (id: %s). Since breakOnDuplicate is set the object will not be added.', $duplicatedObjectID))); + return array('value' => array(__('Duplicate object found (id: %s, uuid: %s). Since breakOnDuplicate is set the object will not be added.', $duplicatedObjectID, $dupicateObjectUuid))); } } $this->create(); @@ -729,6 +734,8 @@ class MispObject extends AppModel * Clean the attribute list up from artifacts introduced by the object form * @param array $attributes * @return string|array + * @throws InternalErrorException + * @throws Exception */ public function attributeCleanup($attributes) { @@ -749,23 +756,19 @@ class MispObject extends AppModel if (isset($attribute['Attachment'])) { // Check if there were problems with the file upload // only keep the last part of the filename, this should prevent directory attacks - $filename = basename($attribute['Attachment']['name']); - $tmpfile = new File($attribute['Attachment']['tmp_name']); if ((isset($attribute['Attachment']['error']) && $attribute['Attachment']['error'] == 0) || (!empty($attribute['Attachment']['tmp_name']) && $attribute['Attachment']['tmp_name'] != 'none') ) { - if (!is_uploaded_file($tmpfile->path)) { + if (!is_uploaded_file($attribute['Attachment']['tmp_name'])) { throw new InternalErrorException('PHP says file was not uploaded. Are you attacking me?'); } } else { - return 'Issues with the file attachment for the ' . $attribute['object_relation'] . ' attribute. The error code returned is ' . $attribute['Attachment']['error']; + throw new InternalErrorException('Issues with the file attachment for the ' . $attribute['object_relation'] . ' attribute. The error code returned is ' . $attribute['Attachment']['error']); } $attributes['Attribute'][$k]['value'] = $attribute['Attachment']['name']; unset($attributes['Attribute'][$k]['Attachment']); - $attributes['Attribute'][$k]['encrypt'] = $attribute['type'] == 'malware-sample' ? 1 : 0; - $attributes['Attribute'][$k]['data'] = base64_encode($tmpfile->read()); - $tmpfile->delete(); - $tmpfile->close(); + $attributes['Attribute'][$k]['encrypt'] = $attribute['type'] === 'malware-sample' ? 1 : 0; + $attributes['Attribute'][$k]['data'] = base64_encode(FileAccessTool::readAndDelete($attribute['Attachment']['tmp_name'])); } if (!isset($attributes['Attribute'][$k]['first_seen'])) { $attributes['Attribute'][$k]['first_seen'] = null; @@ -978,10 +981,11 @@ class MispObject extends AppModel } if (!empty($object['Object']['breakOnDuplicate']) || $breakOnDuplicate) { $duplicatedObjectID = null; - $duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID); + $duplicateObjectUuid = null; + $duplicate = $this->checkForDuplicateObjects($object, $eventId, $duplicatedObjectID, $duplicateObjectUuid); if ($duplicate) { $this->loadLog()->createLogEntry($user, 'add', 'Object', 0, - __('Object dropped due to it being a duplicate (ID: %s) and breakOnDuplicate being requested for Event %s', $duplicatedObjectID, $eventId), + __('Object dropped due to it being a duplicate (ID: %s, UUID: %s) and breakOnDuplicate being requested for Event %s', $duplicatedObjectID, $dupicateObjectUuid, $eventId), 'Duplicate object found.' ); return true; @@ -1157,15 +1161,15 @@ class MispObject extends AppModel */ public function updateTimestamp($id, $timestamp = false) { - $object = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('Object.id' => $id) - )); - $object['Object']['timestamp'] = $timestamp === false ? time() : $timestamp; - $object['Object']['skip_zmq'] = 1; - $object['Object']['skip_kafka'] = 1; - $result = $this->save($object); - return $result; + $object = [ + 'Object' => [ + 'id' => $id, + 'timestamp' => $timestamp === false ? time() : $timestamp, + 'skip_zmq' => 1, + 'skip_kafka' => 1, + ], + ]; + return $this->save($object, true, ['timestamp']); } // Hunt down all LEDA and CASTOR clones diff --git a/app/Model/Module.php b/app/Model/Module.php index fc7611e18..003e050cd 100644 --- a/app/Model/Module.php +++ b/app/Model/Module.php @@ -1,5 +1,6 @@ __getModuleServer($moduleFamily); - if (!$url) { + $serverUrl = $this->__getModuleServer($moduleFamily); + if (!$serverUrl) { throw new Exception("Module type $moduleFamily is not enabled."); } - App::uses('HttpSocket', 'Network/Http'); + App::uses('HttpSocketExtended', 'Tools'); + $httpSocketSetting = ['timeout' => $timeout]; $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile'); foreach ($sslSettings as $sslSetting) { - if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') { - $settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); + $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); + if ($value && $value !== '') { + $httpSocketSetting[$sslSetting] = $value; } } - $httpSocket = new HttpSocket(['timeout' => $timeout]); - $request = array( - 'header' => array( - 'Content-Type' => 'application/json', - ) - ); - if ($moduleFamily == 'Cortex') { + $httpSocket = new HttpSocketExtended($httpSocketSetting); + $request = []; + if ($moduleFamily === 'Cortex') { if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) { $request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey'); } @@ -264,26 +264,23 @@ class Module extends AppModel if (!is_array($postData)) { throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given."); } - $post = json_encode($postData); - $response = $httpSocket->post($url . $uri, $post, $request); + $post = JsonTool::encode($postData); + $request['header']['Content-Type'] = 'application/json'; + $response = $httpSocket->post($serverUrl . $uri, $post, $request); } else { - if ($moduleFamily == 'Cortex') { - unset($request['header']['Content-Type']); - } - $response = $httpSocket->get($url . $uri, false, $request); + $response = $httpSocket->get($serverUrl . $uri, false, $request); } if (!$response->isOk()) { - if ($httpSocket->lastError()) { - throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']); - } - throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code); + $e = new HttpSocketHttpException($response, $serverUrl . $uri); + throw new Exception("Failed to get response from `$moduleFamily` module", 0, $e); } - return $this->jsonDecode($response->body); + return $response->json(); } /** * @param string $moduleFamily * @return array + * @throws JsonException */ public function getModuleSettings($moduleFamily = 'Enrichment') { diff --git a/app/Model/ObjectReference.php b/app/Model/ObjectReference.php index c80034d62..8bc37aae1 100644 --- a/app/Model/ObjectReference.php +++ b/app/Model/ObjectReference.php @@ -38,17 +38,38 @@ class ObjectReference extends AppModel ) ); + public $validate = [ + 'uuid' => 'uuid', + 'object_id' => [ + 'rule' => 'numeric', + 'required' => true, + 'on' => 'create', + ], + 'event_id' => [ + 'rule' => 'numeric', + 'required' => true, + 'on' => 'create', + ], + 'source_uuid' => 'uuid', + 'referenced_uuid' => 'uuid', + 'referenced_id' => 'numeric', + 'referenced_type' => [ + 'rule' => ['inList', ['0', '1']], + ], + 'deleted' => 'boolean', + ]; + public function beforeValidate($options = array()) { - parent::beforeValidate(); - if (empty($this->data['ObjectReference']['uuid'])) { - $this->data['ObjectReference']['uuid'] = CakeText::uuid(); + $reference = &$this->data['ObjectReference']; + if (empty($reference['uuid'])) { + $reference['uuid'] = CakeText::uuid(); } - if (empty($this->data['ObjectReference']['timestamp'])) { - $this->data['ObjectReference']['timestamp'] = time(); + if (empty($reference['timestamp'])) { + $reference['timestamp'] = time(); } - if (!isset($this->data['ObjectReference']['comment'])) { - $this->data['ObjectReference']['comment'] = ''; + if (!isset($reference['comment'])) { + $reference['comment'] = ''; } return true; } @@ -78,18 +99,24 @@ class ObjectReference extends AppModel return true; } - public function updateTimestamps($id, $objectReference = false) + /** + * @param int|array $objectReference + * @return false|void + * @throws Exception + */ + public function updateTimestamps($objectReference) { - if (!$objectReference) { + if (is_numeric($objectReference)) { $objectReference = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('ObjectReference.id' => $id), + 'conditions' => array('ObjectReference.id' => $objectReference), 'fields' => array('event_id', 'object_id') )); + if (empty($objectReference)) { + return false; + } } - if (empty($objectReference)) { - return false; - } + if (!isset($objectReference['ObjectReference'])) { $objectReference = array('ObjectReference' => $objectReference); } @@ -165,12 +192,18 @@ class ObjectReference extends AppModel if (!$result) { return $this->validationErrors; } else { - $this->updateTimestamps($this->id, $objectReference); + $this->updateTimestamps($objectReference); } return true; } - public function captureReference($reference, $eventId, $user) + /** + * @param array $reference + * @param int $eventId + * @return array|bool + * @throws Exception + */ + public function captureReference(array $reference, $eventId) { if (isset($reference['uuid'])) { $existingReference = $this->find('first', array( @@ -257,6 +290,9 @@ class ObjectReference extends AppModel $reference['object_uuid'] = $sourceObject['Object']['uuid']; $reference['event_id'] = $eventId; $result = $this->save(array('ObjectReference' => $reference)); + if (!$result) { + return $this->validationErrors; + } return true; } @@ -308,7 +344,8 @@ class ObjectReference extends AppModel return array($referenced_id, $referenced_uuid, $referenced_type); } - function isValidExtendedEventForReference($sourceEvent, $targetEventID, $user) { + private function isValidExtendedEventForReference(array $sourceEvent, $targetEventID, array $user) + { if ($sourceEvent['Event']['orgc_id'] != $user['org_id']) { return false; } diff --git a/app/Model/ObjectTemplate.php b/app/Model/ObjectTemplate.php index 69856c9de..d582cabfe 100644 --- a/app/Model/ObjectTemplate.php +++ b/app/Model/ObjectTemplate.php @@ -1,7 +1,11 @@ true, ) ); - public $validate = array( - ); public $objectsDir = APP . 'files/misp-objects/objects'; public function afterFind($results, $primary = false) { foreach ($results as $k => $result) { - if (isset($results[$k]['ObjectTemplate']['requirements'])) { - $results[$k]['ObjectTemplate']['requirements'] = json_decode($results[$k]['ObjectTemplate']['requirements'], true); + if (isset($result['ObjectTemplate']['requirements'])) { + $results[$k]['ObjectTemplate']['requirements'] = json_decode($result['ObjectTemplate']['requirements'], true); } } return $results; @@ -65,9 +67,8 @@ class ObjectTemplate extends AppModel if (!file_exists($this->objectsDir . DS . $dir . DS . 'definition.json')) { continue; } - $file = new File($this->objectsDir . DS . $dir . DS . 'definition.json'); - $template = json_decode($file->read(), true); - $file->close(); + $file = FileAccessTool::readFromFile($this->objectsDir . DS . $dir . DS . 'definition.json'); + $template = JsonTool::decode($file); if (!isset($template['version'])) { $template['version'] = 1; } @@ -98,7 +99,6 @@ class ObjectTemplate extends AppModel private function __updateObjectTemplate($template, $current, $user = false) { - $success = false; $template['requirements'] = array(); $requirementFields = array('required', 'requiredOneOf'); foreach ($requirementFields as $field) { @@ -121,14 +121,16 @@ class ObjectTemplate extends AppModel } $id = $this->id; $this->setActive($id); - $fieldsToCompare = array('object_relation', 'type', 'ui-priority', 'categories', 'sane_default', 'values_list', 'multiple', 'disable_correlation'); + + $attributes = []; foreach ($template['attributes'] as $k => $attribute) { - $attribute['object_relation'] = $k; $attribute = $this->__convertJSONToElement($attribute); - $this->ObjectTemplateElement->create(); + $attribute['object_relation'] = $k; $attribute['object_template_id'] = $id; - $result = $this->ObjectTemplateElement->save(array('ObjectTemplateElement' => $attribute)); + $attributes[] = ['ObjectTemplateElement' => $attribute]; } + $this->ObjectTemplateElement->saveMany($attributes); + return true; } @@ -136,17 +138,17 @@ class ObjectTemplate extends AppModel { $result = array(); $translation_table = array( - 'misp-usage-frequency' => 'frequency', - 'misp-attribute' => 'type', - 'description' => 'description', - 'ui-priority' => 'ui-priority', - 'type' => 'type', - 'disable_correlation' => 'disable_correlation', - 'object_relation' => 'object_relation', - 'categories' => 'categories', - 'sane_default' => 'sane_default', - 'values_list' => 'values_list', - 'multiple' => 'multiple' + 'misp-usage-frequency' => 'frequency', + 'misp-attribute' => 'type', + 'description' => 'description', + 'ui-priority' => 'ui-priority', + 'type' => 'type', + 'disable_correlation' => 'disable_correlation', + 'object_relation' => 'object_relation', + 'categories' => 'categories', + 'sane_default' => 'sane_default', + 'values_list' => 'values_list', + 'multiple' => 'multiple' ); foreach ($translation_table as $from => $to) { if (isset($attribute[$from])) { @@ -291,18 +293,20 @@ class ObjectTemplate extends AppModel { $template = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('ObjectTemplate.id' => $id) + 'conditions' => array('ObjectTemplate.id' => $id), + 'fields' => ['ObjectTemplate.id', 'ObjectTemplate.uuid', 'ObjectTemplate.active'], )); if (empty($template)) { return false; } if ($template['ObjectTemplate']['active']) { $template['ObjectTemplate']['active'] = 0; - $this->save($template); + $this->save($template, true, ['active']); return 0; } $similar_templates = $this->find('all', array( 'recursive' => -1, + 'fields' => ['ObjectTemplate.id'], 'conditions' => array( 'ObjectTemplate.uuid' => $template['ObjectTemplate']['uuid'], 'NOT' => array( @@ -311,42 +315,41 @@ class ObjectTemplate extends AppModel ) )); $template['ObjectTemplate']['active'] = 1; - $this->save($template); + $this->save($template, true, ['active']); foreach ($similar_templates as $st) { $st['ObjectTemplate']['active'] = 0; - $this->save($st); + $this->save($st, true, ['active']); } return 1; } public function getRawFromDisk($uuidOrName) { - $template = []; if (Validation::uuid($uuidOrName)) { foreach ($this->readTemplatesFromDisk() as $templateFromDisk) { - if ($templateFromDisk['uuid'] == $uuidOrName) { - $template = $templateFromDisk; - break; + if ($templateFromDisk['uuid'] === $uuidOrName) { + return $templateFromDisk; } } } else { $allTemplateNames = $this->getTemplateDirectoryPaths(false); - if (in_array($uuidOrName, $allTemplateNames)) { // ensure the path is not out of scope - $template = $this->readTemplateFromDisk($this->getFullPathFromTemplateName($uuidOrName)); + if (in_array($uuidOrName, $allTemplateNames, true)) { // ensure the path is not out of scope + return $this->readTemplateFromDisk($this->getFullPathFromTemplateName($uuidOrName)); } } - return $template; + return []; } + /** + * @throws Exception + */ private function readTemplateFromDisk($path) { - $file = new File($path, false); - if (!$file->exists()) { + if (!file_exists($path)) { return false; } - $template = json_decode($file->read(), true); - $file->close(); - return $template; + $content = FileAccessTool::readFromFile($path); + return JsonTool::decode($content); } private function readTemplatesFromDisk() diff --git a/app/Model/Post.php b/app/Model/Post.php index a36f7fecd..dbcfeb4e2 100644 --- a/app/Model/Post.php +++ b/app/Model/Post.php @@ -32,27 +32,33 @@ class Post extends AppModel { if (Configure::read('MISP.background_jobs')) { $user = $this->User->findById($user_id); + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'email', - 'job_type' => 'posts_alert', - 'job_input' => 'Post: ' . $post_id, - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['User']['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => 'Sending...', + $jobId = $job->createJob( + $user['User'], + Job::WORKER_EMAIL, + 'posts_alert', + 'Post: ' . $post_id, + 'Sending...' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'email', - 'EventShell', - array('postsemail', $user_id, $post_id, $event_id, $title, $message, $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::EMAIL_QUEUE, + BackgroundJobsTool::CMD_EVENT, + [ + 'postsemail', + $user_id, + $post_id, + $event_id, + $title, + $message, + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return true; } else { return $this->sendPostsEmail($user_id, $post_id, $event_id, $title, $message); diff --git a/app/Model/Server.php b/app/Model/Server.php index 05997180e..67f89a7d3 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2,7 +2,10 @@ App::uses('AppModel', 'Model'); App::uses('GpgTool', 'Tools'); App::uses('ServerSyncTool', 'Tools'); -App::uses('FileAccessTool', 'Tools'); +App::uses('SystemSetting', 'Model'); +App::uses('EncryptedValue', 'Tools'); +App::uses('GitTool', 'Tools'); +App::uses('ProcessTool', 'Tools'); /** * @property-read array $serverSettings @@ -161,20 +164,15 @@ class Server extends AppModel ) ); - private $__settingTabMergeRules = array( - 'GnuPG' => 'Encryption', - 'SMIME' => 'Encryption', - 'misc' => 'Security', - 'Security' => 'Security', - 'Session' => 'Security' - ); - public $validEventIndexFilters = array('searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute'); public function beforeSave($options = array()) { - $this->data['Server']['url'] = rtrim($this->data['Server']['url'], '/'); - if (empty($this->data['Server']['id'])) { + $server = &$this->data['Server']; + if (!empty($server['url'])) { + $server['url'] = rtrim($server['url'], '/'); + } + if (empty($server['id'])) { $max_prio = $this->find('first', array( 'recursive' => -1, 'order' => array('Server.priority' => 'DESC'), @@ -185,7 +183,11 @@ class Server extends AppModel } else { $max_prio = $max_prio['Server']['priority']; } - $this->data['Server']['priority'] = $max_prio + 1; + $server['priority'] = $max_prio + 1; + } + // Encrypt authkey if plain key provided and encryption is enabled + if (!empty($server['authkey']) && strlen($server['authkey']) === 40) { + $server['authkey'] = EncryptedValue::encryptIfEnabled($server['authkey']); } return true; } @@ -233,7 +235,12 @@ class Server extends AppModel return false; } - private function __updatePulledEventBeforeInsert(&$event, $server, $user) + /** + * @param array $event + * @param array $server + * @param array $user + */ + private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user) { // we have an Event array // The event came from a pull, so it should be locked. @@ -261,9 +268,9 @@ class Server extends AppModel } } } - if (isset($event['Event']['Attribute']) && !empty($event['Event']['Attribute'])) { - foreach ($event['Event']['Attribute'] as $key => $a) { - switch ($a['distribution']) { + if (isset($event['Event']['Attribute'])) { + foreach ($event['Event']['Attribute'] as $key => $attribute) { + switch ($attribute['distribution']) { case '1': $event['Event']['Attribute'][$key]['distribution'] = '0'; break; @@ -272,8 +279,8 @@ class Server extends AppModel break; } // We remove local tags obtained via pull - if (isset($a['Tag'])) { - foreach ($a['Tag'] as $k => $v) { + if (isset($attribute['Tag'])) { + foreach ($attribute['Tag'] as $k => $v) { if ($v['local']) { unset($event['Event']['Attribute'][$key]['Tag'][$k]); } @@ -281,9 +288,9 @@ class Server extends AppModel } } } - if (isset($event['Event']['Object']) && !empty($event['Event']['Object'])) { - foreach ($event['Event']['Object'] as $i => $o) { - switch ($o['distribution']) { + if (isset($event['Event']['Object'])) { + foreach ($event['Event']['Object'] as $i => $object) { + switch ($object['distribution']) { case '1': $event['Event']['Object'][$i]['distribution'] = '0'; break; @@ -291,8 +298,8 @@ class Server extends AppModel $event['Event']['Object'][$i]['distribution'] = '1'; break; } - if (isset($event['Event']['Object'][$i]['Attribute']) && !empty($event['Event']['Object'][$i]['Attribute'])) { - foreach ($event['Event']['Object'][$i]['Attribute'] as $j => $a) { + if (isset($object['Attribute'])) { + foreach ($object['Attribute'] as $j => $a) { switch ($a['distribution']) { case '1': $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0'; @@ -313,7 +320,7 @@ class Server extends AppModel } } } - if (isset($event['Event']['EventReport']) && !empty($event['Event']['EventReport'])) { + if (isset($event['Event']['EventReport'])) { foreach ($event['Event']['EventReport'] as $key => $r) { switch ($r['distribution']) { case '1': @@ -329,10 +336,13 @@ class Server extends AppModel // Distribution, set reporter of the event, being the admin that initiated the pull $event['Event']['user_id'] = $user['id']; - return $event; } - private function __checkIfEventSaveAble($event) + /** + * @param array $event + * @return bool True if event is not empty + */ + private function __checkIfEventSaveAble(array $event) { if (!empty($event['Event']['Attribute'])) { foreach ($event['Event']['Attribute'] as $attribute) { @@ -379,7 +389,7 @@ class Server extends AppModel $result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId); if ($result) { $successes[] = $eventId; - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + if ($this->pubToZmq('event')) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'add_from_connected_server'); } @@ -393,7 +403,7 @@ class Server extends AppModel $result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force); if ($result === true) { $successes[] = $eventId; - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + if ($this->pubToZmq('event')) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'edit_from_connected_server'); } @@ -431,7 +441,7 @@ class Server extends AppModel if ($this->__checkIfEventIsBlockedBeforePull($event)) { return false; } - $event = $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user); + $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user); if (!$this->__checkIfEventSaveAble($event)) { $fails[$eventId] = __('Empty event detected.'); } else { @@ -444,12 +454,22 @@ class Server extends AppModel return true; } - public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false) + /** + * @param array $user + * @param string $technique + * @param array $server + * @param int|false $jobId + * @param bool $force + * @return array|string + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + * @throws JsonException + */ + public function pull(array $user, $technique, array $server, $jobId = false, $force = false) { if ($jobId) { Configure::write('CurrentUserId', $user['id']); $job = ClassRegistry::init('Job'); - $job->read(null, $jobId); $email = "Scheduled job"; } else { $email = $user['email']; @@ -474,19 +494,17 @@ class Server extends AppModel if (!empty($server['Server']['pull_galaxy_clusters'])) { $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); if ($jobId) { - $job->saveField('message', $technique == 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.')); + $job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.')); } $pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique); - if ($technique == 'pull_relevant_clusters') { + if ($technique === 'pull_relevant_clusters') { if ($jobId) { - $job->saveField('progress', 100); - $job->saveField('message', 'Pulling complete.'); + $job->saveStatus($jobId, true, 'Pulling complete.'); } return array(array(), array(), 0, 0, $pulledClusters); } if ($jobId) { - $job->saveField('progress', 10); - $job->saveField('message', 'Pulling events.'); + $job->saveProgress($jobId, 'Pulling events.', 10); } } @@ -516,16 +534,12 @@ class Server extends AppModel } foreach ($eventIds as $k => $eventId) { $this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force); - if ($jobId) { - if ($k % 10 == 0) { - $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds))); - } + if ($jobId && $k % 10 === 0) { + $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds))); } } - } - if (!empty($fails)) { foreach ($fails as $eventid => $message) { - $this->loadLog()->createLogEntry($user, 'pull', 'Server', $id, "Failed to pull event #$eventid.", 'Reason: ' . $message); + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message); } } if ($jobId) { @@ -541,27 +555,18 @@ class Server extends AppModel $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync); } if ($jobId) { - $job->saveProgress($jobId, 'Pull completed.', 100); + $job->saveStatus($jobId, true, 'Pull completed.'); } - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => $id, - 'email' => $user['email'], - 'action' => 'pull', - 'user_id' => $user['id'], - 'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, - 'change' => sprintf( - '%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.', - count($successes), - $pulledProposals, - $pulledSightings, - $pulledClusters, - count($fails) - ) - )); + + $change = sprintf( + '%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.', + count($successes), + $pulledProposals, + $pulledSightings, + $pulledClusters, + count($fails) + ); + $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change); return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters); } @@ -731,7 +736,7 @@ class Server extends AppModel * @throws HttpSocketJsonException * @throws InvalidArgumentException */ - public function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false) + private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false) { if (!in_array($scope, ['events', 'sightings'], true)) { throw new InvalidArgumentException("Scope must be 'events' or 'sightings', '$scope' given."); @@ -1347,12 +1352,21 @@ class Server extends AppModel public function serverSettingsRead($unsorted = false) { + $settingTabMergeRules = array( + 'GnuPG' => 'Encryption', + 'SMIME' => 'Encryption', + 'misc' => 'Security', + 'Security' => 'Security', + 'Session' => 'Security', + 'SimpleBackgroundJobs' => 'SimpleBackgroundJobs' + ); + $serverSettings = $this->getCurrentServerSettings(); $currentSettings = Configure::read(); $finalSettingsUnsorted = $this->__serverSettingsRead($serverSettings, $currentSettings); foreach ($finalSettingsUnsorted as $key => $temp) { - if (isset($this->__settingTabMergeRules[$temp['tab']])) { - $finalSettingsUnsorted[$key]['tab'] = $this->__settingTabMergeRules[$temp['tab']]; + if (isset($settingTabMergeRules[$temp['tab']])) { + $finalSettingsUnsorted[$key]['tab'] = $settingTabMergeRules[$temp['tab']]; } } if ($unsorted) { @@ -1377,8 +1391,20 @@ class Server extends AppModel private function __evaluateLeaf($leafValue, $leafKey, $setting) { if (isset($setting)) { + if ($setting instanceof EncryptedValue) { + try { + $setting = $setting->decrypt(); + } catch (Exception $e) { + $leafValue['errorMessage'] = 'Could not decrypt.'; + return $leafValue; + } + } if (!empty($leafValue['test'])) { - $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']); + if ($leafValue['test'] instanceof Closure) { + $result = $leafValue['test']($setting); + } else { + $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']); + } if ($result !== true) { $leafValue['error'] = 1; if ($result !== false) { @@ -1402,7 +1428,7 @@ class Server extends AppModel { $dirs = glob(APP . 'Locale/*', GLOB_ONLYDIR); $languages = array('eng' => 'eng'); - foreach ($dirs as $k => $dir) { + foreach ($dirs as $dir) { $dir = str_replace(APP . 'Locale' . DS, '', $dir); $languages[$dir] = $dir; } @@ -1432,13 +1458,17 @@ class Server extends AppModel private function loadLocalOrganisations($strict = false) { - $localOrgs = $this->Organisation->find('list', array( - 'conditions' => array('local' => 1), - 'recursive' => -1, - 'fields' => array('Organisation.id', 'Organisation.name') - )); + static $localOrgs; - if(!$strict){ + if ($localOrgs === null) { + $localOrgs = $this->Organisation->find('list', array( + 'conditions' => array('local' => 1), + 'recursive' => -1, + 'fields' => array('Organisation.id', 'Organisation.name') + )); + } + + if (!$strict) { return array_replace(array(0 => __('No organisation selected.')), $localOrgs); } @@ -1510,15 +1540,10 @@ class Server extends AppModel public function testLocalOrgStrict($value) { - $this->Organisation = ClassRegistry::init('Organisation'); if ($value == 0) { return 'No organisation selected'; } - $local_orgs = $this->Organisation->find('list', array( - 'conditions' => array('local' => 1), - 'recursive' => -1, - 'fields' => array('Organisation.id', 'Organisation.name') - )); + $local_orgs = $this->loadLocalOrganisations(true); if (in_array($value, array_keys($local_orgs))) { return true; } @@ -1624,7 +1649,6 @@ class Server extends AppModel return true; } - public function getHost() { if (function_exists('apache_request_headers')) { @@ -1673,11 +1697,14 @@ class Server extends AppModel } return true; } + if (empty($value)) { + return true; + } if ($this->testForEmpty($value) !== true) { return $this->testForEmpty($value); } $regex = "/^(?https?):\/\/(?([\w,\-,\.]+))(?::(?[0-9]+))?(?\/[a-z0-9_\-\.]+)?$/i"; - if ( + if ( !preg_match($regex, $value, $matches) || strtolower($matches['proto']) != strtolower($this->getProto()) || ( @@ -2012,26 +2039,28 @@ class Server extends AppModel $jobType = 'jobGenerateCorrelation'; $jobTypeText = 'generate correlation'; } + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => $jobTypeText, - 'job_input' => 'All attributes', - 'status' => 0, - 'retries' => 0, - 'org' => 'ADMIN', - 'message' => 'Job created.', + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_PRIO, + $jobTypeText, + 'All attributes', + 'Job created.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array($jobType, $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'updateAfterPull', + $jobType, + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); } return true; } @@ -2074,10 +2103,10 @@ class Server extends AppModel private function __serverSettingNormaliseValue($data, $value, $setting) { if (!empty($data['type'])) { - if ($data['type'] == 'boolean') { - $value = $value ? true : false; - } elseif ($data['type'] == 'numeric') { - $value = intval($value); + if ($data['type'] === 'boolean') { + $value = (bool)$value; + } elseif ($data['type'] === 'numeric') { + $value = (int)$value; } } return $value; @@ -2128,30 +2157,41 @@ class Server extends AppModel if ($beforeResult !== true) { $this->Log = ClassRegistry::init('Log'); $this->Log->create(); - $result = $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => 0, - 'email' => $user['email'], - 'action' => 'serverSettingsEdit', - 'user_id' => $user['id'], - 'title' => 'Server setting issue', - 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.', + $this->Log->save(array( + 'org' => $user['Organisation']['name'], + 'model' => 'Server', + 'model_id' => 0, + 'email' => $user['email'], + 'action' => 'serverSettingsEdit', + 'user_id' => $user['id'], + 'title' => 'Server setting issue', + 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.', )); return $beforeResult; } } - $value = trim($value); - if ($setting['type'] === 'boolean') { - $value = (bool)$value; - } else if ($setting['type'] === 'numeric') { - $value = (int)($value); - } - if (!empty($setting['test'])) { - $testResult = $this->{$setting['test']}($value); + if ($value !== null) { + $value = trim($value); + if ($setting['type'] === 'boolean') { + $value = (bool)$value; + } else if ($setting['type'] === 'numeric') { + $value = (int)($value); + } + if (isset($setting['test'])) { + if ($setting['test'] instanceof Closure) { + $testResult = $setting['test']($value); + } else { + $testResult = $this->{$setting['test']}($value); + } + } else { + $testResult = true; # No test defined for this setting: cannot fail + } + } else if (isset($setting['null']) && $setting['null']) { + $testResult = true; } else { - $testResult = true; # No test defined for this setting: cannot fail + $testResult = __('Value could not be null.'); } + if (!$forceSave && $testResult !== true) { if ($testResult === false) { $errorMessage = $setting['errorMessage']; @@ -2161,14 +2201,23 @@ class Server extends AppModel return $errorMessage; } $oldValue = Configure::read($setting['name']); - $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value); + $fileOnly = isset($setting['cli_only']) && $setting['cli_only']; + $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value, $fileOnly); if ($settingSaveResult) { - $change = array($setting['name'] => array($oldValue, $value)); + if (SystemSetting::isSensitive($setting['name'])) { + $change = array($setting['name'] => array('*****', '*****')); + } else { + $change = array($setting['name'] => array($oldValue, $value)); + } $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting changed', $change); // execute after hook if (isset($setting['afterHook'])) { - $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value)); + if ($setting['afterHook'] instanceof Closure) { + $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue); + } else { + $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue)); + } if ($afterResult !== true) { $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); @@ -2184,11 +2233,18 @@ class Server extends AppModel /** * @param string $setting * @param mixed $value + * @param bool $fileOnly If true, always store value in config file even when `MISP.system_setting_db` is enabled * @return bool * @throws Exception */ - public function serverSettingsSaveValue($setting, $value) + public function serverSettingsSaveValue($setting, $value, $fileOnly = false) { + if (!$fileOnly && Configure::read('MISP.system_setting_db')) { + /** @var SystemSetting $systemSetting */ + $systemSetting = ClassRegistry::init('SystemSetting'); + return $systemSetting->setSetting($setting, $value); + } + $configFilePath = APP . 'Config' . DS . 'config.php'; if (!is_writable($configFilePath)) { return false; // config file is not writeable @@ -2222,31 +2278,26 @@ class Server extends AppModel } } } - Configure::write($setting, $value); - $arrayFix = array( - 'Security.auth', - 'ApacheSecureAuth.ldapFilter' - ); - foreach ($arrayFix as $settingFix) { - if (Configure::read($settingFix) && is_array(Configure::read($settingFix)) && !empty(Configure::read($settingFix))) { - $arrayElements = array(); - foreach (Configure::read($settingFix) as $array) { - if (!in_array($array, $arrayElements)) { - $arrayElements[] = $array; - } - } - Configure::write($settingFix, $arrayElements); - } + + /** @var array $config */ + require $configFilePath; + if (!isset($config)) { + throw new Exception("Could not load config file `$configFilePath`."); } + $config = Hash::insert($config, $setting, $value); + $settingsToSave = array( 'debug', 'MISP', 'GnuPG', 'SMIME', 'Proxy', 'SecureAuth', 'Security', 'Session.defaults', 'Session.timeout', 'Session.cookieTimeout', 'Session.autoRegenerate', 'Session.checkAgent', 'site_admin_debug', - 'Plugin', 'CertAuth', 'ApacheShibbAuth', 'ApacheSecureAuth', 'OidcAuth', 'AadAuth' + 'Plugin', 'CertAuth', 'ApacheShibbAuth', 'ApacheSecureAuth', 'OidcAuth', + 'AadAuth', 'SimpleBackgroundJobs' ); $settingsArray = array(); foreach ($settingsToSave as $setting) { - $settingsArray[$setting] = Configure::read($setting); + if (Hash::check($config, $setting)) { + $settingsArray[$setting] = Hash::get($config, $setting); + } } $settingsString = var_export($settingsArray, true); $settingsString = 'checkMISPVersion(); - $current = 'v' . $version_array['major'] . '.' . $version_array['minor'] . '.' . $version_array['hotfix']; - $newest_array = $this->__dissectVersion($newest); - $upToDate = $this->__compareVersions(array($version_array['major'], $version_array['minor'], $version_array['hotfix']), $newest_array, 0); - return array('current' => $current, 'newest' => $newest, 'upToDate' => $upToDate); - } - - private function __dissectVersion($version) - { - $version = substr($version, 1); - return explode('.', $version); - } - - private function __compareVersions($current, $newest, $i) - { - if ($current[$i] == $newest[$i]) { - if ($i < 2) { - return $this->__compareVersions($current, $newest, $i+1); - } else { - return 'same'; - } - } elseif ($current[$i] < $newest[$i]) { - return 'older'; - } else { - return 'newer'; - } - } - public function getFileRules() { $validItems = array( @@ -2481,61 +2502,47 @@ class Server extends AppModel return ['status' => 1, 'content-encoding' => $contentEncoding]; } - public function checkVersionCompatibility(array $server, $user = array(), $HttpSocket = false) + /** + * @param array $server + * @param array $user + * @param ServerSyncTool|null $serverSync + * @return array|string + * @throws JsonException + */ + public function checkVersionCompatibility(array $server, $user = [], ServerSyncTool $serverSync = null) { // for event publishing when we don't have a user. if (empty($user)) { - $user = array('Organisation' => array('name' => 'SYSTEM'), 'email' => 'SYSTEM', 'id' => 0); + $user = 'SYSTEM'; } - $localVersion = $this->checkMISPVersion(); - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - $request = $this->setupSyncRequest($server); - $uri = $server['Server']['url'] . '/servers/getVersion'; + + $serverSync = $serverSync ? $serverSync : new ServerSyncTool($server, $this->setupSyncRequest($server)); + try { - $response = $HttpSocket->get($uri, '', $request); + $remoteVersion = $serverSync->info(); } catch (Exception $e) { - $error = $e->getMessage(); - } - if (!isset($response) || $response->code != '200') { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - if (isset($response->code)) { - $title = 'Error: Connection to the server has failed.' . (isset($response->code) ? ' Returned response code: ' . $response->code : ''); + $this->logException("Connection to the server {$server['Server']['id']} has failed", $e); + + if ($e instanceof HttpSocketHttpException) { + $title = 'Error: Connection to the server has failed. Returned response code: ' . $e->getCode(); } else { - $title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $error; + $title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage(); } - $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => $server['Server']['id'], - 'email' => $user['email'], - 'action' => 'error', - 'user_id' => $user['id'], - 'title' => $title - )); + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['Server']['id'], $title); return $title; } - $remoteVersion = $this->jsonDecode($response->body); + $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; $supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']); $canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false; $remoteVersion = explode('.', $remoteVersion['version']); if (!isset($remoteVersion[0])) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); $message = __('Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.'); - $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => $server['Server']['id'], - 'email' => $user['email'], - 'action' => 'error', - 'user_id' => $user['id'], - 'title' => $message, - )); + $this->loadLog()->createLogEntry($user, 'error', 'Server', $server['Server']['id'], $message); return $message; } + $localVersion = $this->checkMISPVersion(); $response = false; $success = false; $issueLevel = "warning"; @@ -2569,17 +2576,7 @@ class Server extends AppModel } if ($response !== false) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => $server['Server']['id'], - 'email' => $user['email'], - 'action' => $issueLevel, - 'user_id' => $user['id'], - 'title' => ucfirst($issueLevel) . ': ' . $response, - )); + $this->loadLog()->createLogEntry($user, $issueLevel, 'Server', $server['Server']['id'], ucfirst($issueLevel) . ': ' . $response); } return [ 'success' => $success, @@ -2830,13 +2827,10 @@ class Server extends AppModel public function getExpectedDBSchema() { - App::uses('Folder', 'Utility'); - $file = new File(ROOT . DS . 'db_schema.json', true); - $dbExpectedSchema = json_decode($file->read(), true); - $file->close(); - if (!is_null($dbExpectedSchema)) { - return $dbExpectedSchema; - } else { + try { + $content = FileAccessTool::readFromFile(ROOT . DS . 'db_schema.json'); + return JsonTool::decode($content); + } catch (Exception $e) { return false; } } @@ -2889,8 +2883,7 @@ class Server extends AppModel } $dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table); } - } - else if ($dataSource == 'Database/Postgres') { + } else if ($dataSource == 'Database/Postgres') { return array('Database/Postgres' => array('description' => __('Can\'t check database schema for Postgres database type'))); } return ['schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes]; @@ -3049,71 +3042,70 @@ class Server extends AppModel ); } + /** + * @throws Exception + */ private function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema) { - $allowedlistTables = array(); $indexDiff = array(); foreach ($expectedIndex as $tableName => $indexes) { if (!array_key_exists($tableName, $actualIndex)) { continue; // If table does not exists, it is covered by the schema diagnostic - } elseif(in_array($tableName, $allowedlistTables)) { - continue; // Ignore allowedlisted tables - } else { - $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes - foreach ($tableIndexDiff as $columnDiff) { - $shouldBeUnique = $indexes[$columnDiff]; - if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) { - $indexDiff[$tableName][$columnDiff] = array( - 'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff), - 'sql' => '', + } + $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes + foreach ($tableIndexDiff as $columnDiff) { + $shouldBeUnique = $indexes[$columnDiff]; + if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) { + $indexDiff[$tableName][$columnDiff] = array( + 'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff), + 'sql' => '', + ); + continue; + } + + $message = __('Column `%s` should be indexed', $columnDiff); + $indexDiff[$tableName][$columnDiff] = array( + 'message' => $message, + 'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique), + ); + } + $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes + foreach ($tableIndexDiff as $columnDiff) { + $message = __('Column `%s` is indexed but should not', $columnDiff); + $indexDiff[$tableName][$columnDiff] = array( + 'message' => $message, + 'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff), + ); + } + foreach ($indexes as $column => $unique) { + if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) { + if ($actualIndex[$tableName][$column]) { + $sql = $this->generateSqlDropIndexQuery($tableName, $column); + $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false); + + $message = __('Column `%s` has unique index, but should be non unique', $column); + $indexDiff[$tableName][$column] = array( + 'message' => $message, + 'sql' => $sql, ); - continue; - } - - $message = __('Column `%s` should be indexed', $columnDiff); - $indexDiff[$tableName][$columnDiff] = array( - 'message' => $message, - 'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique), - ); - } - $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes - foreach ($tableIndexDiff as $columnDiff) { - $message = __('Column `%s` is indexed but should not', $columnDiff); - $indexDiff[$tableName][$columnDiff] = array( - 'message' => $message, - 'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff), - ); - } - foreach ($indexes as $column => $unique) { - if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) { - if ($actualIndex[$tableName][$column]) { - $sql = $this->generateSqlDropIndexQuery($tableName, $column); - $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false); - - $message = __('Column `%s` has unique index, but should be non unique', $column); + } else { + if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) { + $message = __('Column `%s` should be unique index, but contains duplicate values', $column); $indexDiff[$tableName][$column] = array( 'message' => $message, - 'sql' => $sql, - ); - } else { - if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) { - $message = __('Column `%s` should be unique index, but contains duplicate values', $column); - $indexDiff[$tableName][$column] = array( - 'message' => $message, - 'sql' => '', - ); - continue; - } - - $sql = $this->generateSqlDropIndexQuery($tableName, $column); - $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true); - - $message = __('Column `%s` should be unique index', $column); - $indexDiff[$tableName][$column] = array( - 'message' => $message, - 'sql' => $sql, + 'sql' => '', ); + continue; } + + $sql = $this->generateSqlDropIndexQuery($tableName, $column); + $sql .= '
' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true); + + $message = __('Column `%s` should be unique index', $column); + $indexDiff[$tableName][$column] = array( + 'message' => $message, + 'sql' => $sql, + ); } } } @@ -3146,40 +3138,36 @@ class Server extends AppModel public function writeableDirsDiagnostics(&$diagnostic_errors) { - App::uses('File', 'Utility'); - App::uses('Folder', 'Utility'); - // check writeable directories $writeableDirs = array( - '/tmp' => 0, - APP . 'tmp' => 0, - APP . 'files' => 0, - APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0, - APP . 'tmp' . DS . 'csv_all' => 0, - APP . 'tmp' . DS . 'csv_sig' => 0, - APP . 'tmp' . DS . 'md5' => 0, - APP . 'tmp' . DS . 'sha1' => 0, - APP . 'tmp' . DS . 'snort' => 0, - APP . 'tmp' . DS . 'suricata' => 0, - APP . 'tmp' . DS . 'text' => 0, - APP . 'tmp' . DS . 'xml' => 0, - APP . 'tmp' . DS . 'files' => 0, - APP . 'tmp' . DS . 'logs' => 0, - APP . 'tmp' . DS . 'bro' => 0, + '/tmp' => 0, + APP . 'tmp' => 0, + APP . 'files' => 0, + APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0, + APP . 'tmp' . DS . 'csv_all' => 0, + APP . 'tmp' . DS . 'csv_sig' => 0, + APP . 'tmp' . DS . 'md5' => 0, + APP . 'tmp' . DS . 'sha1' => 0, + APP . 'tmp' . DS . 'snort' => 0, + APP . 'tmp' . DS . 'suricata' => 0, + APP . 'tmp' . DS . 'text' => 0, + APP . 'tmp' . DS . 'xml' => 0, + APP . 'tmp' . DS . 'files' => 0, + APP . 'tmp' . DS . 'logs' => 0, + APP . 'tmp' . DS . 'bro' => 0, ); foreach ($writeableDirs as $path => &$error) { - $dir = new Folder($path); - if (is_null($dir->path)) { - $error = 1; + if (!file_exists($path)) { + // Try to create directory if not exists + if (!mkdir($path, 0700, true)) { + $error = 1; + } } - $file = new File($path . DS . 'test.txt', true); - if ($error == 0 && !$file->write('test')) { + if (!is_writable($path)) { $error = 2; } - if ($error != 0) { + if ($error !== 0) { $diagnostic_errors++; } - $file->delete(); - $file->close(); } return $writeableDirs; } @@ -3187,8 +3175,8 @@ class Server extends AppModel public function writeableFilesDiagnostics(&$diagnostic_errors) { $writeableFiles = array( - APP . 'Config' . DS . 'config.php' => 0, - ROOT . DS . '.git' . DS . 'ORIG_HEAD' => 0, + APP . 'Config' . DS . 'config.php' => 0, + ROOT . DS . '.git' . DS . 'ORIG_HEAD' => 0, ); foreach ($writeableFiles as $path => &$error) { if (!file_exists($path)) { @@ -3206,7 +3194,7 @@ class Server extends AppModel public function readableFilesDiagnostics(&$diagnostic_errors) { $readableFiles = array( - APP . 'files' . DS . 'scripts' . DS . 'stixtest.py' => 0 + APP . 'files' . DS . 'scripts' . DS . 'stixtest.py' => 0 ); foreach ($readableFiles as $path => &$error) { if (!is_readable($path)) { @@ -3219,7 +3207,8 @@ class Server extends AppModel public function yaraDiagnostics(&$diagnostic_errors) { - $scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'yaratest.py'); + $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'yaratest.py'; + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); $scriptResult = json_decode($scriptResult, true); return array('operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara']); } @@ -3228,7 +3217,8 @@ class Server extends AppModel { $expected = array('stix' => '>1.2.0.11', 'cybox' => '>2.1.0.21', 'mixbox' => '>1.0.5', 'maec' => '>4.1.0.17', 'stix2' => '>3.0.0', 'pymisp' => '>2.4.120'); // check if the STIX and Cybox libraries are working using the test script stixtest.py - $scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'); + $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'; + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); try { $scriptResult = $this->jsonDecode($scriptResult); } catch (Exception $e) { @@ -3286,7 +3276,7 @@ class Server extends AppModel try { $output['version'] = $gpg->getVersion(); } catch (Exception $e) { - // ingore + // ignore } try { @@ -3354,10 +3344,8 @@ class Server extends AppModel $proxyStatus = 0; $proxy = Configure::read('Proxy'); if (!empty($proxy['host'])) { - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); try { - $HttpSocket = $syncTool->setupHttpSocket(); + $HttpSocket = $this->setupHttpSocket(null); $proxyResponse = $HttpSocket->get('https://www.github.com/'); } catch (Exception $e) { $proxyStatus = 2; @@ -3405,18 +3393,18 @@ class Server extends AppModel 'update' => array('ok' => false), 'scheduler' => array('ok' => false) ); - try { - $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis()); - } catch (Exception $e) { - // redis connection failed - return $worker_array; + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + unset($worker_array['scheduler']); } - $workers = $this->ResqueStatus->getWorkers(); + + $workers = $this->getWorkers(); + if (function_exists('posix_getpwuid')) { $currentUser = posix_getpwuid(posix_geteuid()); $currentUser = $currentUser['name']; } else { - $currentUser = trim(shell_exec('whoami')); + $currentUser = trim(ProcessTool::execute(['whoami'])); } $procAccessible = file_exists('/proc'); foreach ($workers as $pid => $worker) { @@ -3452,7 +3440,7 @@ class Server extends AppModel } } if ($k != 'scheduler') { - $worker_array[$k]['jobCount'] = CakeResque::getQueueSize($k); + $worker_array[$k]['jobCount'] = $this->getBackgroundJobsTool()->getQueueSize($k); } if (!isset($queue['workers'])) { $workerIssueCount++; @@ -3467,6 +3455,16 @@ class Server extends AppModel return $worker_array; } + public function backgroundJobsDiagnostics(&$diagnostic_errors) + { + $backgroundJobsStatus = $this->getBackgroundJobsTool()->getStatus(); + + if ($backgroundJobsStatus > 0) { + $diagnostic_errors++; + } + return $backgroundJobsStatus; + } + public function retrieveCurrentSettings($branch, $subString) { $settings = array(); @@ -3632,7 +3630,8 @@ class Server extends AppModel } $relativeUri = '/events/index' . $urlParams; - list($events, $response) = $this->serverGetRequest($server, $relativeUri); + $response = $this->serverGetRequest($server, $relativeUri); + $events = $response->json(); $totalCount = $response->getHeader('X-Result-Count') ?: 0; foreach ($events as $k => $event) { @@ -3660,8 +3659,9 @@ class Server extends AppModel */ public function previewEvent(array $server, $eventId) { - $relativeUri = '/events/' . $eventId; - list($event) = $this->serverGetRequest($server, $relativeUri); + $relativeUri = '/events/' . $eventId; + $response = $this->serverGetRequest($server, $relativeUri); + $event = $response->json(); if (!isset($event['Event']['Orgc'])) { $event['Event']['Orgc']['name'] = $event['Event']['orgc']; @@ -3795,20 +3795,87 @@ class Server extends AppModel return true; } - public function getCurrentGitStatus() + /** + * @param string $newest + * @return array + * @throws JsonException + */ + private function checkVersion($newest) { - $latestCommit = exec('timeout 3 git ls-remote https://github.com/MISP/MISP | head -1 | sed "s/HEAD//"'); + $version_array = $this->checkMISPVersion(); + $current = implode('.', $version_array); - $status = array(); - $status['commit'] = exec('git rev-parse HEAD'); - $status['branch'] = $this->getCurrentBranch(); - $status['latestCommit'] = $latestCommit; - return $status; + $upToDate = version_compare($current, substr($newest, 1)); + if ($upToDate === 0) { + $upToDate = 'same'; + } else { + $upToDate = $upToDate === -1 ? 'older' : 'newer'; + } + return array('current' => 'v' . $current, 'newest' => $newest, 'upToDate' => $upToDate); + } + + /** + * Fetch latest MISP version from GitHub + * @return array|false + * @throws JsonException + */ + private function checkRemoteVersion($HttpSocket) + { + try { + $json_decoded_tags = GitTool::getLatestTags($HttpSocket); + } catch (Exception $e) { + return false; + } + // find the latest version tag in the v[major].[minor].[hotfix] format + foreach ($json_decoded_tags as $tag) { + if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) { + return $this->checkVersion($tag['name']); + } + } + return false; + } + + /** + * @param bool $checkVersion + * @return array + * @throws JsonException + */ + public function getCurrentGitStatus($checkVersion = false) + { + $HttpSocket = $this->setupHttpSocket(null, null, 3); + try { + $latestCommit = GitTool::getLatestCommit($HttpSocket); + } catch (Exception $e) { + $latestCommit = false; + } + + $output = [ + 'commit' => $this->checkMIPSCommit(), + 'branch' => $this->getCurrentBranch(), + 'latestCommit' => $latestCommit, + ]; + if ($checkVersion) { + $output['version'] = $latestCommit ? $this->checkRemoteVersion($HttpSocket) : false; + } + return $output; } public function getCurrentBranch() { - return exec("git symbolic-ref HEAD | sed 's!refs\/heads\/!!'"); + try { + return GitTool::currentBranch(); + } catch (Exception $e) { + return false; + } + } + + /** + * Check if MISP update is possible. + * @return bool + */ + public function isUpdatePossible() + { + return $this->getCurrentBranch() !== false && is_writable(APP); } public function checkoutMain() @@ -3819,14 +3886,16 @@ class Server extends AppModel public function getSubmodulesGitStatus() { - exec('cd ' . APP . '../; git submodule status --cached | grep -v ^- | cut -b 2- | cut -d " " -f 1,2 ', $submodules_names); + try { + $submodules = GitTool::submoduleStatus(); + } catch (Exception $e) { + $this->logException('Could not fetch git submodules status', $e, LOG_NOTICE); + return []; + } $status = array(); - foreach ($submodules_names as $submodule_name_info) { - $submodule_name_info = explode(' ', $submodule_name_info); - list($superproject_submodule_commit_id, $submodule_name) = $submodule_name_info; - $temp = $this->getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id); - if (!empty($temp) ) { - $status[$submodule_name] = $temp; + foreach ($submodules as $submodule) { + if ($this->_isAcceptedSubmodule($submodule['name'])) { + $status[$submodule['name']] = $this->getSubmoduleGitStatus($submodule['name'], $submodule['commit']);; } } return $status; @@ -3858,57 +3927,59 @@ class Server extends AppModel */ private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) { - $status = array(); - if ($this->_isAcceptedSubmodule($submodule_name)) { - $path = APP . '../' . $submodule_name; - $submodule_name=(strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); - $submodule_name=end($submodule_name); - //$submoduleRemote=exec('cd ' . $path . '; git config --get remote.origin.url'); - exec(sprintf('cd %s; git rev-parse HEAD', $path), $submodule_current_commit_id); - if (!empty($submodule_current_commit_id[0])) { - $submodule_current_commit_id = $submodule_current_commit_id[0]; - } else { - $submodule_current_commit_id = null; - } - $status = array( - 'moduleName' => $submodule_name, - 'current' => $submodule_current_commit_id, - 'currentTimestamp' => exec(sprintf('cd %s; git log -1 --pretty=format:%%ct', $path)), - 'remoteTimestamp' => exec(sprintf('cd %s; git show -s --pretty=format:%%ct %s', $path, $superproject_submodule_commit_id)), - 'remote' => $superproject_submodule_commit_id, - 'upToDate' => '', - 'isReadable' => is_readable($path) && is_readable($path . '/.git'), - ); + $path = APP . '../' . $submodule_name; + $submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); + $submodule_name = end($submodule_name); - if (!empty($status['remote'])) { - if ($status['remote'] == $status['current']) { - $status['upToDate'] = 'same'; - } else if ($status['currentTimestamp'] < $status['remoteTimestamp']) { - $status['upToDate'] = 'older'; - } else { - $status['upToDate'] = 'younger'; - } - } else { - $status['upToDate'] = 'error'; - } + $submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path); - if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { - $date1 = new DateTime(); - $date1->setTimestamp($status['remoteTimestamp']); - $date2 = new DateTime(); - $date2->setTimestamp($status['currentTimestamp']); - $status['timeDiff'] = $date1->diff($date2); - } else { - $status['upToDate'] = 'error'; - } + $currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path); + if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) { + $remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path); + } else { + $remoteTimestamp = $currentTimestamp; } + + $status = array( + 'moduleName' => $submodule_name, + 'current' => $submoduleCurrentCommitId, + 'currentTimestamp' => $currentTimestamp, + 'remote' => $superproject_submodule_commit_id, + 'remoteTimestamp' => $remoteTimestamp, + 'upToDate' => '', + 'isReadable' => is_readable($path) && is_readable($path . '/.git'), + ); + + if (!empty($status['remote'])) { + if ($status['remote'] === $status['current']) { + $status['upToDate'] = 'same'; + } else if ($status['currentTimestamp'] < $status['remoteTimestamp']) { + $status['upToDate'] = 'older'; + } else { + $status['upToDate'] = 'younger'; + } + } else { + $status['upToDate'] = 'error'; + } + + if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { + $date1 = new DateTime(); + $date1->setTimestamp($status['remoteTimestamp']); + $date2 = new DateTime(); + $date2->setTimestamp($status['currentTimestamp']); + $status['timeDiff'] = $date1->diff($date2); + } else { + $status['upToDate'] = 'error'; + } + return $status; } - public function updateSubmodule($user, $submodule_name=false) { + public function updateSubmodule($user, $submodule_name=false) + { $path = APP . '../'; if ($submodule_name == false) { - $command = sprintf('cd %s; git submodule update 2>&1', $path); + $command = sprintf('cd %s; git submodule update --init --recursive 2>&1', $path); exec($command, $output, $return_code); $output = implode("\n", $output); $res = array('status' => ($return_code==0 ? true : false), 'output' => $output); @@ -3916,7 +3987,7 @@ class Server extends AppModel $res = array_merge($res, $this->updateDatabaseAfterPullRouter($submodule_name, $user)); } } else if ($this->_isAcceptedSubmodule($submodule_name)) { - $command = sprintf('cd %s; git submodule update -- %s 2>&1', $path, $submodule_name); + $command = sprintf('cd %s; git submodule update --init --recursive -- %s 2>&1', $path, $submodule_name); exec($command, $output, $return_code); $output = implode("\n", $output); $res = array('status' => ($return_code==0 ? true : false), 'output' => $output); @@ -3929,29 +4000,32 @@ class Server extends AppModel return $res; } - public function updateDatabaseAfterPullRouter($submodule_name, $user) { + public function updateDatabaseAfterPullRouter($submodule_name, $user) + { if (Configure::read('MISP.background_jobs')) { + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'prio', - 'job_type' => __('update_after_pull'), - 'job_input' => __('Updating: ' . $submodule_name), - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => 'Update the database after PULLing the submodule(s).', + $jobId = $job->createJob( + $user, + Job::WORKER_PRIO, + 'update_after_pull', + __('Updating: ' . $submodule_name), + 'Update the database after PULLing the submodule(s).' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'prio', - 'AdminShell', - array('updateAfterPull', $submodule_name, $jobId, $user['id']), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'updateAfterPull', + $submodule_name, + $jobId, + $user['id'] + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return array('job_sent' => true, 'sync_result' => __('unknown')); } else { $result = $this->updateAfterPull($submodule_name, $user['id']); @@ -3959,30 +4033,31 @@ class Server extends AppModel } } - public function updateAfterPull($submodule_name, $userId) { + public function updateAfterPull($submodule_name, $userId) + { $user = $this->User->getAuthUser($userId); $result = array(); if ($user['Role']['perm_site_admin']) { $updateAll = empty($submodule_name); if ($submodule_name == 'app/files/misp-galaxy' || $updateAll) { $this->Galaxy = ClassRegistry::init('Galaxy'); - $result[] = ($this->Galaxy->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; + $result[] = ($this->Galaxy->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; } if ($submodule_name == 'app/files/misp-objects' || $updateAll) { $this->ObjectTemplate = ClassRegistry::init('ObjectTemplate'); - $result[] = ($this->ObjectTemplate->update($user, false, false) ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; + $result[] = ($this->ObjectTemplate->update($user, false, false) ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; } if ($submodule_name == 'app/files/noticelists' || $updateAll) { $this->Noticelist = ClassRegistry::init('Noticelist'); - $result[] = ($this->Noticelist->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; + $result[] = ($this->Noticelist->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; } if ($submodule_name == 'app/files/taxonomies' || $updateAll) { $this->Taxonomy = ClassRegistry::init('Taxonomy'); - $result[] = ($this->Taxonomy->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; + $result[] = ($this->Taxonomy->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; } if ($submodule_name == 'app/files/warninglists' || $updateAll) { $this->Warninglist = ClassRegistry::init('Warninglist'); - $result[] = ($this->Warninglist->update() ? 'Update `' . h($submodule_name) . '` Sucessful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; + $result[] = ($this->Warninglist->update() ? 'Update `' . h($submodule_name) . '` Successful.' : 'Update `'. h($submodule_name) . '` failed.') . PHP_EOL; } } return implode('\n', $result); @@ -4134,14 +4209,14 @@ class Server extends AppModel public function cacheServerInitiator($user, $id = 'all', $jobId = false) { - $params = array( - 'conditions' => array('caching_enabled' => 1), - 'recursive' => -1 - ); $redis = $this->setupRedis(); if ($redis === false) { return 'Redis not reachable.'; } + $params = array( + 'conditions' => array('caching_enabled' => 1), + 'recursive' => -1 + ); if ($id !== 'all') { $params['conditions']['Server.id'] = $id; } else { @@ -4166,65 +4241,67 @@ class Server extends AppModel return true; } + /** + * @param array $server + * @param Redis $redis + * @param int|false $jobId + * @return bool + * @throws JsonException + */ private function __cacheInstance($server, $redis, $jobId = false) { - $continue = true; + $serverId = $server['Server']['id']; $i = 0; + $chunk_size = 50000; if ($jobId) { $job = ClassRegistry::init('Job'); $job->id = $jobId; } - $redis->del('misp:server_cache:' . $server['Server']['id']); - $HttpSocket = null; - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); - while ($continue) { + $redis->del('misp:server_cache:' . $serverId); + + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + while (true) { $i++; - $chunk_size = 50000; - $data = $this->__getCachedAttributes($server, $HttpSocket, $chunk_size, $i); - if (empty(trim($data))) { - $continue = false; - } else { - $pipe = $redis->multi(Redis::PIPELINE); - $data = explode(PHP_EOL, trim($data)); - foreach ($data as $entry) { - list($value, $uuid) = explode(',', $entry); - if (!Validation::uuid($uuid)) { - $continue = false; - break; - } - if (!empty($value)) { - $redis->sAdd('misp:server_cache:' . $server['Server']['id'], $value); - $redis->sAdd('misp:server_cache:combined', $value); - $redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $server['Server']['id'] . '/' . $uuid); - } + $rules = [ + 'returnFormat' => 'cache', + 'includeEventUuid' => 1, + 'page' => $i, + 'limit' => $chunk_size, + ]; + try { + $data = $serverSync->attributeSearch($rules)->body(); + } catch (Exception $e) { + $this->logException("Could not fetch cached attribute from server {$serverSync->serverId()}.", $e); + break; + } + + $data = trim($data); + if (empty($data)) { + break; + } + + $data = explode(PHP_EOL, $data); + $pipe = $redis->pipeline(); + foreach ($data as $entry) { + list($value, $uuid) = explode(',', $entry); + if (!Validation::uuid($uuid)) { + break 2; } - $pipe->exec(); - if ($jobId) { - $job->saveField('message', 'Server ' . $server['Server']['id'] . ': ' . ((($i -1) * $chunk_size) + count($data)) . ' attributes cached.'); + if (!empty($value)) { + $redis->sAdd('misp:server_cache:' . $serverId, $value); + $redis->sAdd('misp:server_cache:combined', $value); + $redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $serverId . '/' . $uuid); } } + $pipe->exec(); + if ($jobId) { + $job->saveProgress($jobId, 'Server ' . $server['Server']['id'] . ': ' . ((($i -1) * $chunk_size) + count($data)) . ' attributes cached.'); + } } - $redis->set('misp:server_cache_timestamp:' . $server['Server']['id'], time()); + $redis->set('misp:server_cache_timestamp:' . $serverId, time()); return true; } - private function __getCachedAttributes($server, $HttpSocket, $chunk_size, $i) - { - $filter_rules = array( - 'returnFormat' => 'cache', - 'includeEventUuid' => 1, - 'page' => $i, - 'limit' => $chunk_size - ); - $request = $this->setupSyncRequest($server); - try { - $response = $HttpSocket->post($server['Server']['url'] . '/attributes/restSearch.json', json_encode($filter_rules), $request); - } catch (SocketException $e) { - return $e->getMessage(); - } - return $response->body; - } - /** * @param array $servers * @return array @@ -4241,7 +4318,7 @@ class Server extends AppModel } $results = $redis->exec(); foreach ($servers as $k => $v) { - $data[$k]['Server']['cache_timestamp'] = $results[$k]; + $servers[$k]['Server']['cache_timestamp'] = $results[$k]; } return $servers; } @@ -4250,8 +4327,8 @@ class Server extends AppModel { $results = array(); foreach (['Galaxy', 'Noticelist', 'Warninglist', 'Taxonomy', 'ObjectTemplate', 'ObjectRelationship'] as $target) { - $this->$target = ClassRegistry::init($target); - $result = $this->$target->update(); + $model = ClassRegistry::init($target); + $result = $model->update(); $results[$target] = $result === false ? false : true; } return $results; @@ -4266,45 +4343,22 @@ class Server extends AppModel if (empty($server)) { return __('Invalid server'); } - $HttpSocket = $this->setupHttpSocket($server); - $request = $this->setupSyncRequest($server); - $uri = $server['Server']['url'] . '/users/resetauthkey/me'; + $serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server)); + try { - $response = $HttpSocket->post($uri, '{}', $request); + $response = $serverSync->resetAuthKey(); } catch (Exception $e) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); $message = 'Could not reset the remote authentication key.'; - $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => $id, - 'email' => 'SYSTEM', - 'action' => 'error', - 'user_id' => 0, - 'title' => 'Error: ' . $message, - )); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $id, 'Error: ' . $message); return $message; } if ($response->isOk()) { try { - $response = $this->jsonDecode($response->body); + $response = $response->json(); } catch (Exception $e) { $message = 'Invalid response received from the remote instance.'; - $this->logException($message, $e); - - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => $id, - 'email' => 'SYSTEM', - 'action' => 'error', - 'user_id' => 0, - 'title' => 'Error: ' . $message, - )); + $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $id, 'Error: ' . $message); return $message; } if (!empty($response['message'])) { @@ -4356,13 +4410,12 @@ class Server extends AppModel /** * @param array $server * @param string $relativeUri - * @param HttpSocket|null $HttpSocket - * @return array + * @return HttpSocketResponseExtended * @throws Exception */ - private function serverGetRequest(array $server, $relativeUri, HttpSocket $HttpSocket = null) + private function serverGetRequest(array $server, $relativeUri) { - $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); + $HttpSocket = $this->setupHttpSocket($server); $request = $this->setupSyncRequest($server); $uri = $server['Server']['url'] . $relativeUri; @@ -4381,9 +4434,7 @@ class Server extends AppModel throw new Exception(__("Fetching the '%s' failed with HTTP error %s: %s", $uri, $response->code, $response->reasonPhrase)); } - $data = $this->jsonDecode($response->body); - - return array($data, $response); + return $response; } /** @@ -4446,40 +4497,39 @@ class Server extends AppModel return parent::__get($name); } + /** + * @return int Number of orphans removed. + */ public function removeOrphanedCorrelations() { $this->Correlation = ClassRegistry::init('Correlation'); $orphansLeft = $this->Correlation->find('all', [ - 'joins' => [ - [ - 'table' => 'attributes', - 'alias' => 'Attribute', - 'type' => 'LEFT', - 'conditions' => [ - 'OR' => [ - 'Correlation.attribute_id = Attribute.id', - ] - - ] - ] - ], + 'contain' => ['Attribute'], 'conditions' => [ 'Attribute.id IS NULL' ], + 'fields' => ['Correlation.id', 'Correlation.attribute_id'], ]); - $orphansRight = $this->Correlation->find('all', [ + if (empty($orphansLeft)) { + return 0; + } + $orphansLeft = array_column($orphansLeft, 'Correlation'); + $orphansRight = $this->Correlation->find('column', [ 'conditions' => [ - '1_attribute_id' => Hash::extract($orphansLeft, '{n}.Correlation.attribute_id') - ] + '1_attribute_id' => array_column($orphansLeft, 'attribute_id'), + ], + 'fields' => ['Correlation.id'], ]); $orphans = array_merge( - Hash::extract($orphansLeft, '{n}.Correlation.id'), - Hash::extract($orphansRight, '{n}.Correlation.id') + array_column($orphansLeft, 'id'), + $orphansRight ); - $success = $this->Correlation->deleteAll([ - 'Correlation.id' => $orphans - ]); - return $success; + if (!empty($orphans)) { + $this->Correlation->deleteAll([ + 'Correlation.id' => $orphans + ], false); + } + return count($orphans); } public function queryAvailableSyncFilteringRules(array $server) @@ -4519,6 +4569,39 @@ class Server extends AppModel ]; } + /** + * @param string|null $old Old (or current) encryption key. + * @param string|null $new New encryption key. If empty, encrypted values will be decrypted. + * @throws Exception + */ + public function reencryptAuthKeys($old, $new) + { + $servers = $this->find('list', [ + 'fields' => ['Server.id', 'Server.authkey'], + ]); + $toSave = []; + foreach ($servers as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + $authkey = BetterSecurity::decrypt(substr($authkey, 2), $old); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for server #$id", 0, $e); + } + } + if (!empty($new)) { + $authkey = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($authkey, $new); + } + $toSave[] = ['Server' => [ + 'id' => $id, + 'authkey' => $authkey, + ]]; + } + if (empty($toSave)) { + return true; + } + return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); + } + /** * Generate just when required * @return array[] @@ -4535,12 +4618,12 @@ class Server extends AppModel 'errorMessage' => __('The currently set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'), 'test' => 'testBaseURL', 'type' => 'string', + 'null' => true ), 'external_baseurl' => array( 'level' => 0, 'description' => __('The base url of the application (in the format https://www.mymispinstance.com) as visible externally/by other MISPs. MISP will encode this URL in sharing groups when including itself. If this value is not set, the baseurl is used as a fallback.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testURL', 'type' => 'string', ), @@ -4548,7 +4631,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Unless set to true, the instance will only be accessible by site admins.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testLive', 'type' => 'boolean', ), @@ -4556,7 +4638,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Select the language MISP should use. The default is english.'), 'value' => 'eng', - 'errorMessage' => '', 'test' => 'testLanguage', 'type' => 'string', 'optionsSource' => function () { @@ -4568,7 +4649,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This values controls the internal fetcher\'s memory envelope when it comes to attributes. The number provided is the amount of attributes that can be loaded for each MB of PHP memory available in one shot. Consider lowering this number if your instance has a lot of attribute tags / attribute galaxies attached.'), 'value' => 80, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true @@ -4577,7 +4657,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This value controls the divisor for attribute weighting when it comes to loading full events. Meaning that it will load coefficient / divisor number of attributes per MB of memory available. Consider raising this number if you have a lot of correlations or highly contextualised events (large number of event level galaxies/tags).'), 'value' => 3, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true @@ -4586,7 +4665,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -4595,7 +4673,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -4604,7 +4681,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT), 'value' => false, - 'errorMessage' => '', 'null' => false, 'test' => 'testForBinExec', 'beforeHook' => 'beforeHookBinExec', @@ -4615,7 +4691,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('MISP will default to the bundled mozilla certificate bundle shipped with the framework, which is rather stale. If you wish to use an alternate bundle, just set this setting using the path to the bundle to use. This setting can only be modified via the CLI.'), 'value' => APP . 'Lib/cakephp/lib/Cake/Config/cacert.pem', - 'errorMessage' => '', 'null' => true, 'test' => 'testForCABundle', 'type' => 'string', @@ -4625,7 +4700,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('In some cases, a heavily used MISP instance can generate unwanted blackhole errors due to a high number of requests hitting the server. Disable the auto logout functionality to ease the burden on the system.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -4634,7 +4708,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set the ssdeep score at which to consider two ssdeep hashes as correlating [1-100]'), 'value' => 40, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -4642,7 +4715,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Sets the maximum number of correlations that can be fetched with a single event. For extreme edge cases this can prevent memory issues. The default value is 5k.'), 'value' => 5000, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true @@ -4659,7 +4731,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4667,7 +4738,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4676,7 +4746,6 @@ class Server extends AppModel 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), 'value' => false, 'null' => true, - 'errorMessage' => '', 'test' => 'testDisableCache', 'type' => 'boolean', 'afterHook' => 'disableCacheAfterHook', @@ -4686,7 +4755,6 @@ class Server extends AppModel 'description' => __('Disable displaying / modifications to the threat level altogether on the instance (deprecated field).'), 'value' => false, 'null' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -4694,7 +4762,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4702,7 +4769,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Footer text prepending the "Powered by MISP" text.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4710,7 +4776,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Footer text following the "Powered by MISP" text.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4718,7 +4783,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4726,7 +4790,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4734,7 +4797,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4742,31 +4804,27 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), 'footer_logo' => array( - 'level' => 2 , + 'level' => 2, 'description' => __('If set, this setting allows you to display a logo on the right side of the footer. Upload it as a custom image in the file management tool.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCustomImage', 'type' => 'string', ), 'home_logo' => array( - 'level' => 2 , + 'level' => 2, 'description' => __('If set, this setting allows you to display a logo as the home icon. Upload it as a custom image in the file management tool.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCustomImage', 'type' => 'string', ), 'main_logo' => array( - 'level' => 2 , + 'level' => 2, 'description' => __('If set, the image specified here will replace the main MISP logo on the login screen. Upload it as a custom image in the file management tool.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCustomImage', 'type' => 'string', ), @@ -4774,7 +4832,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The organisation tag of the hosting organisation. This is used in the e-mail subjects.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4782,7 +4839,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The hosting organisation of this instance. If this is not selected then replication instances cannot be added.'), 'value' => '0', - 'errorMessage' => '', 'test' => 'testLocalOrgStrict', 'type' => 'numeric', 'optionsSource' => function () { @@ -4801,7 +4857,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4809,7 +4864,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Setting this setting to \'false\' will hide all organisation names / logos.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -4817,7 +4871,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Put the event threat level in the notification E-mail subject.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -4825,7 +4878,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), 'value' => 'tlp:amber', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4833,7 +4885,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('If this tag is set on an event it\'s value will be sent in the E-mail subject. If the tag is not set the email_subject_TLP_string will be used.'), 'value' => 'tlp', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4841,7 +4892,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Include in name of the email_subject_tag in the subject. When false only the tag value is used.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -4849,7 +4899,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Notification e-mail sender name.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ], @@ -4857,7 +4906,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4865,7 +4913,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4873,33 +4920,22 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enables the use of MISP\'s background processing.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBoolTrue', 'type' => 'boolean', ), 'attachments_dir' => array( 'level' => 2, 'description' => __('Directory where attachments are stored. MISP will NOT migrate the existing data if you change this setting. The only safe way to change this setting is in config.php, when MISP is not running, and after having moved/copied the existing data to the new location. This directory must already exist and be writable and readable by the MISP application.'), - 'value' => APP . '/files', # GUI display purpose only. - 'errorMessage' => '', + 'value' => APP . '/files', # GUI display purpose only. 'null' => false, 'test' => 'testForWritableDir', 'type' => 'string', 'cli_only' => 1 ), - 'cached_attachments' => array( - 'level' => 1, - 'description' => __('Allow the XML caches to include the encoded attachments.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), 'download_attachments_on_load' => array( 'level' => 2, 'description' => __('Always download attachments when loaded by a user in a browser'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -4907,7 +4943,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The Unix user MISP (php) is running as'), 'value' => 'www-data', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4915,7 +4950,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The e-mail address that MISP should use for all notifications'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4923,7 +4957,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('You can disable all e-mailing using this setting. When enabled, no outgoing e-mails will be sent by MISP.'), 'value' => false, - 'errorMessage' => '', 'null' => true, 'test' => 'testDisableEmail', 'type' => 'boolean', @@ -4932,7 +4965,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This setting is deprecated. Please use `MISP.event_alert_metadata_only` instead.'), 'value' => false, - 'errorMessage' => '', 'null' => true, 'test' => 'testBool', 'type' => 'boolean', @@ -4941,7 +4973,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The e-mail address that MISP should include as a contact address for the instance\'s support team.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4949,7 +4980,6 @@ class Server extends AppModel 'level' => 3, 'description' => __('This setting is deprecated and can be safely removed.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4957,7 +4987,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Turn Vulnerability type attributes into links linking to the provided CVE lookup'), 'value' => 'https://cve.circl.lu/cve/', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4965,7 +4994,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Turn Weakness type attributes into links linking to the provided CWE lookup'), 'value' => 'https://cve.circl.lu/cwe/', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -4973,7 +5001,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This setting controls whether notification e-mails will be sent when an event is created via the REST interface. It might be a good idea to disable this setting when first setting up a link to another instance to avoid spamming your users during the initial pull. Quick recap: True = Emails are NOT sent, False = Emails are sent on events published via sync / REST.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -4981,7 +5008,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling this flag will allow the event description to be transmitted in the alert e-mail\'s subject. Be aware that this is not encrypted by GnuPG, so only enable it if you accept that part of the event description will be sent out in clear-text.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -4989,7 +5015,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('If enabled, any requested URL before login will have their HTTP part replaced by HTTPS. This can be usefull if MISP is running behind a reverse proxy responsible for SSL and communicating unencrypted with MISP.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -4997,7 +5022,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Send just event metadata (attributes and objects will be omitted) for event alert.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ], @@ -5005,7 +5029,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The default distribution setting for events (0-3).'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities')), @@ -5014,16 +5037,20 @@ class Server extends AppModel 'level' => 0, 'description' => __('The default distribution setting for attributes, set it to \'event\' if you would like the attributes to default to the event distribution level. (0-3 or "event")'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', - 'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities'), 'event' => __('Inherit from event')), + 'options' => array( + '0' => __('Your organisation only'), + '1' => __('This community only'), + '2' => __('Connected communities'), + '3' => __('All communities'), + 'event' => __('Inherit from event') + ), ), 'default_event_threat_level' => array( 'level' => 1, 'description' => __('The default threat level setting when creating events.'), 'value' => 4, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'options' => array('1' => 'High', '2' => 'Medium', '3' => 'Low', '4' => 'undefined'), @@ -5032,7 +5059,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The tag collection to be applied to all events created manually.'), 'value' => 0, - 'errorMessage' => '', 'test' => 'testTagCollections', 'type' => 'numeric', 'optionsSource' => function () { @@ -5043,7 +5069,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The default setting for publish alerts when creating users.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5052,7 +5077,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable the tagging feature of MISP. This is highly recommended.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5060,7 +5084,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Show the full tag names on the event index.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'options' => array(0 => 'Minimal tags', 1 => 'Full tags', 2 => 'Shortened tags'), @@ -5069,7 +5092,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Used on the login page, before the MISP logo'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5077,7 +5099,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Used on the login page, after the MISP logo'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5085,7 +5106,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Used on the login page, to the left of the MISP logo, upload it as a custom image in the file management tool.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCustomImage', 'type' => 'string', ), @@ -5093,7 +5113,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Used on the login page, to the right of the MISP logo, upload it as a custom image in the file management tool.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCustomImage', 'type' => 'string', ), @@ -5101,7 +5120,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Used in the page title, after the name of the page'), 'value' => 'MISP', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5109,7 +5127,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Allows users to take ownership of an event uploaded via the "Add MISP XML" button. This allows spoofing the creator of a manually imported event, also breaking possibly breaking the original intended releasability. Synchronising with an instance that has a different creator for the same event can lead to unwanted consequences.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5117,7 +5134,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Choose whether the terms and conditions should be displayed inline (false) or offered as a download (true)'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -5125,7 +5141,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The filename of the terms and conditions file. Make sure that the file is located in your MISP/app/files/terms directory'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForTermsFile', 'type' => 'string' ), @@ -5133,7 +5148,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -5141,7 +5155,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('True will deny access to unpublished events to users outside the organization of the submitter except site admins.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -5150,7 +5163,6 @@ class Server extends AppModel 'bigField' => true, 'description' => __('The message sent to the user after account creation (has to be sent manually from the administration interface). Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $org = the organisation that the instance belongs to, as set in MISP.org, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), 'value' => 'Dear new MISP user,\n\nWe would hereby like to welcome you to the $org MISP community.\n\n Use the credentials below to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nPassword: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', 'test' => 'testPasswordResetText', 'type' => 'string' ), @@ -5159,7 +5171,6 @@ class Server extends AppModel 'bigField' => true, 'description' => __('The message sent to the users when a password reset is triggered. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), 'value' => 'Dear MISP user,\n\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nYour temporary password: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', 'test' => 'testPasswordResetText', 'type' => 'string' ), @@ -5181,7 +5192,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If enabled, all log entries will include the IP address of the user.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'beforeHook' => 'ipLogBeforeHook' @@ -5190,7 +5200,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy infront of your MISP instance.'), 'value' => 'REMOTE_ADDR', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true, @@ -5199,7 +5208,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5216,7 +5224,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('If this functionality is enabled all page requests will be logged. Keep in mind this is extremely verbose and will become a burden to your database.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', 'null' => true @@ -5225,7 +5232,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('You can decide to skip the logging of the paranoid logs to the database.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testParanoidSkipDb', 'type' => 'boolean', 'null' => true @@ -5234,7 +5240,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('If paranoid logging is enabled, include the POST body in the entries.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5243,7 +5248,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Log user IPs on each request. 30 day retention for lookups by IP to get the last authenticated user ID for the given IP, whilst on the reverse, indefinitely stores all associated IPs for a user ID.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5252,7 +5256,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('Log user IP and key usage on each API request. All logs for given keys are deleted after one year when this key is not used.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5261,7 +5264,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('Enable new audit log system.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5270,7 +5272,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Compress log changes by brotli algorithm. This will reduce log database size.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5279,7 +5280,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This feature allows users to create org only events and ask another organisation to take ownership of the event. This allows organisations to remain anonymous by asking a partner to publish an event for them.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5288,7 +5288,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled, the number of correlations visible to the currently logged in user will be visible on the event index UI. This comes at a performance cost but can be very useful to see correlating events at a glance.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5297,7 +5296,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled, the number of proposals for the events are shown on the index.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5306,7 +5304,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled, the aggregate number of attribute sightings within the event becomes visible to the currently logged in user on the event index UI.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5315,7 +5312,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled, the aggregate number of discussion posts for the event becomes visible to the currently logged in user on the event index UI.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5324,7 +5320,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled, the aggregate number of event reports for the event becomes visible to the currently logged in user on the event index UI.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5333,7 +5328,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When enabled only Org and Site admins can edit a user\'s profile.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5342,7 +5336,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('When enabled only Site admins can change user email. This should be enabled if you manage user logins by external system.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5351,7 +5344,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('When enabled only Site admins can change user password. This should be enabled if you manage user passwords by external system.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5360,7 +5352,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('When enabled, Org Admins could not add new users. This should be enabled if you manage users by external system.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5369,7 +5360,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to start blocking alert e-mails for events with a certain tag. Define the tag in MISP.block_event_alert_tag.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5378,7 +5368,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.block_event_alert setting is set, alert e-mails for events tagged with the tag defined by this setting will be blocked.'), 'value' => 'no-alerts="true"', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => false, @@ -5387,7 +5376,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to start blocking alert e-mails for events that have already been published since a specified amount of time. This threshold is defined by MISP.event_alert_republish_ban_threshold'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5396,7 +5384,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control how long no alerting by email will be done. Expected format: integer, in minutes'), 'value' => 5, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => false, @@ -5405,7 +5392,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control if a ban time should be reset if emails are tried to be sent during the ban.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5414,7 +5400,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to start blocking users to send too many e-mails notification since a specified amount of time. This threshold is defined by MISP.user_email_notification_ban_threshold'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5423,7 +5408,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how long no notification by email will be done. Expected format: integer, in minutes'), 'value' => 120, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => false, @@ -5432,7 +5416,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how many notification by email can be send for the timeframe defined in MISP.user_email_notification_ban_time_threshold. Expected format: integer'), 'value' => 10, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => false, @@ -5441,7 +5424,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'), 'value' => 0, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true, @@ -5450,7 +5432,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to start blocking alert e-mails for old events. The exact timing of what constitutes an old event is defined by MISP.block_old_event_alert_age.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5459,7 +5440,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control how old an event can be for it to be alerted on. The "timestamp" field of the event is used. Expected format: integer, in days'), 'value' => false, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => false, @@ -5468,7 +5448,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control the threshold for the event.date field, indicating how old an event can be for it to be alerted on. The "date" field of the event is used. Expected format: integer, in days'), 'value' => false, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => false, @@ -5477,7 +5456,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Please indicate the temp directory you wish to use for certain functionalities in MISP. By default this is set to /tmp and will be used among others to store certain temporary files extracted from imports during the import process.'), 'value' => '/tmp', - 'errorMessage' => '', 'test' => 'testForPath', 'type' => 'string', 'null' => true, @@ -5487,7 +5465,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForStyleFile', 'type' => 'string', 'null' => true, @@ -5496,7 +5473,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable this setting to allow blocking attributes from to_ids sensitive exports if a proposal has been made to it to remove the IDS flag or to remove the attribute altogether. This is a powerful tool to deal with false-positives efficiently.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false, @@ -5505,7 +5481,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this settings if new tags synced / added via incoming events from any source should not be selectable by users by default.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => false @@ -5514,7 +5489,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('*WARNING* This setting will completely disable the correlation on this instance and remove any existing saved correlations. Enabling this will trigger a full recorrelation of all data which is an extremely long and costly procedure. Only enable this if you know what you\'re doing.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', 'null' => true, @@ -5524,7 +5498,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('*WARNING* This setting will give event creators the possibility to disable the correlation of individual events / attributes that they have created.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', 'null' => true @@ -5533,7 +5506,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The host running the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), 'value' => '127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -5541,7 +5513,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The port used by the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), 'value' => 6379, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric' ), @@ -5549,7 +5520,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The database on the redis server to be used for generic MISP tasks. If you run more than one MISP instance, please make sure to use a different database on each instance.'), 'value' => 13, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric' ), @@ -5557,7 +5527,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The password on the redis server (if any) to be used for generic MISP tasks.'), 'value' => '', - 'errorMessage' => '', 'test' => null, 'type' => 'string', 'redacted' => true @@ -5566,7 +5535,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'), 'value' => 'id, uuid, value, comment, type, category, Tag.name', - 'errorMessage' => '', 'test' => null, 'type' => 'string', ), @@ -5574,7 +5542,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Set this to false if you would like to disable MISP managing its own worker processes (for example, if you are managing the workers with a systemd unit).'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -5582,7 +5549,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Only enable this if you have some tools using MISP with extreme high concurency. General performance will be lower as normal as certain transactional queries are avoided in favour of shorter table locks.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5599,7 +5565,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5608,7 +5573,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'), 'value' => '', - 'errorMessage' => '', 'type' => 'string', 'null' => true, ], @@ -5616,7 +5580,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -5625,20 +5588,41 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('How long to wait for scan results in seconds.'), 'value' => 30, - 'errorMessage' => '', 'test' => 'testForPositiveInteger', 'type' => 'numeric', 'null' => true, ], 'warning_for_all' => [ - 'level' => 1, - 'description' => __('Enable warning list triggers regardless of the IDS flag value'), + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('Enable warning list triggers regardless of the IDS flag value.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true ], + 'system_setting_db' => [ + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('Enable storing setting in database.'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + 'cli_only' => true, + ], + 'menu_custom_right_link' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Custom right menu URL.'), + 'value' => null, + 'type' => 'string', + 'null' => true, + ], + 'menu_custom_right_link_html' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Custom right menu text (it is possible to use HTML).'), + 'value' => null, + 'type' => 'string', + 'null' => true, + ], ), 'GnuPG' => array( 'branch' => 1, @@ -5646,7 +5630,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The location of the GnuPG executable. If you would like to use a different GnuPG executable than /usr/bin/gpg, you can set it here. If the default is fine, just keep the setting suggested by MISP.'), 'value' => '/usr/bin/gpg', - 'errorMessage' => '', 'test' => 'testForGPGBinary', 'type' => 'string', 'cli_only' => 1 @@ -5655,7 +5638,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Allow (false) unencrypted e-mails to be sent to users that don\'t have a GnuPG key.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5663,7 +5645,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Allow (false) the body of unencrypted e-mails to contain details about the event.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5671,7 +5652,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enable the signing of GnuPG emails. By default, GnuPG emails are signed'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5679,7 +5659,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The e-mail address that the instance\'s GnuPG key is tied to.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5687,7 +5666,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The password (if it is set) of the GnuPG key of the instance.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'redacted' => true @@ -5696,7 +5674,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The location of the GnuPG homedir.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5704,7 +5681,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('When enabled, the subject in signed and encrypted e-mails will not be sent in unencrypted form.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ) @@ -5715,7 +5691,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enable S/MIME encryption. The encryption posture of the GnuPG.onlyencrypted and GnuPG.bodyonlyencrypted settings are inherited if S/MIME is enabled.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5723,7 +5698,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The e-mail address that the instance\'s S/MIME key is tied to.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5731,7 +5705,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The location of the public half of the signing certificate.'), 'value' => '/var/www/MISP/.smime/email@address.com.pem', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5739,7 +5712,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The location of the private half of the signing certificate.'), 'value' => '/var/www/MISP/.smime/email@address.com.key', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5747,7 +5719,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The password (if it is set) of the S/MIME key of the instance.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'redacted' => true @@ -5759,7 +5730,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5767,7 +5737,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The TCP port for the HTTP proxy.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric', ), @@ -5775,7 +5744,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5783,7 +5751,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The authentication username for the HTTP proxy.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5791,7 +5758,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The authentication password for the HTTP proxy.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -5811,7 +5777,6 @@ class Server extends AppModel 'level' => self::SETTING_CRITICAL, 'description' => __('Enforce CSP. Content Security Policy (CSP) is an added layer of security that helps to detect and mitigate certain types of attacks, including Cross Site Scripting (XSS) and data injection attacks. When disabled, violations will be just logged.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ], @@ -5819,17 +5784,15 @@ class Server extends AppModel 'level' => 0, 'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testSalt', 'type' => 'string', 'editable' => false, 'redacted' => true ), - 'log_each_individual_auth_fail' =>[ + 'log_each_individual_auth_fail' => [ 'level' => 1, 'description' => __('By default API authentication failures that happen within the same hour for the same key are omitted and a single log entry is generated. This allows administrators to more easily keep track of attackers that try to brute force API authentication, by reducing the noise generated by expired API keys. On the other hand, this makes little sense for internal MISP instances where detecting the misconfiguration of tools becomes more interesting, so if you fall into the latter category, enable this feature.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ], @@ -5837,7 +5800,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Advanced authkeys will allow each user to create and manage a set of authkeys for themselves, each with individual expirations and comments. API keys are stored in a hashed state and can no longer be recovered from MISP. Users will be prompted to note down their key when creating a new authkey. You can generate a new set of API keys for all users on demand in the diagnostics page, or by triggering %s.', sprintf('%s', $this->baseurl, __('the advanced upgrade'))), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -5845,7 +5807,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Maximal key lifetime in days. Use can limit that validity even more. Just newly created keys will be affected. When not set, key validity is not limited.'), 'value' => '', - 'errorMessage' => '', 'type' => 'numeric', 'test' => 'testForNumeric', 'null' => true, @@ -5854,7 +5815,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('When enabled, session is kept between API requests.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -5863,7 +5823,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('This optionally can be enabled if an external auth provider is used. When set to true, it will disable the default form authentication.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ], @@ -5871,7 +5830,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5880,7 +5838,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'), 'value' => false, - 'errorMessage' => '', 'test' => null, 'type' => 'string' ), @@ -5888,7 +5845,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5897,7 +5853,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Write syslog messages also to standard error output.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5906,7 +5861,6 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('Syslog message identifier.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -5915,7 +5869,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -5924,7 +5877,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -5933,7 +5885,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('If enabled, any POST, PUT or AJAX request will be allow just when Sec-Fetch-Site header is not defined or contains "same-origin".'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -5942,35 +5893,31 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('If enabled, MISP server will consider all requests as secure. This is usually useful when you run MISP behind reverse proxy that terminates HTTPS.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, ], 'email_otp_enabled' => array( - 'level'=> 2, + 'level' => 2, 'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'beforeHook' => 'otpBeforeHook', 'type' => 'boolean', 'null' => true ), - 'email_otp_length' => array ( + 'email_otp_length' => array( 'level' => 2, 'description' => __('Define the length of the OTP code sent by email'), 'value' => '6', - 'errorMessage' => '', 'type' => 'numeric', 'test' => 'testForNumeric', 'null' => true, ), - 'email_otp_validity' => array ( + 'email_otp_validity' => array( 'level' => 2, 'description' => __('Define the validity (in minutes) of the OTP code sent by email'), 'value' => '5', - 'errorMessage' => '', 'type' => 'numeric', 'test' => 'testForNumeric', 'null' => true, @@ -5980,7 +5927,6 @@ class Server extends AppModel 'bigField' => true, 'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'), 'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true, @@ -5990,7 +5936,6 @@ class Server extends AppModel 'bigField' => true, 'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true, @@ -5999,7 +5944,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6009,7 +5953,6 @@ class Server extends AppModel 'bigField' => true, 'description' => __('The message sent shown to anyone trying to self-register.'), 'value' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.', - 'errorMessage' => '', 'test' => false, 'type' => 'string' ), @@ -6017,7 +5960,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'), 'value' => '12', - 'errorMessage' => '', 'test' => 'testPasswordLength', 'type' => 'numeric', ), @@ -6025,7 +5967,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Default (simple 3 out of 4 or minimum 16 characters): /^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/'), 'value' => '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/', - 'errorMessage' => '', 'test' => 'testPasswordRegex', 'type' => 'string', ), @@ -6033,7 +5974,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling this setting will require users to submit their current password on any edits to their profile (including a triggered password change). For administrators, the confirmation will be required when changing the profile of any user. Could potentially mitigate an attacker trying to change a compromised user\'s password in order to establish persistance, however, enabling this feature will be highly annoying to users.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6042,7 +5982,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling this setting will sanitise the contents of an attribute on a soft delete'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6051,7 +5990,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling this setting will block the organisation index from being visible to anyone besides site administrators on the current instance. Keep in mind that users can still see organisations that produce data via events, proposals, event history log entries, etc.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6060,7 +5998,6 @@ class Server extends AppModel 'level' => self::SETTING_RECOMMENDED, 'description' => __('Enabling this setting will block the organisation list from being visible in sharing group besides user with sharing group permission.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6069,7 +6006,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Disabling this setting will allow the creation/modification of local feeds (as opposed to network feeds). Enabling this setting will restrict feed sources to be network based only. When disabled, keep in mind that a malicious site administrator could get access to any arbitrary file on the system that the apache user has access to. Make sure that proper safe-guards are in place. This setting can only be modified via the CLI.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -6088,7 +6024,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Allow cross-origin requests to this instance, matching origins given in Security.cors_origins. Set to false to totally disable'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6097,7 +6032,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set the origins from which MISP will allow cross-origin requests. Useful for external integration. Comma seperate if you need more than one.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -6106,7 +6040,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable this setting to create verbose logs of synced event data for debugging reasons. Logs are saved in your MISP directory\'s app/files/scripts/tmp/ directory.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', 'null' => true @@ -6115,7 +6048,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enables the functionality to monitor users - thereby enabling all logging functionalities for a single user. This functionality is intrusive and potentially heavy on the system - use it with care.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6124,11 +6056,37 @@ class Server extends AppModel 'level' => self::SETTING_OPTIONAL, 'description' => __('When enabled, logged in username will be included in X-Username HTTP response header. This is useful for request logging on webserver/proxy side.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true - ] + ], + 'encryption_key' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Encryption key used to store sensitive data (like authkeys) in database encrypted. If empty, data are stored unecrypted. Required PHP 7.1 or newer.'), + 'value' => '', + 'test' => function ($value) { + if (strlen($value) < 32) { + return __('Encryption key must be at least 32 chars long.'); + } + return true; + }, + 'afterHook' => function ($setting, $new, $old) { + /** @var SystemSetting $systemSetting */ + $systemSetting = ClassRegistry::init('SystemSetting'); + $systemSetting->reencrypt($old, $new); + + $this->reencryptAuthKeys($old, $new); + + /** @var Cerebrate $cerebrate */ + $cerebrate = ClassRegistry::init('Cerebrate'); + $cerebrate->reencryptAuthKeys($old, $new); + return true; + }, + 'type' => 'string', + 'null' => true, + 'cli_only' => true, + 'redacted' => true, + ], ), 'SecureAuth' => array( 'branch' => 1, @@ -6136,7 +6094,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The number of tries a user can try to login and fail before the bruteforce protection kicks in.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'string', ), @@ -6144,7 +6101,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The duration (in seconds) of how long the user will be locked out when the allowed number of login attempts are exhausted.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'string', ), @@ -6155,7 +6111,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Set to true to automatically regenerate sessions after x number of requests. This might lead to the user getting de-authenticated and is frustrating in general, so only enable it if you really need to regenerate sessions. (Not recommended)'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', ), @@ -6163,7 +6118,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Set to true to check for the user agent string in each request. This can lead to occasional logouts (not recommended).'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBoolFalse', 'type' => 'boolean', ), @@ -6171,7 +6125,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The session type used by MISP. The default setting is php, which will use the session settings configured in php.ini for the session data (supported options: php, database). The recommended option is php and setting your PHP up to use redis sessions via your php.ini. Just add \'session.save_handler = redis\' and "session.save_path = \'tcp://localhost:6379\'" (replace the latter with your redis connection) to '), 'value' => '', - 'errorMessage' => '', 'test' => 'testForSessionDefaults', 'type' => 'string', 'options' => array('php' => 'php', 'database' => 'database', 'cake' => 'cake', 'cache' => 'cache'), @@ -6180,7 +6133,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The timeout duration of sessions (in MINUTES). 0 does not mean infinite for the PHP session handler, instead sessions will invalidate immediately.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'string' ), @@ -6188,7 +6140,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The expiration of the cookie (in MINUTES). The session timeout gets refreshed frequently, however the cookies do not. Generally it is recommended to have a much higher cookie_timeout than timeout.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForCookieTimeout', 'type' => 'numeric' ) @@ -6199,16 +6150,14 @@ class Server extends AppModel 'level' => 2, 'description' => __('The default policy action for the values added to the RPZ.'), 'value' => 1, - 'errorMessage' => '', 'test' => 'testForRPZBehaviour', 'type' => 'numeric', - 'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only' ), + 'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only'), ), 'RPZ_walled_garden' => array( 'level' => 2, 'description' => __('The default walled garden used by the RPZ export if the Local-Data policy setting is picked for the export.'), 'value' => '127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6216,7 +6165,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The serial in the SOA portion of the zone file. (numeric, best practice is yyyymmddrr where rr is the two digit sub-revision of the file. $date will automatically get converted to the current yyyymmdd, so $date00 is a valid setting). Setting it to $time will give you an unixtime-based serial (good then you need more than 99 revisions per day).'), 'value' => '$date00', - 'errorMessage' => '', 'test' => 'testForRPZSerial', 'type' => 'string', ), @@ -6224,7 +6172,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The refresh specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), 'value' => '2h', - 'errorMessage' => '', 'test' => 'testForRPZDuration', 'type' => 'string', ), @@ -6232,7 +6179,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The retry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), 'value' => '30m', - 'errorMessage' => '', 'test' => 'testForRPZDuration', 'type' => 'string', ), @@ -6240,7 +6186,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The expiry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), 'value' => '30d', - 'errorMessage' => '', 'test' => 'testForRPZDuration', 'type' => 'string', ), @@ -6248,7 +6193,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The minimum TTL specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), 'value' => '1h', - 'errorMessage' => '', 'test' => 'testForRPZDuration', 'type' => 'string', ), @@ -6256,7 +6200,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The TTL of the zone file. (in seconds, or shorthand duration such as 15m)'), 'value' => '1w', - 'errorMessage' => '', 'test' => 'testForRPZDuration', 'type' => 'string', ), @@ -6264,7 +6207,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Nameserver'), 'value' => 'localhost.', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6272,7 +6214,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Alternate nameserver'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6280,7 +6221,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The e-mail address specified in the SOA portion of the zone file.'), 'value' => 'root.localhost', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6288,7 +6228,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -6296,7 +6235,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('A comma separated list of Kafka bootstrap brokers'), 'value' => 'kafka:9092', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6304,7 +6242,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'), 'value' => '/etc/rdkafka.ini', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', ), @@ -6312,7 +6249,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6320,7 +6256,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6328,7 +6263,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing event creations/edits/deletions.'), 'value' => 'misp_event', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6336,7 +6270,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6344,7 +6277,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing event information on publish.'), 'value' => 'misp_event_publish', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6352,7 +6284,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6360,7 +6291,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing object creations/edits/deletions.'), 'value' => 'misp_object', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6368,7 +6298,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6376,7 +6305,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing object reference creations/deletions.'), 'value' => 'misp_object_reference', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6384,7 +6312,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6392,7 +6319,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing attribute creations/edits/soft deletions.'), 'value' => 'misp_attribute', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6400,7 +6326,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6408,7 +6333,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing proposal creations/edits/deletions.'), 'value' => 'misp_shadow_attribute', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6416,7 +6340,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6424,7 +6347,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), 'value' => 'misp_tag', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6432,7 +6354,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new sightings.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6440,7 +6361,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing sightings.'), 'value' => 'misp_sighting', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6448,7 +6368,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new/modified users.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6456,7 +6375,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing new/modified users.'), 'value' => 'misp_user', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6464,7 +6382,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new/modified organisations.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6472,7 +6389,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing new/modified organisations.'), 'value' => 'misp_organisation', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6480,7 +6396,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6488,7 +6403,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Topic for publishing log entries.'), 'value' => 'misp_audit', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6496,7 +6410,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'afterHook' => 'zmqAfterHook', @@ -6505,7 +6418,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The host that the pub/sub feature will use.'), 'value' => '127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6514,7 +6426,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The port that the pub/sub feature will use.'), 'value' => 50000, - 'errorMessage' => '', 'test' => 'testForZMQPortNumber', 'type' => 'numeric', 'afterHook' => 'zmqAfterHook', @@ -6523,7 +6434,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The username that client need to use to connect to ZeroMQ.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6532,7 +6442,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The password that client need to use to connect to ZeroMQ.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6541,7 +6450,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'), 'value' => 'localhost', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6550,7 +6458,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The port that Redis is listening on.'), 'value' => 6379, - 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric', 'afterHook' => 'zmqAfterHook', @@ -6559,7 +6466,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The password, if set for Redis.'), 'value' => '', - 'errorMessage' => '', 'type' => 'string', 'afterHook' => 'zmqAfterHook', ), @@ -6567,7 +6473,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The database to be used for queuing messages for the pub/sub functionality.'), 'value' => 1, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6576,7 +6481,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The namespace to be used for queuing messages for the pub/sub functionality.'), 'value' => 'mispq', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'afterHook' => 'zmqAfterHook', @@ -6585,7 +6489,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6593,7 +6496,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6601,7 +6503,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6609,7 +6510,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6617,7 +6517,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6625,7 +6524,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6633,7 +6531,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new sightings to the ZMQ pubsub feed.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6641,7 +6538,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new/modified users to the ZMQ pubsub feed.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6649,7 +6545,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of new/modified organisations to the ZMQ pubsub feed.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6657,15 +6552,13 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables the publishing of log entries to the ZMQ pubsub feed. Keep in mind, this can get pretty verbose depending on your logging settings.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), - 'ZeroMQ_warninglist_notifications_enable' => array( + 'ZeroMQ_warninglist_notifications_enable' => array( 'level' => 2, 'description' => __('Enables or disables the publishing of new/modified warninglist to the ZMQ pubsub feed.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6673,7 +6566,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enabled logging to an ElasticSearch instance'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6681,7 +6573,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The URL(s) at which to access ElasticSearch - comma separate if you want to have more than one.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6689,7 +6580,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The index in which to place logs'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6697,7 +6587,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6737,7 +6626,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Bucket name to upload to, please make sure that the bucket exists. We will not create the bucket for you'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6745,7 +6633,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Region in which your S3 bucket resides'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6753,7 +6640,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6761,7 +6647,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('AWS secret key to use when uploading samples'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6769,7 +6654,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner organisation alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and any organisation that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'), 'value' => 0, - 'errorMessage' => '', 'test' => 'testForSightingVisibility', 'type' => 'numeric', 'options' => array( @@ -6783,7 +6667,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enabling the anonymisation of sightings will simply aggregate all sightings instead of showing the organisations that have reported a sighting. Users will be able to tell the number of sightings their organisation has submitted and the number of sightings for other organisations'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', ), @@ -6791,7 +6674,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('When pushing sightings to another server, report all sightings from this instance as this organisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'), 'value' => '0', - 'errorMessage' => '', 'test' => 'testLocalOrg', 'type' => 'numeric', 'optionsSource' => function () { @@ -6802,7 +6684,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'), 'value' => 365, - 'errorMessage' => '', 'test' => 'testForNumeric', 'type' => 'numeric' ), @@ -6810,7 +6691,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable SightingDB integration.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6818,7 +6698,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true, @@ -6828,7 +6707,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'), 'value' => 'Authorization', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -6837,7 +6715,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Use a header namespace for the auth header - default setting is enabled'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6846,7 +6723,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The default header namespace for the auth header - default setting is HTTP_'), 'value' => 'HTTP_', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -6855,7 +6731,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('If this setting is enabled then the only way to authenticate will be using the custom header. Alternatively, you can run in mixed mode that will log users in via the header if found, otherwise users will be redirected to the normal login page.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -6864,7 +6739,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('If you are using an external tool to authenticate with MISP and would like to only allow the tool\'s url as a valid point of entry then set this field. '), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -6873,7 +6747,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('The name of the authentication method, this is cosmetic only and will be shown on the user creation page and logs.'), 'value' => 'External authentication', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -6882,7 +6755,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Disable the logout button for users authenticate with the external auth mechanism.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6890,7 +6762,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable/disable the enrichment services'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6898,7 +6769,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a timeout for the enrichment services'), 'value' => 10, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -6906,7 +6776,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable/disable the import services'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6914,7 +6783,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a timeout for the import services'), 'value' => 10, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -6922,7 +6790,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The url used to access the import services. By default, it is accessible at http://127.0.0.1:6666'), 'value' => 'http://127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6930,7 +6797,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The port used to access the import services. By default, it is accessible at 127.0.0.1:6666'), 'value' => '6666', - 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric' ), @@ -6938,7 +6804,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The url used to access the export services. By default, it is accessible at http://127.0.0.1:6666'), 'value' => 'http://127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -6946,7 +6811,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The port used to access the export services. By default, it is accessible at 127.0.0.1:6666'), 'value' => '6666', - 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric' ), @@ -6954,7 +6818,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable/disable the export services'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6962,7 +6825,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a timeout for the export services'), 'value' => 10, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -6970,7 +6832,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6978,7 +6839,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('When enabled, users have to click on the magnifier icon to show the enrichment'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -6986,7 +6846,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a timeout for the hover services'), 'value' => 5, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -6994,7 +6853,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The url used to access the enrichment services. By default, it is accessible at http://127.0.0.1:6666'), 'value' => 'http://127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -7002,7 +6860,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The port used to access the enrichment services. By default, it is accessible at 127.0.0.1:6666'), 'value' => 6666, - 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric' ), @@ -7010,7 +6867,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'), 'value' => 'http://127.0.0.1', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string' ), @@ -7018,7 +6874,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('The port used to access Cortex. By default, this is port 9000'), 'value' => 9000, - 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric' ), @@ -7026,7 +6881,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('Enable/disable the Cortex services'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean' ), @@ -7034,7 +6888,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set an authentication key to be passed to Cortex'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -7043,7 +6896,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set a timeout for the Cortex services'), 'value' => 120, - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'numeric' ), @@ -7051,7 +6903,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set to false to disable SSL verification. This is not recommended.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -7060,7 +6911,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set to false if you wish to ignore hostname match errors when validating certificates.'), 'value' => true, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -7069,7 +6919,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set to true to enable self-signed certificates to be accepted. This requires Cortex_ssl_verify_peer to be enabled.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -7078,7 +6927,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Set to the absolute path of the Certificate Authority file that you wish to use for verifying SSL certificates.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -7087,7 +6935,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Provide your custom authentication users with an external URL to the authentication system to reset their passwords.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -7096,7 +6943,6 @@ class Server extends AppModel 'level' => 2, 'description' => __('Provide a custom logout URL for your users that will log them out using the authentication system you use.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true @@ -7105,7 +6951,6 @@ class Server extends AppModel 'level' => 1, 'description' => __('Enable lookups for additional relations via CyCat.'), 'value' => false, - 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', 'null' => true @@ -7114,17 +6959,97 @@ class Server extends AppModel 'level' => 2, 'description' => __('URL to use for CyCat lookups, if enabled.'), 'value' => 'https://api.cycat.org', - 'errorMessage' => '', 'test' => 'testForEmpty', 'type' => 'string', 'null' => true ] ), + 'SimpleBackgroundJobs' => [ + 'branch' => 1, + 'enabled' => [ + 'level' => 2, + 'description' => __('Enables or disables background jobs with Supervisor backend.'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean' + ], + 'redis_host' => [ + 'level' => 2, + 'description' => __('The host running the redis server to be used for background jobs.'), + 'value' => '127.0.0.1', + 'test' => 'testForEmpty', + 'type' => 'string' + ], + 'redis_port' => [ + 'level' => 2, + 'description' => __('The port used by the redis server to be used for background jobs.'), + 'value' => 6379, + 'test' => 'testForNumeric', + 'type' => 'numeric' + ], + 'redis_database' => [ + 'level' => 2, + 'description' => __('The database on the redis server to be used for background jobs. If you run more than one MISP instance, please make sure to use a different database or redis_namespace on each instance.'), + 'value' => 1, + 'test' => 'testForNumeric', + 'type' => 'numeric' + ], + 'redis_password' => [ + 'level' => 2, + 'description' => __('The password on the redis server (if any) to be used for background jobs.'), + 'value' => '', + 'test' => null, + 'type' => 'string', + 'redacted' => true + ], + 'redis_namespace' => [ + 'level' => 2, + 'description' => __('The namespace to be used for the background jobs related keys.'), + 'value' => 'background_jobs', + 'test' => null, + 'type' => 'string' + ], + 'max_job_history_ttl' => [ + 'level' => 2, + 'description' => __('The time in seconds the job statuses history will be kept.'), + 'value' => 86400, + 'test' => 'testForNumeric', + 'type' => 'numeric' + ], + 'supervisor_host' => [ + 'level' => 2, + 'description' => __('The host where the Supervisor XML-RPC API is running.'), + 'value' => 'localhost', + 'test' => 'testForEmpty', + 'type' => 'string' + ], + 'supervisor_port' => [ + 'level' => 2, + 'description' => __('The port where the Supervisor XML-RPC API is running.'), + 'value' => 9001, + 'test' => 'testForNumeric', + 'type' => 'numeric' + ], + 'supervisor_user' => [ + 'level' => 2, + 'description' => __('The user of the Supervisor XML-RPC API.'), + 'value' => '', + 'test' => null, + 'type' => 'string' + ], + 'supervisor_password' => [ + 'level' => 2, + 'description' => __('The password of the Supervisor XML-RPC API.'), + 'value' => '', + 'test' => null, + 'type' => 'string', + 'redacted' => true + ], + ], 'debug' => array( 'level' => 0, 'description' => __('The debug level of the instance, always use 0 for production instances.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testDebug', 'type' => 'numeric', 'options' => array(0 => 'Debug off', 1 => 'Debug on', 2 => 'Debug + SQL dump'), @@ -7133,7 +7058,6 @@ class Server extends AppModel 'level' => 0, 'description' => __('The debug level of the instance for site admins. This feature allows site admins to run debug mode on a live instance without exposing it to other users. The most verbose option of debug and site_admin_debug is used for site admins.'), 'value' => '', - 'errorMessage' => '', 'test' => 'testDebugAdmin', 'type' => 'boolean', 'null' => true @@ -7237,4 +7161,36 @@ class Server extends AppModel ) ); } + + /** + * Get workers + * + * @return array + */ + private function getWorkers(): array + { + if (!Configure::read('SimpleBackgroundJobs.enabled')) { + try { + $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis()); + } catch (Exception $e) { + // redis connection failed + $this->logException("Failed to get Resque workers status.", $e); + return []; + } + return $this->ResqueStatus->getWorkers(); + } + + $worker_array = []; + $workers = $this->getBackgroundJobsTool()->getWorkers(); + + foreach ($workers as $worker) { + $worker_array[$worker->pid()] = [ + 'queue' => $worker->queue(), + 'type' => 'regular', // for compatibility with Resque response + 'user' => $worker->user() + ]; + } + + return $worker_array; + } } diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 66540162a..5f55a05cb 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -5,6 +5,7 @@ App::uses('File', 'Utility'); App::uses('AttachmentTool', 'Tools'); App::uses('ComplexTypeTool', 'Tools'); App::uses('ServerSyncTool', 'Tools'); +App::uses('AttributeValidationTool', 'Tools'); /** * @property Event $Event @@ -129,13 +130,13 @@ class ShadowAttribute extends AppModel 'first_seen' => array( 'rule' => array('datetimeOrNull'), 'required' => false, - 'message' => array('Invalid ISO 8601 format') + 'message' => array('Invalid ISO 8601 format'), ), 'last_seen' => array( 'datetimeOrNull' => array( 'rule' => array('datetimeOrNull'), 'required' => false, - 'message' => array('Invalid ISO 8601 format') + 'message' => array('Invalid ISO 8601 format'), ), 'validateLastSeenValue' => array( 'rule' => array('validateLastSeenValue'), @@ -173,7 +174,7 @@ class ShadowAttribute extends AppModel $compositeTypes = $this->getCompositeTypes(); // explode composite types in value1 and value2 $pieces = explode('|', $this->data['ShadowAttribute']['value']); - if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes)) { + if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes, true)) { if (2 != count($pieces)) { throw new InternalErrorException('Composite type, but value not explodable'); } @@ -300,58 +301,67 @@ class ShadowAttribute extends AppModel public function beforeValidate($options = array()) { - parent::beforeValidate(); - // remove leading and trailing blanks - //$this->trimStringFields(); // TODO - - if (!isset($this->data['ShadowAttribute']['comment'])) { - $this->data['ShadowAttribute']['comment'] = ''; - } - - if (!isset($this->data['ShadowAttribute']['type'])) { + $proposal = &$this->data['ShadowAttribute']; + if (!isset($proposal['type'])) { + $this->invalidate('type', 'No value provided.'); return false; } + if (!isset($proposal['comment'])) { + $proposal['comment'] = ''; + } + // make some changes to the inserted value - if (isset($this->data['ShadowAttribute']['value'])) { - $value = trim($this->data['ShadowAttribute']['value']); - $value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']); - $value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value); - $this->data['ShadowAttribute']['value'] = $value; + if (isset($proposal['value'])) { + $value = trim($proposal['value']); + $value = ComplexTypeTool::refangValue($value, $proposal['type']); + $value = AttributeValidationTool::modifyBeforeValidation($proposal['type'], $value); + $proposal['value'] = $value; } - if (!isset($this->data['ShadowAttribute']['org'])) { - $this->data['ShadowAttribute']['org'] = ''; + if (!isset($proposal['org'])) { + $proposal['org'] = ''; } - if (empty($this->data['ShadowAttribute']['timestamp'])) { - $date = new DateTime(); - $this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp(); + if (empty($proposal['timestamp'])) { + $proposal['timestamp'] = time(); } - if (!isset($this->data['ShadowAttribute']['proposal_to_delete'])) { - $this->data['ShadowAttribute']['proposal_to_delete'] = 0; + if (!isset($proposal['proposal_to_delete'])) { + $proposal['proposal_to_delete'] = 0; } // generate UUID if it doesn't exist - if (empty($this->data['ShadowAttribute']['uuid'])) { - $this->data['ShadowAttribute']['uuid'] = CakeText::uuid(); + if (empty($proposal['uuid'])) { + $proposal['uuid'] = CakeText::uuid(); } else { - $this->data['ShadowAttribute']['uuid'] = strtolower($this->data['ShadowAttribute']['uuid']); + $proposal['uuid'] = strtolower($proposal['uuid']); } - if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) { - $this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category']; + if (empty($proposal['category'])) { + $proposal['category'] = $this->Attribute->typeDefinitions[$proposal['type']]['default_category']; + } + + if (isset($proposal['first_seen'])) { + $proposal['first_seen'] = $proposal['first_seen'] === '' ? null : $proposal['first_seen']; + } + if (isset($proposal['last_seen'])) { + $proposal['last_seen'] = $proposal['last_seen'] === '' ? null : $proposal['last_seen']; } - // always return true, otherwise the object cannot be saved return true; } public function afterFind($results, $primary = false) { - foreach ($results as $k => $v) { - $results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias); + foreach ($results as &$v) { + $proposal = &$v['ShadowAttribute']; + if (!empty($proposal['first_seen'])) { + $proposal['first_seen'] = $this->microTimestampToIso($proposal['first_seen']); + } + if (!empty($proposal['last_seen'])) { + $proposal['last_seen'] = $this->microTimestampToIso($proposal['last_seen']); + } } return $results; } @@ -373,7 +383,7 @@ class ShadowAttribute extends AppModel public function validateAttributeValue($fields) { $value = $fields['value']; - return $this->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']); + return AttributeValidationTool::validate($this->data['ShadowAttribute']['type'], $value); } public function getCompositeTypes() @@ -803,26 +813,28 @@ class ShadowAttribute extends AppModel )); } } else { + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generate proposal correlation', - 'job_input' => 'All attributes', - 'retries' => 0, - 'status' => 1, - 'org' => 'SYSTEM', - 'message' => 'Correlating Proposals.', + $jobId = $job->createJob( + 'SYSTEM', + Job::WORKER_DEFAULT, + 'generate proposal correlation', + 'All attributes', + 'Correlating Proposals.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'default', - 'AdminShell', - array('jobGenerateShadowAttributeCorrelation', $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'jobGenerateShadowAttributeCorrelation', + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + $this->Log->create(); $this->Log->save(array( 'org' => 'SYSTEM', diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index aa4db6076..171f48020 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -5,6 +5,9 @@ App::uses('AppModel', 'Model'); * @property SharingGroupOrg $SharingGroupOrg * @property SharingGroupServer $SharingGroupServer * @property Organisation $Organisation + * @property Event $Event + * @property Attribute $Attribute + * @property Thread $Thread */ class SharingGroup extends AppModel { @@ -101,6 +104,12 @@ class SharingGroup extends AppModel if ($this->Attribute->hasAny(['sharing_group_id' => $this->id])) { return false; } + if ($this->Attribute->Object->hasAny(['sharing_group_id' => $this->id])) { + return false; + } + if ($this->Event->EventReport->hasAny(['sharing_group_id' => $this->id])) { + return false; + } return true; } diff --git a/app/Model/SharingGroupOrg.php b/app/Model/SharingGroupOrg.php index bb381dbf7..719cd5c67 100644 --- a/app/Model/SharingGroupOrg.php +++ b/app/Model/SharingGroupOrg.php @@ -1,29 +1,41 @@ array( - 'className' => 'SharingGroup', - 'foreignKey' => 'sharing_group_id' - ), - 'Organisation' => array( - 'className' => 'Organisation', - 'foreignKey' => 'org_id', - //'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid') - ) + 'SharingGroup' => array( + 'className' => 'SharingGroup', + 'foreignKey' => 'sharing_group_id' + ), + 'Organisation' => array( + 'className' => 'Organisation', + 'foreignKey' => 'org_id', + //'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid') + ) ); public function beforeValidate($options = array()) { - parent::beforeValidate(); + $data = $this->data[$this->alias]; + $conditions = [ + 'sharing_group_id' => $data['sharing_group_id'], + 'org_id' => $data['org_id'], + ]; + if (isset($data['id'])) { + $conditions['id !='] = $data['id']; + } + if ($this->hasAny($conditions)) { + $this->log("Trying to save duplicate organisation `{$data['org_id']}` for sharing group `{$data['sharing_group_id']}. This should never happened."); + $this->invalidate('org_id', 'The same organisation is already assigned to this sharing group.'); + return false; + } } public function updateOrgsForSG($id, $new_orgs, $old_orgs, $user) { - $log = ClassRegistry::init('Log'); // Loop through all of the organisations we want to add. foreach ($new_orgs as $org) { $SgO = array( @@ -54,16 +66,16 @@ class SharingGroupOrg extends AppModel } if ($this->save($SgO)) { if ($isChange) { - $log->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.')); + $this->loadLog()->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.')); } else { - $log->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : '')); + $this->loadLog()->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : '')); } } } // We are left with some "old orgs" that are not in the new list. This means that they can be safely deleted. foreach ($old_orgs as $old_org) { if ($this->delete($old_org['id'])) { - $log->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.'); + $this->loadLog()->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.'); } } } @@ -83,7 +95,12 @@ class SharingGroupOrg extends AppModel return $sgs; } - // pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object + /** + * Pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object + * @param int $id + * @param int $org_id + * @return bool + */ public function checkIfAuthorised($id, $org_id) { return $this->hasAny([ diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index 2947588f9..f9b433c56 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -49,7 +49,7 @@ class Sighting extends AppModel ), ); - public $type = array( + const TYPE = array( 0 => 'sighting', 1 => 'false-positive', 2 => 'expiration' @@ -63,7 +63,6 @@ class Sighting extends AppModel public function beforeValidate($options = array()) { - parent::beforeValidate(); if (empty($this->data['Sighting']['id']) && empty($this->data['Sighting']['date_sighting'])) { $this->data['Sighting']['date_sighting'] = date('Y-m-d H:i:s'); } @@ -77,7 +76,7 @@ class Sighting extends AppModel public function afterSave($created, $options = array()) { - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable'); + $pubToZmq = $this->pubToZmq('sighting'); $kafkaTopic = $this->kafkaTopic('sighting'); if ($pubToZmq || $kafkaTopic) { $user = array( @@ -101,8 +100,7 @@ class Sighting extends AppModel public function beforeDelete($cascade = true) { - parent::beforeDelete(); - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable'); + $pubToZmq = $this->pubToZmq('sighting'); $kafkaTopic = $this->kafkaTopic('sighting'); if ($pubToZmq || $kafkaTopic) { $user = array( @@ -441,7 +439,7 @@ class Sighting extends AppModel $sparklineData = []; $range = $this->getMaximumRange(); foreach ($groupedSightings as $sighting) { - $type = $this->type[$sighting['type']]; + $type = self::TYPE[$sighting['type']]; $orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : __('Others'); $count = (int)$sighting['sighting_count']; $inRange = strtotime($sighting['date']) >= $range; diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php new file mode 100644 index 000000000..bf98c49f0 --- /dev/null +++ b/app/Model/SystemSetting.php @@ -0,0 +1,170 @@ +databaseExists()) { + return; + } + $settings = $systemSetting->getSettings(); + foreach ($settings as $settingName => $settingValue) { + $firstPart = explode('.', $settingName)[0]; + if (in_array($firstPart, self::ALLOWED_CATEGORIES, true) && !in_array($settingName, self::BLOCKED_SETTINGS, true)) { + Configure::write($settingName, $settingValue); + } + } + } + + public function databaseExists() + { + $tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources(); + return in_array('system_settings', $tables, true); + } + + /** + * @return array + * @throws JsonException + */ + public function getSettings() + { + $settings = $this->find('list', [ + 'fields' => ['SystemSetting.setting', 'SystemSetting.value'], + ]); + return array_map(function ($value) { + if (EncryptedValue::isEncrypted($value)) { + return new EncryptedValue($value, true); + } else { + return JsonTool::decode($value); + } + }, $settings); + } + + /** + * @param string $setting Setting name + * @param mixed $value + * @throws Exception + */ + public function setSetting($setting, $value) + { + $firstPart = explode('.', $setting)[0]; + if (!in_array($firstPart, self::ALLOWED_CATEGORIES, true) || in_array($setting, self::BLOCKED_SETTINGS, true)) { + return false; // blocked setting + } + + if ($value === '' || $value === null) { + if ($this->hasAny(['SystemSetting.setting' => $setting])) { + return $this->delete($setting); // delete the whole setting when value is empty + } + return true; + } + + $value = JsonTool::encode($value); + + // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it + if (self::isSensitive($setting)) { + $value = EncryptedValue::encryptIfEnabled($value); + } + + $valid = $this->save(['SystemSetting' => [ + 'setting' => $setting, + 'value' => $value, + ]]); + if (!$valid) { + throw new Exception("Could not save system setting `$setting` because of validation errors: " . JsonTool::encode($this->validationErrors)); + } + return true; + } + + /** + * @param string|null $old Old (or current) encryption key. + * @param string|null $new New encryption key. If empty, encrypted values will be decrypted. + * @throws Exception + */ + public function reencrypt($old, $new) + { + $settings = $this->find('list', [ + 'fields' => ['SystemSetting.setting', 'SystemSetting.value'], + ]); + $toSave = []; + foreach ($settings as $setting => $value) { + if (!self::isSensitive($setting)) { + continue; + } + if (EncryptedValue::isEncrypted($value)) { + try { + $value = BetterSecurity::decrypt(substr($value, 2), $old); + } catch (Exception $e) { + throw new Exception("Could not decrypt `$setting` setting.", 0, $e); + } + } + if (!empty($new)) { + $value = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $new); + } + $toSave[] = ['SystemSetting' => [ + 'setting' => $setting, + 'value' => $value, + ]]; + } + if (empty($toSave)) { + return true; + } + return $this->saveMany($toSave); + } + + /** + * Sensitive setting are passwords or api keys. + * @param string $setting Setting name + * @return bool + */ + public static function isSensitive($setting) + { + if ($setting === 'Security.encryption_key' || $setting === 'Security.salt') { + return true; + } + if (substr($setting, 0, 7) === 'Plugin.' && (strpos($setting, 'apikey') !== false || strpos($setting, 'secret') !== false)) { + return true; + } + return strpos($setting, 'password') !== false; + } +} diff --git a/app/Model/Tag.php b/app/Model/Tag.php index f6b2bc0ea..964fba875 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -4,6 +4,8 @@ App::uses('AppModel', 'Model'); /** * @property EventTag $EventTag * @property AttributeTag $AttributeTag + * @property FavouriteTag $FavouriteTag + * @property Organisation $Organisation */ class Tag extends AppModel { @@ -22,28 +24,28 @@ class Tag extends AppModel ); public $validate = array( - 'name' => array( - 'required' => array( - 'rule' => array('notBlank', 'name'), - 'message' => 'This field is required.' - ), - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty', 'name'), - ), - 'unique' => array( - 'rule' => 'isUnique', - 'message' => 'A similar name already exists.', - ), + 'name' => array( + 'required' => array( + 'rule' => array('notBlank', 'name'), + 'message' => 'This field is required.' ), - 'colour' => array( - 'valueNotEmpty' => array( - 'rule' => array('valueNotEmpty', 'colour'), - ), - 'userdefined' => array( - 'rule' => 'validateColour', - 'message' => 'Colour has to be in the RGB format (#FFFFFF)', - ), + 'valueNotEmpty' => array( + 'rule' => array('valueNotEmpty', 'name'), ), + 'unique' => array( + 'rule' => 'isUnique', + 'message' => 'A similar name already exists.', + ), + ), + 'colour' => array( + 'valueNotEmpty' => array( + 'rule' => array('valueNotEmpty', 'colour'), + ), + 'userdefined' => array( + 'rule' => 'validateColour', + 'message' => 'Colour has to be in the RGB format (#FFFFFF)', + ), + ), ); public $hasMany = array( @@ -83,24 +85,27 @@ class Tag extends AppModel public function beforeValidate($options = array()) { - parent::beforeValidate(); - if (!isset($this->data['Tag']['org_id'])) { - $this->data['Tag']['org_id'] = 0; + $tag = &$this->data['Tag']; + if (!isset($tag['org_id'])) { + $tag['org_id'] = 0; } - if (!isset($this->data['Tag']['user_id'])) { - $this->data['Tag']['user_id'] = 0; + if (!isset($tag['user_id'])) { + $tag['user_id'] = 0; } - if (!isset($this->data['Tag']['hide_tag'])) { - $this->data['Tag']['hide_tag'] = Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0; + if (!isset($tag['hide_tag'])) { + $tag['hide_tag'] = Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0; } - if (!isset($this->data['Tag']['exportable'])) { - $this->data['Tag']['exportable'] = 1; + if (!isset($tag['exportable'])) { + $tag['exportable'] = 1; } - if (isset($this->data['Tag']['name']) && strlen($this->data['Tag']['name']) >= 255) { - $this->data['Tag']['name'] = substr($this->data['Tag']['name'], 0, 255); + if (!isset($tag['local_only'])) { + $tag['local_only'] = 0; } - $this->data['Tag']['is_galaxy'] = preg_match($this->reGalaxy, $this->data['Tag']['name']); - $this->data['Tag']['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $this->data['Tag']['name']); + if (isset($tag['name']) && strlen($tag['name']) >= 255) { + $tag['name'] = substr($tag['name'], 0, 255); + } + $tag['is_galaxy'] = preg_match($this->reGalaxy, $tag['name']); + $tag['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $tag['name']); return true; } @@ -149,8 +154,7 @@ class Tag extends AppModel public function afterFind($results, $primary = false) { - $results = $this->checkForOverride($results); - return $results; + return $this->checkForOverride($results); } public function validateColour($fields) @@ -161,12 +165,41 @@ class Tag extends AppModel return true; } + /** + * @param array $user + * @param string $tagName + * @return mixed|null + */ + public function lookupTagIdForUser(array $user, $tagName) + { + $conditions = ['LOWER(Tag.name)' => mb_strtolower($tagName)]; + if (!$user['Role']['perm_site_admin']) { + $conditions['Tag.org_id'] = [0, $user['org_id']]; + $conditions['Tag.user_id'] = [0, $user['id']]; + } + $tagId = $this->find('first', array( + 'conditions' => $conditions, + 'recursive' => -1, + 'fields' => array('Tag.id'), + 'callbacks' => false, + )); + if (empty($tagId)) { + return null; + } + return $tagId['Tag']['id']; + } + + /** + * @param string $tagName + * @return int|mixed + */ public function lookupTagIdFromName($tagName) { $tagId = $this->find('first', array( - 'conditions' => array('LOWER(Tag.name)' => strtolower($tagName)), + 'conditions' => array('LOWER(Tag.name)' => mb_strtolower($tagName)), 'recursive' => -1, - 'fields' => array('Tag.id') + 'fields' => array('Tag.id'), + 'callbacks' => false, )); if (empty($tagId)) { return -1; @@ -186,44 +219,14 @@ class Tag extends AppModel return $this->find('all', array('conditions' => $conditions, 'recursive' => -1)); } - // find all of the tag ids that belong to the accepted tag names and the rejected tag names - public function fetchTagIdsFromFilter($accept = array(), $reject = array()) + /** + * @param array $accept + * @param array $reject + * @deprecated Use EventTag::fetchEventTagIds instead + */ + public function fetchEventTagIds($accept, $reject) { - $results = array(0 => array(), 1 => array()); - if (!empty($accept)) { - foreach ($accept as $tag) { - $temp = $this->lookupTagIdFromName($tag); - if (!in_array($temp, $results[0])) { - $results[0][] = $temp; - } - } - } - if (!empty($reject)) { - foreach ($reject as $tag) { - $temp = $this->lookupTagIdFromName($tag); - if (!in_array($temp, $results[1])) { - $results[1][] = $temp; - } - } - } - return $results; - } - - // find all of the event Ids that belong to the accepted tags and the rejected tags - public function fetchEventTagIds($accept = array(), $reject = array()) - { - $acceptIds = array(); - $rejectIds = array(); - if (!empty($accept)) { - $acceptIds = $this->findEventIdsByTagNames($accept); - if (empty($acceptIds)) { - $acceptIds[] = -1; - } - } - if (!empty($reject)) { - $rejectIds = $this->findEventIdsByTagNames($reject); - } - return array($acceptIds, $rejectIds); + $this->EventTag->fetchEventTagIds($accept, $reject); } // find all of the tag Ids that belong to the accepted tags and the rejected tags @@ -256,78 +259,38 @@ class Tag extends AppModel return array($acceptIds, $rejectIds); } - // pass a list of tag names to receive a list of matched tag IDs + /** + * pass a list of tag names to receive a list of matched tag IDs + * @param string|array $array + * @return array|int|null + */ public function findTagIdsByTagNames($array) { - $ids = array(); - $tag_ids = array(); if (!is_array($array)) { $array = array($array); } - foreach ($array as $k => $tag) { + $tagIds = []; + $tagNames = []; + foreach ($array as $tag) { if (is_numeric($tag)) { - $tag_ids[] = $tag; - unset($array[$k]); - } - } - $array = array_values($array); - if (!empty($array)) { - foreach ($array as $a) { - $conditions['OR'][] = array('Tag.name like' => $a); - } - $params = array( - 'recursive' => 1, - 'conditions' => $conditions, - 'fields' => array('Tag.id', 'Tag.id') - ); - $result = $this->find('list', $params); - $tag_ids = array_merge($result, $tag_ids); - } - return array_values($tag_ids); - } - - public function findEventIdsByTagNames($array) - { - $ids = array(); - foreach ($array as $a) { - if (is_numeric($a)) { - $conditions['OR'][] = array('id' => $a); + $tagIds[] = $tag; } else { - $conditions['OR'][] = array('LOWER(name) like' => strtolower($a)); + $tagNames[] = $tag; } } - $params = array( + if (!empty($tagNames)) { + $conditions = []; + foreach ($tagNames as $tagName) { + $conditions[] = array('Tag.name LIKE' => $tagName); + } + $result = $this->find('column', array( 'recursive' => 1, - 'contain' => 'EventTag', - 'conditions' => $conditions - ); - $result = $this->find('all', $params); - foreach ($result as $tag) { - foreach ($tag['EventTag'] as $eventTag) { - $ids[] = $eventTag['event_id']; - } + 'conditions' => ['OR' => $conditions], + 'fields' => array('Tag.id') + )); + $tagIds = array_merge($result, $tagIds); } - return $ids; - } - - public function findAttributeIdsByAttributeTagNames($array) - { - $ids = array(); - foreach ($array as $a) { - $conditions['OR'][] = array('LOWER(name) LIKE' => strtolower($a)); - } - $params = array( - 'recursive' => 1, - 'contain' => 'AttributeTag', - 'conditions' => $conditions - ); - $result = $this->find('all', $params); - foreach ($result as $tag) { - foreach ($tag['AttributeTag'] as $attributeTag) { - $ids[] = $attributeTag['attribute_id']; - } - } - return $ids; + return $tagIds; } /** @@ -337,23 +300,25 @@ class Tag extends AppModel * @return false|int * @throws Exception */ - public function captureTag($tag, $user, $force=false) + public function captureTag(array $tag, array $user, $force=false) { $existingTag = $this->find('first', array( 'recursive' => -1, 'conditions' => array('LOWER(name)' => mb_strtolower($tag['name'])), 'fields' => ['id', 'org_id', 'user_id'], + 'callbacks' => false, )); if (empty($existingTag)) { if ($force || $user['Role']['perm_tag_editor']) { $this->create(); - if (!isset($tag['colour']) || empty($tag['colour'])) { + if (empty($tag['colour'])) { $tag['colour'] = $this->random_color(); } $tag = array( 'name' => $tag['name'], 'colour' => $tag['colour'], 'exportable' => isset($tag['exportable']) ? $tag['exportable'] : 1, + 'local_only' => $tag['local_only'] ?? 0, 'org_id' => 0, 'user_id' => 0, 'hide_tag' => Configure::read('MISP.incoming_tags_disabled_by_default') ? 1 : 0 @@ -363,22 +328,21 @@ class Tag extends AppModel } else { return false; } - } else { - if ( - !$user['Role']['perm_site_admin'] && + } + if ( + !$user['Role']['perm_site_admin'] && + ( ( - ( - $existingTag['Tag']['org_id'] != 0 && - $existingTag['Tag']['org_id'] != $user['org_id'] - ) || - ( - $existingTag['Tag']['user_id'] != 0 && - $existingTag['Tag']['user_id'] != $user['id'] - ) + $existingTag['Tag']['org_id'] != 0 && + $existingTag['Tag']['org_id'] != $user['org_id'] + ) || + ( + $existingTag['Tag']['user_id'] != 0 && + $existingTag['Tag']['user_id'] != $user['id'] ) - ) { - return false; - } + ) + ) { + return false; } return $existingTag['Tag']['id']; } @@ -392,6 +356,13 @@ class Tag extends AppModel return $colour; } + /** + * @param string $name + * @param string|false $colour + * @param null $numerical_value + * @return int|false Created tag ID or false on error + * @throws Exception + */ public function quickAdd($name, $colour = false, $numerical_value = null) { $this->create(); @@ -401,19 +372,26 @@ class Tag extends AppModel $data = array( 'name' => $name, 'colour' => $colour, - 'exportable' => 1 + 'exportable' => 1, ); - if (!is_null($numerical_value)) { + if ($numerical_value !== null) { $data['numerical_value'] = $numerical_value; } - return ($this->save($data)); + if ($this->save(['Tag' => $data])) { + return $this->id; + } else { + return false; + } } - public function quickEdit($tag, $name, $colour, $hide = false, $numerical_value = null) + public function quickEdit($tag, $name, $colour, $hide = false, $numerical_value = null, $local_only = -1) { - if ($tag['Tag']['colour'] !== $colour || $tag['Tag']['name'] !== $name || $hide !== false || $tag['Tag']['numerical_value'] !== $numerical_value) { + if ($tag['Tag']['colour'] !== $colour || $tag['Tag']['name'] !== $name || $hide !== false || $tag['Tag']['numerical_value'] !== $numerical_value || ($tag['Tag']['local_only'] !== $local_only && $local_only !== -1)) { $tag['Tag']['name'] = $name; $tag['Tag']['colour'] = $colour; + if ($tag['Tag']['local_only'] !== -1) { + $tag['Tag']['local_only'] = $local_only; + } if ($hide !== false) { $tag['Tag']['hide_tag'] = $hide; } @@ -434,15 +412,21 @@ class Tag extends AppModel } /** - * Recover user_id from the session and override numerical_values from userSetting - */ - public function checkForOverride($tags) + * Recover user_id from the session and override numerical_values from userSetting. + * + * @param array $tags + * @return array + */ + private function checkForOverride($tags) { $userId = Configure::read('CurrentUserId'); if ($this->tagOverrides === false && $userId > 0) { $this->UserSetting = ClassRegistry::init('UserSetting'); $this->tagOverrides = $this->UserSetting->getTagNumericalValueOverride($userId); } + if (empty($this->tagOverrides)) { + return $tags; + } foreach ($tags as $k => $tag) { if (isset($tag['Tag']['name'])) { $tagName = $tag['Tag']['name']; diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 8ebf18935..1f0c479f6 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -242,7 +242,7 @@ class Taxonomy extends AppModel // returns all tags associated to a taxonomy // returns all tags not associated to a taxonomy if $inverse is true - public function getAllTaxonomyTags($inverse = false, $user = false, $full = false, $hideUnselectable = true) + public function getAllTaxonomyTags($inverse = false, $user = false, $full = false, $hideUnselectable = true, $local_tag = false) { $this->Tag = ClassRegistry::init('Tag'); $taxonomyIdList = $this->find('column', array('fields' => array('Taxonomy.id'))); @@ -260,6 +260,10 @@ class Taxonomy extends AppModel if (Configure::read('MISP.incoming_tags_disabled_by_default') || $hideUnselectable) { $conditions['Tag.hide_tag'] = 0; } + // If the tag is to be added as global, we filter out the local_only tags + if (!$local_tag) { + $conditions['Tag.local_only'] = 0; + } if ($full) { $allTags = $this->Tag->find( 'all', @@ -666,11 +670,15 @@ class Taxonomy extends AppModel return true; } + /** + * @param array $tagList + * @return array[] + */ public function checkIfTagInconsistencies($tagList) { $eventTags = array(); $localEventTags = array(); - foreach($tagList as $tag) { + foreach ($tagList as $tag) { if ($tag['local'] == 0) { $eventTags[] = $tag['Tag']['name']; } else { diff --git a/app/Model/Template.php b/app/Model/Template.php index 4bc516960..44c92c70d 100644 --- a/app/Model/Template.php +++ b/app/Model/Template.php @@ -72,9 +72,4 @@ class Template extends AppModel return false; } } - - public function generateRandomFileName() - { - return (new RandomTool())->random_str(false, 12); - } } diff --git a/app/Model/User.php b/app/Model/User.php index 81f9c130f..4e20ca9f0 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -231,9 +231,9 @@ class User extends AppModel public function __construct($id = false, $table = null, $ds = null) { parent::__construct($id, $table, $ds); - $this->AdminSetting = ClassRegistry::init('AdminSetting'); - $db_version = $this->AdminSetting->getSetting('db_version'); - if ($db_version >= 62) { + + // bind AuthKey just when authkey table already exists. This is important for updating from old versions + if (in_array('auth_keys', $this->getDataSource()->listSources(), true)) { $this->bindModel([ 'hasMany' => ['AuthKey'] ], false); @@ -1067,27 +1067,29 @@ class User extends AppModel public function resetAllSyncAuthKeysRouter($user, $jobId = false) { if (Configure::read('MISP.background_jobs')) { + + /** @var Job $job */ $job = ClassRegistry::init('Job'); - $job->create(); - $data = array( - 'worker' => 'prio', - 'job_type' => __('reset_all_sync_api_keys'), - 'job_input' => __('Reseting all API keys'), - 'status' => 0, - 'retries' => 0, - 'org_id' => $user['org_id'], - 'org' => $user['Organisation']['name'], - 'message' => 'Issuing new API keys to all sync users.', + $jobId = $job->createJob( + $user, + Job::WORKER_PRIO, + 'reset_all_sync_api_keys', + __('Reseting all API keys'), + 'Issuing new API keys to all sync users.' ); - $job->save($data); - $jobId = $job->id; - $process_id = CakeResque::enqueue( - 'prio', - 'AdminShell', - array('resetSyncAuthkeys', $user['id'], $jobId), - true + + $this->getBackgroundJobsTool()->enqueue( + BackgroundJobsTool::PRIO_QUEUE, + BackgroundJobsTool::CMD_ADMIN, + [ + 'resetSyncAuthkeys', + $user['id'], + $jobId + ], + true, + $jobId ); - $job->saveField('process_id', $process_id); + return true; } else { return $this->resetAllSyncAuthKeys($user); @@ -1504,4 +1506,68 @@ class User extends AppModel $banStatus['message'] = __('User email notification ban setting is not enabled'); return $banStatus; } + + /** + * @param array $user + * @return bool + */ + public function hasNotifications(array $user) + { + $hasProposal = $this->Event->ShadowAttribute->hasAny([ + 'ShadowAttribute.event_org_id' => $user['org_id'], + 'ShadowAttribute.deleted' => 0, + ]); + if ($hasProposal) { + return true; + } + + if (Configure::read('MISP.delegation') && $this->_getDelegationCount($user)) { + return true; + } + return false; + } + + /** + * @param array $user + * @return array + */ + public function populateNotifications(array $user) + { + $notifications = array(); + list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user); + $notifications['total'] = $notifications['proposalCount']; + if (Configure::read('MISP.delegation')) { + $notifications['delegationCount'] = $this->_getDelegationCount($user); + $notifications['total'] += $notifications['delegationCount']; + } + return $notifications; + } + + // if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu. + private function _getProposalCount($user, $mode = 'full') + { + $results[0] = $this->Event->ShadowAttribute->find('count', [ + 'conditions' => array( + 'ShadowAttribute.event_org_id' => $user['org_id'], + 'ShadowAttribute.deleted' => 0, + ) + ]); + $results[1] = $this->Event->ShadowAttribute->find('count', [ + 'conditions' => array( + 'ShadowAttribute.event_org_id' => $user['org_id'], + 'ShadowAttribute.deleted' => 0, + ), + 'fields' => 'distinct event_id' + ]); + return $results; + } + + private function _getDelegationCount($user) + { + $this->EventDelegation = ClassRegistry::init('EventDelegation'); + return $this->EventDelegation->find('count', array( + 'recursive' => -1, + 'conditions' => array('EventDelegation.org_id' => $user['org_id']) + )); + } } diff --git a/app/Model/UserSetting.php b/app/Model/UserSetting.php index 2ab3bcfd8..131f37942 100644 --- a/app/Model/UserSetting.php +++ b/app/Model/UserSetting.php @@ -1,5 +1,9 @@ array( - 'isValidJson' => array( - 'rule' => array('isValidJson'), - ) - ) + 'value' => 'valueIsJson', ); public $belongsTo = array( 'User' ); - public $validSettings = array( + // private + const VALID_SETTINGS = array( 'publish_alert_filter' => array( 'placeholder' => array( 'AND' => array( @@ -96,12 +97,14 @@ class UserSetting extends AppModel 'event_index_hide_columns' => [ 'placeholder' => ['clusters'], ], + 'oidc' => [ // Data saved by OIDC plugin + 'restricted' => 'perm_site_admin', + ], ); // massage the data before we send it off for validation before saving anything public function beforeValidate($options = array()) { - parent::beforeValidate(); // add a timestamp if it is not set if (empty($this->data['UserSetting']['timestamp'])) { $this->data['UserSetting']['timestamp'] = time(); @@ -124,35 +127,39 @@ class UserSetting extends AppModel public function afterFind($results, $primary = false) { foreach ($results as $k => $v) { - $results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true); + if (isset($v['UserSetting']['value'])) { + $results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true); + } } return $results; } public function checkSettingValidity($setting) { - return isset($this->validSettings[$setting]); + return isset(self::VALID_SETTINGS[$setting]); } - public function checkSettingAccess($user, $setting) + /** + * @param array $user + * @param string $setting + * @return bool|string + */ + public function checkSettingAccess(array $user, $setting) { - if (!empty($this->validSettings[$setting]['restricted'])) { - $role_check = $this->validSettings[$setting]['restricted']; - if (!is_array($role_check)) { - $role_check = array($role_check); + if (!empty(self::VALID_SETTINGS[$setting]['restricted'])) { + $roleCheck = self::VALID_SETTINGS[$setting]['restricted']; + if (!is_array($roleCheck)) { + $roleCheck = array($roleCheck); } - $userHasValidRole = false; - foreach ($role_check as $role) { + foreach ($roleCheck as $role) { if (!empty($user['Role'][$role])) { return true; } } - if (!$userHasValidRole) { - foreach ($role_check as &$role) { - $role = substr($role, 5); - } - return implode(', ', $role_check); + foreach ($roleCheck as &$role) { + $role = substr($role, 5); } + return implode(', ', $roleCheck); } return true; } @@ -203,7 +210,7 @@ class UserSetting extends AppModel return false; } - public function getDefaulRestSearchParameters($user) + public function getDefaultRestSearchParameters($user) { return $this->getValueForUser($user['id'], 'default_restsearch_parameters') ?: []; } @@ -236,8 +243,8 @@ class UserSetting extends AppModel /** * Check whether the event is something the user is interested (to be alerted on) - * @param $user - * @param $event + * @param array $user + * @param array $event * @return bool */ public function checkPublishFilter(array $user, array $event) @@ -295,7 +302,7 @@ class UserSetting extends AppModel } } - /* + /** * Checks if an event matches the given rule * valid filters: * - AttributeTag.name @@ -307,6 +314,11 @@ class UserSetting extends AppModel * Values passed can be used for direct string comparisons or alternatively * as substring matches by encapsulating the string in a pair of "%" characters * Each rule can take a list of values + * + * @param string $rule + * @param array|string $lookup_values + * @param array $event + * @return bool */ private function __checkEvent($rule, $lookup_values, $event) { @@ -341,7 +353,7 @@ class UserSetting extends AppModel $extracted_value = mb_strtolower($extracted_value); foreach ($lookup_values as $lookup_value) { $lookup_value_trimmed = trim($lookup_value, "%"); - if (strlen($lookup_value_trimmed) != strlen($lookup_value)) { + if (strlen($lookup_value_trimmed) !== strlen($lookup_value)) { if (strpos($extracted_value, $lookup_value_trimmed) !== false) { return true; } @@ -356,7 +368,13 @@ class UserSetting extends AppModel return false; } - public function setSetting($user, &$data) + /** + * @param array $user + * @param array $data + * @return bool + * @throws Exception + */ + public function setSetting(array $user, array $data) { $userSetting = array(); if (!empty($data['UserSetting']['user_id']) && is_numeric($data['UserSetting']['user_id'])) { @@ -391,21 +409,42 @@ class UserSetting extends AppModel } else { $userSetting['value'] = ''; } + + return $this->setSettingInternal($userSetting['user_id'], $userSetting['setting'], $userSetting['value']); + } + + /** + * Set user setting without checking permission. + * @param int $userId + * @param string $setting + * @param mixed $value + * @return array|bool|mixed|null + * @throws Exception + */ + public function setSettingInternal($userId, $setting, $value) + { + $userSetting = [ + 'user_id' => $userId, + 'setting' => $setting, + 'value' => $value, + ]; + $existingSetting = $this->find('first', array( 'recursive' => -1, 'conditions' => array( - 'UserSetting.user_id' => $userSetting['user_id'], - 'UserSetting.setting' => $userSetting['setting'] - ) + 'UserSetting.user_id' => $userId, + 'UserSetting.setting' => $setting, + ), + 'fields' => ['UserSetting.id'], + 'callbacks' => false, )); if (empty($existingSetting)) { $this->create(); } else { $userSetting['id'] = $existingSetting['UserSetting']['id']; } - // save the setting - $result = $this->save(array('UserSetting' => $userSetting)); - return true; + + return $this->save($userSetting); } /** diff --git a/app/Plugin/ApcuCache/Engine/ApcuEngine.php b/app/Plugin/ApcuCache/Engine/ApcuEngine.php new file mode 100644 index 000000000..6d400e6b6 --- /dev/null +++ b/app/Plugin/ApcuCache/Engine/ApcuEngine.php @@ -0,0 +1,167 @@ + 'Apc'); + parent::init($settings); + return function_exists('apcu_dec'); + } + + /** + * Write data for key into cache + * + * @param string $key Identifier for the data + * @param mixed $value Data to be cached + * @param int $duration How long to cache the data, in seconds + * @return bool True if the data was successfully cached, false on failure + */ + public function write($key, $value, $duration) { + return apcu_store($key, $value, $duration); + } + + /** + * Read a key from the cache + * + * @param string $key Identifier for the data + * @return mixed The cached data, or false if the data doesn't exist, has expired, or if there was an error fetching it + */ + public function read($key) { + return apcu_fetch($key); + } + + /** + * Increments the value of an integer cached key + * + * @param string $key Identifier for the data + * @param int $offset How much to increment + * @return false|int New incremented value, false otherwise + */ + public function increment($key, $offset = 1) { + return apcu_inc($key, $offset); + } + + /** + * Decrements the value of an integer cached key + * + * @param string $key Identifier for the data + * @param int $offset How much to subtract + * @return false|int New decremented value, false otherwise + */ + public function decrement($key, $offset = 1) { + return apcu_dec($key, $offset); + } + + /** + * Delete a key from the cache + * + * @param string $key Identifier for the data + * @return bool True if the value was successfully deleted, false if it didn't exist or couldn't be removed + */ + public function delete($key) { + return apcu_delete($key); + } + + /** + * Delete all keys from the cache. This will clear every cache config using APC. + * + * @param bool $check If true, nothing will be cleared, as entries are removed + * from APC as they expired. This flag is really only used by FileEngine. + * @return bool True Returns true. + */ + public function clear($check) { + if ($check) { + return true; + } + $iterator = new APCUIterator( + '/^' . preg_quote($this->settings['prefix'], '/') . '/', + APC_ITER_NONE + ); + apcu_delete($iterator); + return true; + } + + /** + * Returns the `group value` for each of the configured groups + * If the group initial value was not found, then it initializes + * the group accordingly. + * + * @return array + */ + public function groups() { + if (empty($this->_compiledGroupNames)) { + foreach ($this->settings['groups'] as $group) { + $this->_compiledGroupNames[] = $this->settings['prefix'] . $group; + } + } + + $groups = apcu_fetch($this->_compiledGroupNames); + if (count($groups) !== count($this->settings['groups'])) { + foreach ($this->_compiledGroupNames as $group) { + if (!isset($groups[$group])) { + apcu_store($group, 1); + $groups[$group] = 1; + } + } + ksort($groups); + } + + $result = array(); + $groups = array_values($groups); + foreach ($this->settings['groups'] as $i => $group) { + $result[] = $group . $groups[$i]; + } + return $result; + } + + /** + * Increments the group value to simulate deletion of all keys under a group + * old values will remain in storage until they expire. + * + * @param string $group The group to clear. + * @return bool success + */ + public function clearGroup($group) { + apcu_inc($this->settings['prefix'] . $group, 1, $success); + return $success; + } + + /** + * Write data for key into cache if it doesn't exist already. + * If it already exists, it fails and returns false. + * + * @param string $key Identifier for the data. + * @param mixed $value Data to be cached. + * @param int $duration How long to cache the data, in seconds. + * @return bool True if the data was successfully cached, false on failure. + * @link http://php.net/manual/en/function.apc-add.php + */ + public function add($key, $value, $duration) { + return apc_add($key, $value, $duration); + } +} diff --git a/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php b/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php index e48ecc5c9..970a9dc74 100644 --- a/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php +++ b/app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php @@ -36,6 +36,7 @@ class OidcAuthenticate extends BaseAuthenticate $this->log($mispUsername, "Trying login."); $verifiedClaims = $oidc->getVerifiedClaims(); + $sub = $verifiedClaims->sub; $organisationProperty = $this->getConfig('organisation_property', 'organization'); if (property_exists($verifiedClaims, $organisationProperty)) { $organisationName = $verifiedClaims->{$organisationProperty}; @@ -52,8 +53,18 @@ class OidcAuthenticate extends BaseAuthenticate $roles = $oidc->requestUserInfo($roleProperty); } - $this->settings['fields'] = ['username' => 'email']; - $user = $this->_findUser($mispUsername); + // Try to find user by `sub` field, that is unique + $this->settings['fields'] = ['username' => 'sub']; + $user = $this->_findUser($sub); + + if (!$user) { // User by sub not found, try to find by email + $this->settings['fields'] = ['username' => 'email']; + $user = $this->_findUser($mispUsername); + if ($user && $user['sub'] !== null && $user['sub'] !== $sub) { + $this->log($mispUsername, "User sub doesn't match ({$user['sub']} != $sub), could not login."); + return false; + } + } $organisationId = $this->checkOrganization($organisationName, $user, $mispUsername); if (!$organisationId) { @@ -69,10 +80,22 @@ class OidcAuthenticate extends BaseAuthenticate if ($user) { $this->log($mispUsername, "Found in database with ID {$user['id']}."); + if ($user['sub'] === null) { + $this->userModel()->updateField($user, 'sub', $sub); + $this->log($mispUsername, "User sub changed from NULL to $sub."); + $user['sub'] = $sub; + } + + if ($user['email'] !== $mispUsername) { + $this->userModel()->updateField($user, 'email', $mispUsername); + $this->log($mispUsername, "User e-mail changed from {$user['email']} to $mispUsername."); + $user['email'] = $mispUsername; + } + if ($user['org_id'] != $organisationId) { - $user['org_id'] = $organisationId; $this->userModel()->updateField($user, 'org_id', $organisationId); $this->log($mispUsername, "User organisation changed from {$user['org_id']} to $organisationId."); + $user['org_id'] = $organisationId; } if ($user['role_id'] != $roleId) { @@ -86,7 +109,7 @@ class OidcAuthenticate extends BaseAuthenticate $this->log($mispUsername, "Unblocking user."); $user['disabled'] = false; } - + $this->storeMetadata($user['id'], $verifiedClaims); $this->log($mispUsername, 'Logged in.'); return $user; } @@ -100,12 +123,15 @@ class OidcAuthenticate extends BaseAuthenticate 'role_id' => $roleId, 'change_pw' => 0, 'date_created' => time(), + 'sub' => $sub, ]; if (!$this->userModel()->save($userData)) { throw new RuntimeException("Could not save user `$mispUsername` to database."); } + $this->storeMetadata($this->userModel()->id, $verifiedClaims); + $this->log($mispUsername, "Saved in database with ID {$this->userModel()->id}"); $this->log($mispUsername, 'Logged in.'); return $this->_findUser($mispUsername); @@ -227,6 +253,24 @@ class OidcAuthenticate extends BaseAuthenticate return $value; } + /** + * @param int $userId + * @param stdClass $verifiedClaims + * @return array|bool|mixed|null + * @throws Exception + */ + private function storeMetadata($userId, $verifiedClaims) + { + $value = []; + foreach (['sub', 'preferred_username', 'given_name', 'family_name'] as $field) { + if (property_exists($verifiedClaims, $field)) { + $value[$field] = $verifiedClaims->{$field}; + } + } + + return $this->userModel()->UserSetting->setSettingInternal($userId, 'oidc', $value); + } + /** * @param string $username * @param string $message diff --git a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php index 34ea870e2..a77fd389f 100644 --- a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php +++ b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php @@ -39,10 +39,11 @@ class ApacheShibbAuthenticate extends BaseAuthenticate * 'group_one' => 1, * ), * 'DefaultOrg' => 'MY_ORG', - * 'BlockOrgModifications' => false // set to true if you wish for the user's organisation never to be updated during login. Especially useful if you manually change organisations in MISP - * 'DefaultRole' => false // set to a specific value if you wish to hard-set users created via ApacheShibbAuth - * 'BlockRoleModifications' => false // set to true if you wish for the roles never to be updated during login. Especially * // useful if you manually change roles in MISP - * 'BlockOrgModifications' => false // set to true if you wish for the organizations never to be updated during login. Especially * // useful if you manually change orgs in MISP + * 'DefaultRole' => false, // set to a specific value if you wish to hard-set users created via ApacheShibbAuth + * 'BlockRoleModifications' => false, // set to true if you wish for the roles never to be updated during login. Especially + * // useful if you manually change roles in MISP + * 'BlockOrgModifications' => false, // set to true if you wish for the organizations never to be updated during login. Especially + * // useful if you manually change orgs in MISP * ), * @param CakeRequest $request The request that contains login information. * @param CakeResponse $response Unused response object. diff --git a/app/Plugin/ShibbAuth/README.md b/app/Plugin/ShibbAuth/README.md index bfcc1ce5b..abfb80efe 100644 --- a/app/Plugin/ShibbAuth/README.md +++ b/app/Plugin/ShibbAuth/README.md @@ -69,6 +69,32 @@ Edit your MISP apache configuration by adding the below (location depends on you SetHandler shib + + + AuthType shibboleth + Require shibboleth + # Optional: always directly redirect to the IdP upon access without + # an active session. If not enabled, a "Login with SAML" button + # will be shown on the login screen to start the login process.. + # ShibRequestSetting requiresession On + + +``` + +The above will always redirect to your IdP if MISP is accessed without +an active session. An alternative is not to require Shibboleth before +the application can be accessed but show the login screen if no +Shibboleth session is present. This will then have a +"Login with SAML" button to trigger the login. + +You may need to tell the rewriterules for MISP in its `.htaccess` file not to apply to Shibboleth, so add a line to exclude this from processing: + +```Apache + RewriteEngine On + RewriteCond %{REQUEST_URI} !^/Shibboleth\.sso + RewriteCond %{REQUEST_FILENAME} !-d + RewriteCond %{REQUEST_FILENAME} !-f + RewriteRule ^(.*)$ index.php?/$1 [QSA,L] ``` Enable the plugin at bootstrap.php: @@ -98,7 +124,7 @@ If the line does not exist, add it to 'Security' array, for example like below. ) ``` -And configure it. MailTag, OrgTag and GroupTag are the keys for the values needed by the plugin. +And configure it. MailTag, OrgTag and GroupTag are the keys for the values (Shibboleth environment variable names) needed by the plugin. For example if you are using ADFS you should replace IDP_FEDERATION_TAG by ADFS_FEDERATION, IDP_GROUP_TAG by ADFS_GROUP, etc. Replace MISP_DEFAULT_ORG by the organization you want users to be assigned to in case no organization value is given by the identity provider. The GroupRoleMatching is an array that allows the definition and correlation between groups and roles in MISP. These get updated @@ -119,8 +145,12 @@ in the list given by apache. By default, you can leave it at ';'. 'possible_group_attribute_value_1' => 1, ), 'DefaultOrg' => 'MISP_DEFAULT_ORG', + 'DefaultRole' => false, // set to a specific value if you wish to hard-set users created via ApacheShibbAuth + 'BlockRoleModifications' => false, // set to true if you wish for the roles never to be updated during login. Especially useful if you manually change roles in MISP + 'BlockOrgModifications' => false, // set to true if you wish for the organizations never to be updated during login. Especially useful if you manually change orgs in MISP ), ``` + If used with Apache as webserver it might be useful to make a distinction to filter out API/Syncs from SSO login. It can be added to the vhost as follows (Added lines are the If/Else clauses): ```Apache @@ -135,8 +165,6 @@ If used with Apache as webserver it might be useful to make a distinction to fil Require valid-user AuthType shibboleth ShibRequestSetting requiresession On - ShibRequestSetting shibexportassertion Off - ShibUseHeaders On ``` diff --git a/app/Test/AttributeValidationToolTest.php b/app/Test/AttributeValidationToolTest.php new file mode 100644 index 000000000..b22b0a2a4 --- /dev/null +++ b/app/Test/AttributeValidationToolTest.php @@ -0,0 +1,151 @@ +shouldBeValid('filename|md5', [ + 'cmd.exe|0cc175b9c0f1b6a831c399e269772661', + ]); + $this->shouldBeInvalid('filename|md5', [ + 'cmd.exe|86f7e437faa5a7fce15d1ddcb9eaeaea377667b8', + ]); + $this->shouldBeValid('tlsh', [ + 'b2317c38fac0333c8ff7d3ff31fcf3b7fb3f9a3ef3bf3c880cfc43ebf97f3cc73fbfc', + 't1fdd4e000b6a1c034f1f612f849b6a3a4b53f7ea1677481cf12d916ea4a79af1ed31317', + ]); + $this->shouldBeValid('filename|tlsh', [ + 'cmd.exe|b2317c38fac0333c8ff7d3ff31fcf3b7fb3f9a3ef3bf3c880cfc43ebf97f3cc73fbfc', + 'cmd.exe|t1fdd4e000b6a1c034f1f612f849b6a3a4b53f7ea1677481cf12d916ea4a79af1ed31317', + ]); + $this->shouldBeValid('ssdeep', [ + '96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO', + '384:EWo4X1WaPW9ZWhWzLo+lWpct/fWbkWsWIwW0/S7dZhgG8:EWo4X1WmW9ZWhWH/WpchfWgWsWTWtf8', + '6144:3wSQSlrBHFjOvwYAU/Fsgi/2WDg5+YaNk5xcHrYw+Zg+XrZsGEREYRGAFU25ttR/:ctM7E0L4q', + ]); + $this->shouldBeValid('filename|ssdeep', [ + 'ahoj.txt|96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO', + ]); + } + + public function testValidateIp(): void + { + foreach (['ip-src', 'ip-dst'] as $type) { + $this->shouldBeValid($type, [ + '127.0.0.1', + '127.0.0.1/32', + '::1', + '::1/128', + ]); + $this->shouldBeInvalid($type, [ + '127', + '127.0.0.', + '127.0.0.1/', + '127.0.0.1/32/1', + '127.0.0.1/128', + '::1/257', + '::1/257', + '::1/128/1', + ]); + } + } + + public function testValidatePort(): void + { + $this->assertTrue(AttributeValidationTool::validate('port', '1')); + $this->assertTrue(AttributeValidationTool::validate('port', 1)); + $this->assertTrue(AttributeValidationTool::validate('port', 80)); + $this->assertNotTrue(AttributeValidationTool::validate('port', -80)); + $this->assertNotTrue(AttributeValidationTool::validate('port', '-80')); + } + + public function testValidateSshFingerprint(): void + { + $this->shouldBeValid('ssh-fingerprint', [ + '7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a', + 'MD5:7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a', + 'SHA256:mVPwvezndPv/ARoIadVY98vAC0g+P/5633yTC4d/wXE', + ]); + } + + public function testValidateDomainIp(): void + { + $this->shouldBeValid('domain|ip', [ + 'example.com|127.0.0.1', + 'example.com|::1', + ]); + $this->shouldBeInvalid('domain|ip', [ + 'example.com|127', + 'example.com|1', + ]); + } + + public function testValidateFilename(): void + { + $this->shouldBeValid('filename', [ + 'cmd.exe', + 'cmd.com', + ]); + $this->shouldBeInvalid('filename', [ + "cmd.exe\ncmd.com", + ]); + $this->shouldBeValid('filename|md5', [ + 'cmd.exe|0cc175b9c0f1b6a831c399e269772661', + 'cmd.com|0cc175b9c0f1b6a831c399e269772661', + ]); + $this->shouldBeInvalid('filename|md5', [ + "cmd.exe\ncmd.com|0cc175b9c0f1b6a831c399e269772661", + ]); + } + + public function testCompressIpv6(): void + { + $this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500')); + $this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500')); + $this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500|80')); + $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1')); + } + + public function testFilenameHashLowercase() + { + $this->assertEquals('CMD.EXE|0cc175b9c0f1b6a831c399e269772661', AttributeValidationTool::modifyBeforeValidation('filename|md5', 'CMD.EXE|0CC175B9C0F1B6A831C399E269772661')); + } + + public function testDomainModify() + { + $this->assertEquals('example.com', AttributeValidationTool::modifyBeforeValidation('domain', 'example.com')); + $this->assertEquals('example.com', AttributeValidationTool::modifyBeforeValidation('domain', 'EXAMPLE.COM')); + $this->assertEquals('example.com|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|127.0.0.1')); + $this->assertEquals('example.com|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'EXAMPLE.COM|127.0.0.1')); + $this->assertEquals('xn--hkyrky-ptac70bc.cz', AttributeValidationTool::modifyBeforeValidation('domain', 'háčkyčárky.cz')); + $this->assertEquals('xn--hkyrky-ptac70bc.cz', AttributeValidationTool::modifyBeforeValidation('domain', 'HÁČKYČÁRKY.CZ')); + $this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'háčkyčárky.cz|127.0.0.1')); + $this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'HÁČKYČÁRKY.CZ|127.0.0.1')); + } + + private function shouldBeValid($type, array $values) + { + foreach ($values as $value) { + $this->assertTrue(AttributeValidationTool::validate($type, $value), "Value `$value` of type `$type` should be valid."); + } + } + + private function shouldBeInvalid($type, array $values) + { + foreach ($values as $value) { + $this->assertNotTrue(AttributeValidationTool::validate($type, $value), "Value `$value` of type `$type` should be invalid."); + } + } +} diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index 952fd3866..407892a1e 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -7,6 +7,7 @@ echo '
'; echo $this->element('/genericElements/IndexTable/index_table', [ 'data' => [ 'title' => __('Attributes'), + 'primary_id_path' => 'Attribute.id', 'data' => $attributes, 'fields' => [ [ @@ -19,7 +20,12 @@ echo $this->element('/genericElements/IndexTable/index_table', [ [ 'name' => __('Event'), 'sort' => 'Attribute.event_id', - 'data_path' => 'Attribute.event_id' + 'data_path' => 'Attribute.event_id', + 'element' => 'simple_link', + 'link_title_path' => 'Event.info', + 'url' => function (array $row) { + return '/events/view/' . $row['Attribute']['event_id'] . '/focus:' . $row['Attribute']['uuid']; + } ], [ 'name' => __('Org'), diff --git a/app/View/Attributes/search.ctp b/app/View/Attributes/search.ctp index f5c9a29e5..e94b014b0 100755 --- a/app/View/Attributes/search.ctp +++ b/app/View/Attributes/search.ctp @@ -28,10 +28,6 @@ 'type' => 'checkbox', 'label' => __('Only find IOCs flagged as to IDS') )); - echo $this->Form->input('alternate', array( - 'type' => 'checkbox', - 'label' => __('Alternate Search Result (Events)') - )); echo $this->Form->input('first_seen', array( 'type' => 'text', 'div' => 'input hidden', diff --git a/app/View/Elements/Events/View/attribute_correlations.ctp b/app/View/Elements/Events/View/attribute_correlations.ctp index 7fa6b41cc..262c863b9 100644 --- a/app/View/Elements/Events/View/attribute_correlations.ctp +++ b/app/View/Elements/Events/View/attribute_correlations.ctp @@ -1,6 +1,7 @@ $relatedAttribute) { @@ -10,7 +11,6 @@ foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) { $relatedEvents[$relatedAttribute['id']] = true; } } -$event['Related' . $scope][$object['id']] = array_values($event['Related' . $scope][$object['id']]); $count = count($event['Related' . $scope][$object['id']]); foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) { if ($i == 4 && $count > 5) { @@ -33,8 +33,10 @@ foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) { } $link = $this->Html->link( $relatedAttribute['id'], - array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['id']), - array('class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue') + $withPivot ? + ['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] : + ['controller' => 'events', 'action' => 'view', $relatedAttribute['id']], + ['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue'] ); echo sprintf( '
  • %s 
    • ', diff --git a/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp b/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp index be3c7feba..8d3e96215 100644 --- a/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp +++ b/app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp @@ -5,20 +5,19 @@ foreach ($warninglists as $id => $name) { } $warninglistsValues = json_encode($warninglistsValues); ?> -