get_all_7_char_chunks($block_data), - $this->get_all_7_char_chunks($double_block_data) - ); - return $result; + $hash = explode(':', $hash); + list($block_data, $double_block_data) = $hash; + + return [ + $blockSize, + $this->getAll7CharChunks($block_data), + $this->getAll7CharChunks($double_block_data) + ]; } - public function get_all_7_char_chunks($hash) + private function getAll7CharChunks($hash) { $results = array(); for ($i = 0; $i < strlen($hash) - 6; $i++) { @@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel return $results; } + /** + * @param string $hash + * @param int $attributeId + * @return array + */ public function query_ssdeep_chunks($hash, $attributeId) { $chunks = $this->ssdeep_prepare($hash); + $bothPartChunks = array_merge($chunks[1], $chunks[2]); // Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale // also propose to insert chunk size to database, but current database schema doesn't contain that column. // This optimisation can be add in future versions. $result = $this->find('column', array( 'conditions' => array( - 'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]), + 'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks, ), 'fields' => array('FuzzyCorrelateSsdeep.attribute_id'), 'unique' => true, @@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel $toSave = []; $attributeId = (int) $attributeId; - foreach (array(1, 2) as $type) { - foreach ($chunks[$type] as $chunk) { - $toSave[] = [$attributeId, $chunk]; - } - } - if (!empty($toSave)) { - $db = $this->getDataSource(); - $db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave); + foreach ($bothPartChunks as $chunk) { + $toSave[] = [$attributeId, $chunk]; } + $db = $this->getDataSource(); + $db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave); return $result; } diff --git a/app/Model/Galaxy.php b/app/Model/Galaxy.php index 906b506f6..bb8138f7d 100644 --- a/app/Model/Galaxy.php +++ b/app/Model/Galaxy.php @@ -264,7 +264,7 @@ class Galaxy extends AppModel $fields = array('galaxy_cluster_id', 'key', 'value'); $db->insertMulti('galaxy_elements', $fields, $elements); } - $allRelations = array_merge($allRelations, $relations); + array_push($allRelations, ...$relations); } // Save relation as last part when all clusters are created if (!empty($allRelations)) { @@ -287,24 +287,42 @@ class Galaxy extends AppModel if (empty($galaxy['uuid'])) { return false; } - $existingGalaxy = $this->find('first', array( + + $existingGalaxy = $this->find('first', [ 'recursive' => -1, - 'conditions' => array('Galaxy.uuid' => $galaxy['uuid']) - )); - if (empty($existingGalaxy)) { - if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) { - $this->create(); - unset($galaxy['id']); - $this->save($galaxy); - $existingGalaxy = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('Galaxy.id' => $this->id) - )); - } else { - return false; + 'conditions' => ['Galaxy.uuid' => $galaxy['uuid']], + ]); + + unset($galaxy['id']); + if (!empty($existingGalaxy)) { + // check if provided galaxy has the same fields as galaxy that are saved in database + $fieldsToSave = []; + foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) { + if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) { + $fieldsToSave[$key] = $galaxy[$key]; + } } + } else { + $fieldsToSave = $galaxy; } - return $existingGalaxy; + + if (empty($fieldsToSave) && !empty($existingGalaxy)) { + return $existingGalaxy; // galaxy already exists and galaxy fields are the same + } + + if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) { + return false; // user has no permission to modify galaxy + } + + if (empty($existingGalaxy)) { + $this->create(); + } + + $this->save($fieldsToSave); + return $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Galaxy.id' => $this->id], + ]); } /** diff --git a/app/Model/GalaxyElement.php b/app/Model/GalaxyElement.php index b648ce605..bb9289532 100644 --- a/app/Model/GalaxyElement.php +++ b/app/Model/GalaxyElement.php @@ -48,37 +48,6 @@ class GalaxyElement extends AppModel $this->saveMany($tempElements); } - public function update($galaxy_id, $oldClusters, $newClusters) - { - $elementsToSave = array(); - // Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them. - foreach ($oldClusters as $oldCluster) { - $this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id'])); - } - foreach ($newClusters as $newCluster) { - $tempCluster = array(); - foreach ($newCluster as $key => $value) { - // Don't store the reserved fields as elements - if ($key == 'description' || $key == 'value') { - continue; - } - if (is_array($value)) { - foreach ($value as $arrayElement) { - $tempCluster[] = array('key' => $key, 'value' => $arrayElement); - } - } else { - $tempCluster[] = array('key' => $key, 'value' => $value); - } - } - - foreach ($tempCluster as $key => $value) { - $tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id']; - } - $elementsToSave = array_merge($elementsToSave, $tempCluster); - } - $this->saveMany($elementsToSave); - } - public function captureElements($user, $elements, $clusterId) { $tempElements = array(); diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 6f9600aec..172a5ae21 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -153,6 +153,8 @@ class MispObject extends AppModel 'object_name' => array('function' => 'set_filter_object_name'), 'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'), 'object_template_version' => array('function' => 'set_filter_object_template_version'), + 'first_seen' => array('function' => 'set_filter_seen'), + 'last_seen' => array('function' => 'set_filter_seen'), 'deleted' => array('function' => 'set_filter_deleted') ), 'Event' => array( @@ -181,8 +183,8 @@ class MispObject extends AppModel 'deleted' => array('function' => 'set_filter_deleted'), 'timestamp' => array('function' => 'set_filter_timestamp'), 'attribute_timestamp' => array('function' => 'set_filter_timestamp'), - 'first_seen' => array('function' => 'set_filter_seen'), - 'last_seen' => array('function' => 'set_filter_seen'), + //'first_seen' => array('function' => 'set_filter_seen'), + //'last_seen' => array('function' => 'set_filter_seen'), 'to_ids' => array('function' => 'set_filter_to_ids'), 'comment' => array('function' => 'set_filter_comment') ) @@ -1678,7 +1680,9 @@ class MispObject extends AppModel $results = $this->Sightingdb->attachToObjects($results, $user); } $params['page'] += 1; - $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true); + foreach ($results as $k => $result) { + $results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true); + } $results = array_values($results); $i = 0; foreach ($results as $object) { diff --git a/app/Model/Module.php b/app/Model/Module.php index cb99a10fd..337861efb 100644 --- a/app/Model/Module.php +++ b/app/Model/Module.php @@ -50,6 +50,8 @@ class Module extends AppModel ) ); + private $httpSocket = []; + public function validateIPField($value) { if (!filter_var($value, FILTER_VALIDATE_IP) === false) { @@ -309,16 +311,9 @@ class Module extends AppModel if (!$serverUrl) { throw new Exception("Module type $moduleFamily is not enabled."); } - App::uses('HttpSocketExtended', 'Tools'); - $httpSocketSetting = ['timeout' => $timeout]; - $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile'); - foreach ($sslSettings as $sslSetting) { - $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); - if ($value && $value !== '') { - $httpSocketSetting[$sslSetting] = $value; - } - } - $httpSocket = new HttpSocketExtended($httpSocketSetting); + + $httpSocket = $this->initHttpSocket($moduleFamily, $timeout); + $request = []; if ($moduleFamily === 'Cortex') { if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) { @@ -422,4 +417,37 @@ class Module extends AppModel return false; } + + /** + * @param string $moduleFamily + * @param int $timeout + * @return HttpSocketExtended|CurlClient + */ + private function initHttpSocket($moduleFamily, $timeout) + { + $unique = "$moduleFamily:$timeout"; + + if (isset($this->httpSocket[$unique])) { + return $this->httpSocket[$unique]; + } + + $httpSocketSetting = ['timeout' => $timeout]; + $sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile']; + foreach ($sslSettings as $sslSetting) { + $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); + if ($value && $value !== '') { + $httpSocketSetting[$sslSetting] = $value; + } + } + + if (function_exists('curl_init')) { + App::uses('CurlClient', 'Tools'); + $httpSocket = new CurlClient($httpSocketSetting); + } else { + App::uses('HttpSocketExtended', 'Tools'); + $httpSocket = new HttpSocketExtended($httpSocketSetting); + } + + return $this->httpSocket[$unique] = $httpSocket; + } } diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index 730944b19..c03e3066a 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -76,14 +76,28 @@ class Organisation extends AppModel ); const ORGANISATION_ASSOCIATIONS = array( + 'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')), + 'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')), 'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')), + 'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')), + 'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')), 'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')), + 'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')), + 'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')), + 'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')), + 'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')), 'Job' => array('table' => 'jobs', 'fields' => array('org_id')), + 'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')), 'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')), 'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')), 'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')), 'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')), + 'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')), + 'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')), + 'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')), 'Thread' => array('table' => 'threads', 'fields' => array('org_id')), + 'Tag' => array('table' => 'tags', 'fields' => array('org_id')), + 'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')), 'User' => array('table' => 'users', 'fields' => array('org_id')) ); @@ -287,6 +301,9 @@ class Organisation extends AppModel public function orgMerge($id, $request, $user) { $currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id))); + if (isset($currentOrg['Organisation']['restricted_to_domain'])) { + $currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']); + } $currentOrgUserCount = $this->User->find('count', array( 'conditions' => array('User.org_id' => $id) )); diff --git a/app/Model/Server.php b/app/Model/Server.php index 0ead31d16..14a3f4ab6 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -472,7 +472,21 @@ class Server extends AppModel return false; } - private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false) + /** + * @param array $event + * @param int|string $eventId + * @param array $successes + * @param array $fails + * @param Event $eventModel + * @param array $server + * @param array $user + * @param int $jobId + * @param bool $force + * @param HttpSocketResponseExtended $response + * @return false|void + * @throws Exception + */ + private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response) { // check if the event already exist (using the uuid) $existingEvent = $eventModel->find('first', [ @@ -485,7 +499,7 @@ class Server extends AppModel if (!$existingEvent) { // add data for newly imported events if (isset($event['Event']['protected']) && $event['Event']['protected']) { - if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) { + if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) { $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); return false; } @@ -505,7 +519,7 @@ class Server extends AppModel $fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.'); } else { if ($existingEvent['Event']['protected']) { - if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) { + if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) { $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); } } @@ -549,12 +563,10 @@ class Server extends AppModel $params['excludeLocalTags'] = 1; } try { - $event = $serverSync->fetchEvent($eventId, $params); - $headers = $event->headers; - $body = $event->body; - $event = $event->json(); + $response = $serverSync->fetchEvent($eventId, $params); + $event = $response->json(); } catch (Exception $e) { - $this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e); + $this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e); $fails[$eventId] = __('failed downloading the event'); return false; } @@ -568,7 +580,7 @@ class Server extends AppModel } return false; } - $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); + $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response); return true; } @@ -2359,23 +2371,21 @@ class Server extends AppModel return $setting; } - public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false) + /** + * @param array|string $user + * @param array $setting + * @param mixed $value + * @param bool $forceSave + * @return mixed|string|true|null + * @throws Exception + */ + public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false) { if (isset($setting['beforeHook'])) { - $beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value)); + $beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value); if ($beforeResult !== true) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->saveOrFailSilently(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => 0, - 'email' => $user['email'], - 'action' => 'serverSettingsEdit', - 'user_id' => $user['id'], - 'title' => 'Server setting issue', - 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.', - )); + $change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.'; + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); return $beforeResult; } } @@ -2384,7 +2394,7 @@ class Server extends AppModel if ($setting['type'] === 'boolean') { $value = (bool)$value; } else if ($setting['type'] === 'numeric') { - $value = (int)($value); + $value = (int)$value; } if (isset($setting['test'])) { if ($setting['test'] instanceof Closure) { @@ -2425,7 +2435,7 @@ class Server extends AppModel if ($setting['afterHook'] instanceof Closure) { $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue); } else { - $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue)); + $afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue); } if ($afterResult !== true) { $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; @@ -2434,9 +2444,8 @@ class Server extends AppModel } } return true; - } else { - return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.'); } + return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.'); } /** @@ -2539,10 +2548,10 @@ class Server extends AppModel 'name' => __('Organisation logos'), 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), 'expected' => array(), - 'valid_format' => __('48x48 pixel .png files'), + 'valid_format' => __('48x48 pixel .png files or .svg file'), 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', - 'regex' => '.*\.(png|PNG)$', - 'regex_error' => __('Filename must be in the following format: *.png'), + 'regex' => '.*\.(png|svg)$', + 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), 'files' => array(), ), 'img' => array( @@ -2578,6 +2587,7 @@ class Server extends AppModel 'read' => $f->isReadable(), 'write' => $f->isWritable(), 'execute' => $f->isExecutable(), + 'link' => $f->isLink(), ]; } } @@ -4155,12 +4165,13 @@ class Server extends AppModel private function checkRemoteVersion($HttpSocket) { try { - $json_decoded_tags = GitTool::getLatestTags($HttpSocket); + $tags = GitTool::getLatestTags($HttpSocket); } catch (Exception $e) { + $this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE); return false; } // find the latest version tag in the v[major].[minor].[hotfix] format - foreach ($json_decoded_tags as $tag) { + foreach ($tags as $tag) { if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) { return $this->checkVersion($tag['name']); } @@ -4182,7 +4193,7 @@ class Server extends AppModel try { $latestCommit = GitTool::getLatestCommit($HttpSocket); } catch (Exception $e) { - $latestCommit = false; + $this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE); } } @@ -4202,6 +4213,7 @@ class Server extends AppModel try { return GitTool::currentBranch(); } catch (Exception $e) { + $this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE); return false; } } @@ -4252,38 +4264,38 @@ class Server extends AppModel 'app/files/scripts/misp-opendata', 'app/files/scripts/python-maec', 'app/files/scripts/python-stix', - ); return in_array($submodule, $accepted_submodules_names, true); } /** - * @param string $submodule_name - * @param string $superproject_submodule_commit_id + * @param string $submoduleName + * @param string $superprojectSubmoduleCommitId * @return array + * @throws Exception */ - private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) + private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId) { - $path = APP . '../' . $submodule_name; - $submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); - $submodule_name = end($submodule_name); + $path = APP . '../' . $submoduleName; + $submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName); + $submoduleName = end($submoduleName); - $submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path); + $submoduleCurrentCommitId = GitTool::currentCommit($path); $currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path); - if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) { - $remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path); + if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) { + $remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path); } else { $remoteTimestamp = $currentTimestamp; } $status = array( - 'moduleName' => $submodule_name, + 'moduleName' => $submoduleName, 'current' => $submoduleCurrentCommitId, 'currentTimestamp' => $currentTimestamp, - 'remote' => $superproject_submodule_commit_id, + 'remote' => $superprojectSubmoduleCommitId, 'remoteTimestamp' => $remoteTimestamp, - 'upToDate' => '', + 'upToDate' => 'error', 'isReadable' => is_readable($path) && is_readable($path . '/.git'), ); @@ -4295,15 +4307,11 @@ class Server extends AppModel } else { $status['upToDate'] = 'younger'; } - } else { - $status['upToDate'] = 'error'; } if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { - $date1 = new DateTime(); - $date1->setTimestamp($status['remoteTimestamp']); - $date2 = new DateTime(); - $date2->setTimestamp($status['currentTimestamp']); + $date1 = new DateTime("@{$status['remoteTimestamp']}"); + $date2 = new DateTime("@{$status['currentTimestamp']}"); $status['timeDiff'] = $date1->diff($date2); } else { $status['upToDate'] = 'error'; @@ -4793,11 +4801,11 @@ class Server extends AppModel $results = [ __('User') => $user['User']['email'], - __('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'), + __('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'), __('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'), ]; - if (isset($response->headers['X-Auth-Key-Expiration'])) { - $date = new DateTime($response->headers['X-Auth-Key-Expiration']); + if ($response->getHeader('X-Auth-Key-Expiration')) { + $date = new DateTime($response->getHeader('X-Auth-Key-Expiration')); $results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s'); } return $results; @@ -4935,6 +4943,28 @@ class Server extends AppModel return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); } + /** + * @param string $encryptionKey + * @return bool + * @throws Exception + */ + public function isEncryptionKeyValid($encryptionKey) + { + $servers = $this->find('list', [ + 'fields' => ['Server.id', 'Server.authkey'], + ]); + foreach ($servers as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for server #$id", 0, $e); + } + } + } + return true; + } + /** * Return all Attribute and Object types */ @@ -5143,9 +5173,9 @@ class Server extends AppModel 'type' => 'string', ), 'disable_cached_exports' => array( - 'level' => 1, - 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), - 'value' => false, + 'level' => 2, + 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Even tough the feature is deprecated and will be removed in the future, you can still decide to enable it.'), + 'value' => true, 'null' => true, 'test' => 'testDisableCache', 'type' => 'boolean', @@ -6143,6 +6173,14 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true, ], + 'block_publishing_for_same_creator' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ], 'self_update' => [ 'level' => self::SETTING_CRITICAL, 'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'), diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index d19148bf8..6f91868ba 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -1017,16 +1017,16 @@ class Sighting extends AppModel * @return TmpFileTool * @throws Exception */ - public function restSearch(array $user, $returnFormat, $filters) + public function restSearch(array $user, $returnFormat, array $filters) { $allowedContext = array('event', 'attribute'); // validate context if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) { - throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context'])); + throw new BadRequestException(__('Invalid context %s.', $filters['context'])); } // ensure that an id or uuid is provided if context is set if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) { - throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.')); + throw new BadRequestException(__('An ID or UUID must be provided if the context is set.')); } if (!isset($this->validFormats[$returnFormat][1])) { @@ -1102,8 +1102,12 @@ class Sighting extends AppModel $conditions['Attribute.uuid'] = $filters['uuid']; $contain[] = 'Attribute'; } elseif ($filters['context'] === 'event') { - $conditions['Event.uuid'] = $filters['uuid']; - $contain[] = 'Event'; + $temp = $this->Event->find('column', [ + 'recursive' => -1, + 'fields' => ['Event.id'], + 'conditions' => ['Event.uuid IN' => $filters['uuid']] + ]); + $conditions['Sighting.event_id'] = empty($temp) ? -1 : $temp; } } @@ -1131,15 +1135,30 @@ class Sighting extends AppModel $tmpfile = new TmpFileTool(); $tmpfile->write($exportTool->header($exportToolParams)); $separator = $exportTool->separator($exportToolParams); - + // fetch sightings matching the query without ACL checks - $sightingIds = $this->find('column', [ - 'conditions' => $conditions, - 'fields' => ['Sighting.id'], - 'contain' => $contain, - 'order' => 'Sighting.id', - ]); - + if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) { + $conditions_copy = $conditions; + $sightingIds = []; + foreach ($conditions['Sighting.event_id'] as $e_id) { + $conditions_copy['Sighting.event_id'] = $e_id; + $tempIds = $this->find('column', [ + 'conditions' => $conditions, + 'fields' => ['Sighting.id'], + 'contain' => $contain + ]); + if (!empty($tempIds)) { + $sightingIds = array_merge($sightingIds, $tempIds); + } + } + } else { + $sightingIds = $this->find('column', [ + 'conditions' => $conditions, + 'fields' => ['Sighting.id'], + 'contain' => $contain + ]); + } + foreach (array_chunk($sightingIds, 500) as $chunk) { // fetch sightings with ACL checks and sighting policies $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid); @@ -1396,7 +1415,7 @@ class Sighting extends AppModel try { $sightings = $serverSync->fetchSightingsForEvents($chunk); } catch (Exception $e) { - $this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e); + $this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e); continue; } diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php index d30eefcd4..875a57c75 100644 --- a/app/Model/SystemSetting.php +++ b/app/Model/SystemSetting.php @@ -46,7 +46,7 @@ class SystemSetting extends AppModel { /** @var self $systemSetting */ $systemSetting = ClassRegistry::init('SystemSetting'); - if (!$systemSetting->databaseExists()) { + if (!$systemSetting->tableExists()) { return; } $settings = $systemSetting->getSettings(); @@ -58,7 +58,7 @@ class SystemSetting extends AppModel } } - public function databaseExists() + private function tableExists() { $tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources(); return in_array('system_settings', $tables, true); @@ -154,6 +154,32 @@ class SystemSetting extends AppModel return $this->saveMany($toSave); } + /** + * Check if provided encryption key is valid for all encrypted settings + * @param string $encryptionKey + * @return bool + * @throws Exception + */ + public function isEncryptionKeyValid($encryptionKey) + { + $settings = $this->find('list', [ + 'fields' => ['SystemSetting.setting', 'SystemSetting.value'], + ]); + foreach ($settings as $setting => $value) { + if (!self::isSensitive($setting)) { + continue; + } + if (EncryptedValue::isEncrypted($value)) { + try { + BetterSecurity::decrypt(substr($value, 2), $encryptionKey); + } catch (Exception $e) { + throw new Exception("Could not decrypt `$setting` setting.", 0, $e); + } + } + } + return true; + } + /** * Sensitive setting are passwords or api keys. * @param string $setting Setting name diff --git a/app/Model/User.php b/app/Model/User.php index 1965ec560..b0d1d0358 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -659,21 +659,18 @@ class User extends AppModel public function getUserById($id) { if (empty($id)) { - throw new NotFoundException('Invalid user ID.'); + throw new InvalidArgumentException('Invalid user ID.'); } - return $this->find( - 'first', - array( - 'conditions' => array('User.id' => $id), - 'recursive' => -1, - 'contain' => array( - 'Organisation', - 'Role', - 'Server', - 'UserSetting', - ) - ) - ); + return $this->find('first', [ + 'conditions' => ['User.id' => $id], + 'recursive' => -1, + 'contain' => [ + 'Organisation', + 'Role', + 'Server', + 'UserSetting', + ] + ]); } /** @@ -740,7 +737,7 @@ class User extends AppModel ], ]); if (empty($user)) { - return $user; + return null; } return $this->rearrangeToAuthForm($user); } @@ -861,6 +858,10 @@ class User extends AppModel return true; } + if (!isset($user['User'])) { + throw new InvalidArgumentException("Invalid user model provided."); + } + if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) { return true; } @@ -937,6 +938,11 @@ class User extends AppModel */ public function describeAuthFields() { + static $fields; // generate array just once + if ($fields) { + return $fields; + } + $fields = $this->schema(); // Do not include keys, because they are big and usually not necessary unset($fields['gpgkey']); @@ -1105,13 +1111,18 @@ class User extends AppModel return $hashed; } - public function createInitialUser($org_id) + /** + * @param int $orgId + * @return string User auth key + * @throws Exception + */ + public function createInitialUser($orgId) { $authKey = $this->generateAuthKey(); $admin = array('User' => array( 'id' => 1, 'email' => 'admin@admin.test', - 'org_id' => $org_id, + 'org_id' => $orgId, 'password' => 'admin', 'confirm_password' => 'admin', 'authkey' => $authKey, @@ -1123,7 +1134,6 @@ class User extends AppModel $this->validator()->remove('password'); // password is too simple, remove validation $this->save($admin); if (!empty(Configure::read("Security.advanced_authkeys"))) { - $this->AuthKey = ClassRegistry::init('AuthKey'); $newKey = [ 'authkey' => $authKey, 'user_id' => 1, @@ -2068,12 +2078,10 @@ class User extends AppModel return false; } - $cutoff = $redis->get('misp:session_destroy:' . $id); - $allcutoff = $redis->get('misp:session_destroy:all'); + list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']); if ( empty($cutoff) || ( - !empty($cutoff) && !empty($allcutoff) && $allcutoff < $cutoff ) @@ -2156,7 +2164,7 @@ class User extends AppModel if (!ctype_alnum($token)) { return false; } - $redis = $this->setupRedis(); + $redis = RedisTool::init(); $userId = $redis->get('misp:forgot:' . $token); if (empty($userId)) { return false; @@ -2167,8 +2175,78 @@ class User extends AppModel public function purgeForgetToken($token) { - $redis = $this->setupRedis(); - $userId = $redis->del('misp:forgot:' . $token); + $redis = RedisTool::init(); + $redis->del('misp:forgot:' . $token); return true; } + + /** + * Create default Role, Organisation and User + * @return string|null Created user auth key + * @throws Exception + */ + public function init() + { + if (!$this->Role->hasAny()) { + $siteAdmin = ['Role' => [ + 'id' => 1, + 'name' => 'Site Admin', + 'permission' => 3, + 'perm_add' => 1, + 'perm_modify' => 1, + 'perm_modify_org' => 1, + 'perm_publish' => 1, + 'perm_sync' => 1, + 'perm_admin' => 1, + 'perm_audit' => 1, + 'perm_auth' => 1, + 'perm_site_admin' => 1, + 'perm_regexp_access' => 1, + 'perm_sharing_group' => 1, + 'perm_template' => 1, + 'perm_tagger' => 1, + ]]; + $this->Role->save($siteAdmin); + // PostgreSQL: update value of auto incremented serial primary key after setting the column by force + if (!$this->isMysql()) { + $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));"; + $this->Role->query($sql); + } + } + + if (!$this->Organisation->hasAny(['Organisation.local' => true])) { + $this->runUpdates(); + $org = ['Organisation' => [ + 'id' => 1, + 'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN', + 'description' => 'Automatically generated admin organisation', + 'type' => 'ADMIN', + 'date_created' => date('Y-m-d H:i:s'), + 'local' => 1, + ]]; + $this->Organisation->save($org); + // PostgreSQL: update value of auto incremented serial primary key after setting the column by force + if (!$this->isMysql()) { + $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));"; + $this->Organisation->query($sql); + } + $orgId = $this->Organisation->id; + } + + if (!$this->hasAny()) { + if (!isset($orgId)) { + $hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1)); + if (!empty($hostOrg)) { + $orgId = $hostOrg['Organisation']['id']; + } else { + $firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC')); + $orgId = $firstOrg['Organisation']['id']; + } + } + $this->runUpdates(); + return $this->createInitialUser($orgId); + } + + return null; + } } diff --git a/app/Model/UserLoginProfile.php b/app/Model/UserLoginProfile.php index 27ca1081a..1d7e7ee33 100644 --- a/app/Model/UserLoginProfile.php +++ b/app/Model/UserLoginProfile.php @@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel ]; const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap'; - const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI + const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/ private $userProfile; private $knownUserProfiles = []; - private function _buildBrowscapCache() + private function browscapGetBrowser() { - $this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO); - $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); - $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); - $logger = new \Monolog\Logger('name'); - $bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger); - $bc->convertFile(UserLoginProfile::BROWSER_INI_FILE); + + if (function_exists('apcu_fetch')) { + App::uses('ApcuCacheTool', 'Tools'); + $cache = new ApcuCacheTool('misp:browscap'); + } else { + $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); + $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); + } + + try { + $bc = new \BrowscapPHP\Browscap($cache, $logger); + return $bc->getBrowser(); + } catch (\BrowscapPHP\Exception $e) { + $this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO); + $bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger); + $bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE)); + } + + $bc = new \BrowscapPHP\Browscap($cache, $logger); + return $bc->getBrowser(); + } + + /** + * @param string $ip + * @return string|null + */ + public function countryByIp($ip) + { + if (class_exists('GeoIp2\Database\Reader')) { + $geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE); + try { + $record = $geoDbReader->country($ip); + return $record->country->isoCode; + } catch (InvalidArgumentException $e) { + $this->logException("Could not get country code for IP address", $e, LOG_NOTICE); + return null; + } + } + return null; } public function beforeSave($options = []) @@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel if (!$this->userProfile) { // below uses https://github.com/browscap/browscap-php if (class_exists('\BrowscapPHP\Browscap')) { - try { - $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); - $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); - $logger = new \Monolog\Logger('name'); - $bc = new \BrowscapPHP\Browscap($cache, $logger); - $browser = $bc->getBrowser(); - } catch (\BrowscapPHP\Exception $e) { - $this->_buildBrowscapCache(); - return $this->_getUserProfile(); - } + $browser = $this->browscapGetBrowser(); } else { // a primitive OS & browser extraction capability $ua = $_SERVER['HTTP_USER_AGENT'] ?? null; @@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel $browser->browser = "browser"; } $ip = $this->_remoteIp(); - if (class_exists('GeoIp2\Database\Reader')) { - try { - $geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE); - $record = $geoDbReader->country($ip); - $country = $record->country->isoCode; - } catch (InvalidArgumentException $e) { - $this->logException("Could not get country code for IP address", $e); - $country = 'None'; - } - } else { - $country = 'None'; - } + $country = $this->countryByIp($ip) ?? 'None'; $this->userProfile = [ 'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null, 'ip' => $ip, @@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel public function emailNewLogin(array $user) { if (!Configure::read('MISP.disable_emailing')) { - $date_time = date('c'); - + $user = $this->User->getUserById($user['id']); // fetch in database format + $datetime = date('c'); // ISO 8601 date $body = new SendEmailTemplate('userloginprofile_newlogin'); $body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile()); $body->set('baseurl', Configure::read('MISP.baseurl')); $body->set('misp_org', Configure::read('MISP.org')); - $body->set('date_time', $date_time); + $body->set('date_time', $datetime); // Fetch user that contains also PGP or S/MIME keys for e-mail encryption $this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in."); } diff --git a/app/Model/Warninglist.php b/app/Model/Warninglist.php index b4313ddc2..bc3e649d4 100644 --- a/app/Model/Warninglist.php +++ b/app/Model/Warninglist.php @@ -390,8 +390,7 @@ class Warninglist extends AppModel $warninglistId = (int)$this->id; $result = true; - $keys = array_keys($list['list']); - if ($keys === array_keys($keys)) { + if (JsonTool::arrayIsList($list['list'])) { foreach (array_chunk($list['list'], 1000) as $chunk) { $valuesToInsert = []; foreach ($chunk as $value) { diff --git a/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php b/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php new file mode 100644 index 000000000..c39173561 --- /dev/null +++ b/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php @@ -0,0 +1,276 @@ + 'BinaryFile', + 'path' => CACHE, + 'prefix' => 'cake_', + 'serialize' => true, + 'mask' => 0660, + ]; + CacheEngine::init($settings); + + $this->useIgbinary = function_exists('igbinary_serialize'); + if (substr($this->settings['path'], -1) !== DS) { + $this->settings['path'] .= DS; + } + if (!empty($this->_groupPrefix)) { + $this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix); + } + return $this->_active(); + } + + /** + * @param string $key + * @param mixed $data + * @param int $duration + * @return bool + */ + public function write($key, $data, $duration) + { + if (!$this->_init) { + return false; + } + + $fileInfo = $this->cacheFilePath($key); + $resource = $this->createFile($fileInfo); + if (!$resource) { + return false; + } + + if (!empty($this->settings['serialize'])) { + if ($this->useIgbinary) { + $data = igbinary_serialize($data); + if ($data === null) { + return false; + } + } else { + $data = serialize($data); + } + } + + $expires = pack("q", time() + $duration); + + flock($resource, LOCK_EX); + + ftruncate($resource, 0); + + $result = fwrite($resource, $expires); + if ($result !== self::BINARY_CACHE_TIME_LENGTH) { + $this->handleWriteError($fileInfo); + fclose($resource); + return false; + } + + $result = fwrite($resource, $data); + if ($result !== strlen($data)) { + $this->handleWriteError($fileInfo); + fclose($resource); + return false; + } + + fclose($resource); + + return true; + } + + /** + * @param string $key + * @return false|mixed|string + */ + public function read($key) + { + if (!$this->_init) { + return false; + } + + $fileInfo = $this->cacheFilePath($key); + + $exists = file_exists($fileInfo->getPathname()); + if (!$exists) { + return false; + } + + $resource = $this->openFile($fileInfo); + if (!$resource) { + return false; + } + + $time = time(); + + flock($resource, LOCK_SH); + + $cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH); + if (!$cacheTimeBinary) { + fclose($resource); + return false; + } + + $cacheTime = $this->unpackCacheTime($cacheTimeBinary); + if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) { + fclose($resource); + return false; // already expired + } + + $data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH); + fclose($resource); + + if (!empty($this->settings['serialize'])) { + if ($this->useIgbinary) { + $data = igbinary_unserialize($data); + } else { + $data = unserialize($data); + } + } + + return $data; + } + + /** + * @param string $path + * @param int $now + * @param int $threshold + * @return void + */ + protected function _clearDirectory($path, $now, $threshold) + { + $prefixLength = strlen($this->settings['prefix']); + + if (!is_dir($path)) { + return; + } + + $dir = dir($path); + if ($dir === false) { + return; + } + + while (($entry = $dir->read()) !== false) { + if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) { + continue; + } + + try { + $file = new SplFileObject($path . $entry, 'rb'); + } catch (Exception $e) { + continue; + } + + if ($threshold) { + $mtime = $file->getMTime(); + if ($mtime > $threshold) { + continue; + } + $expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH)); + if ($expires > $now) { + continue; + } + } + if ($file->isFile()) { + $filePath = $file->getRealPath(); + $file = null; + @unlink($filePath); + } + } + } + + /** + * @param SplFileInfo $fileInfo + * @return false|resource + */ + private function createFile(SplFileInfo $fileInfo) + { + $exists = file_exists($fileInfo->getPathname()); + if (!$exists) { + $resource = $this->openFile($fileInfo, 'cb'); + if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) { + trigger_error(__d( + 'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"', + [$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING); + } + return $resource; + } + + return $this->openFile($fileInfo, 'cb'); + } + + /** + * @param SplFileInfo $fileInfo + * @param string $mode + * @return false|resource + */ + private function openFile(SplFileInfo $fileInfo, $mode = 'rb') + { + $resource = fopen($fileInfo->getPathname(), $mode); + if (!$resource) { + trigger_error(__d( + 'cake_dev', 'Could not open file %s', + array($fileInfo->getPathname())), E_USER_WARNING); + } + return $resource; + } + + /** + * @param string $key + * @return SplFileInfo + */ + private function cacheFilePath(string $key): SplFileInfo + { + $groups = null; + if (!empty($this->_groupPrefix)) { + $groups = vsprintf($this->_groupPrefix, $this->groups()); + } + $dir = $this->settings['path'] . $groups; + + if (!is_dir($dir)) { + mkdir($dir, 0775, true); + } + + $suffix = '.bin'; + if ($this->settings['serialize'] && $this->useIgbinary) { + $suffix = '.igbin'; + } + + return new SplFileInfo($dir . $key . $suffix); + } + + /** + * @param SplFileInfo $fileInfo + * @return void + */ + private function handleWriteError(SplFileInfo $fileInfo) + { + unlink($fileInfo->getPathname()); // delete file in case file was just partially written + trigger_error(__d( + 'cake_dev', 'Could not write to file %s', + array($fileInfo->getPathname())), E_USER_WARNING); + } + + /** + * @param string $cacheTimeBinary + * @return int + */ + private function unpackCacheTime($cacheTimeBinary) + { + if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) { + throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'"); + } + return unpack("q", $cacheTimeBinary)[1]; + } +} \ No newline at end of file diff --git a/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php b/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php index 6cb3c8d4b..4e08b1eb3 100644 --- a/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php +++ b/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php @@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface 'log' => [ 'level' => $type, ], - 'message' => $message, + 'message' => JsonTool::escapeNonUnicode($message), ]; static::writeMessage($message); diff --git a/app/Plugin/OidcAuth/Lib/Oidc.php b/app/Plugin/OidcAuth/Lib/Oidc.php index 67f712088..d8d4dd210 100644 --- a/app/Plugin/OidcAuth/Lib/Oidc.php +++ b/app/Plugin/OidcAuth/Lib/Oidc.php @@ -28,7 +28,6 @@ class Oidc $claims = $oidc->getVerifiedClaims(); $mispUsername = $claims->email ?? $oidc->requestUserInfo('email'); - if (empty($mispUsername)) { $sub = $claims->sub ?? 'UNKNOWN'; throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP."); @@ -66,13 +65,13 @@ class Oidc $roleProperty = $this->getConfig('roles_property', 'roles'); $roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty); if ($roles === null) { - $this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING); + $this->log($mispUsername, "Role property `$roleProperty` is missing in claims, access prohibited.", LOG_WARNING); return false; } $roleId = $this->getUserRole($roles, $mispUsername); if ($roleId === null) { - $this->log($mispUsername, 'No role was assigned.'); + $this->log($mispUsername, 'No role was assigned, access prohibited.', LOG_WARNING); if ($user) { $this->block($user); } diff --git a/app/Test/AttributeValidationToolTest.php b/app/Test/AttributeValidationToolTest.php index b1781220d..b08a13fda 100644 --- a/app/Test/AttributeValidationToolTest.php +++ b/app/Test/AttributeValidationToolTest.php @@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase ]); } + public function testRemoveCidrFromIp(): void + { + $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32')); + $this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31')); + $this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128')); + $this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80')); + $this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80')); + $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1')); + } + public function testCompressIpv6(): void { $this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500')); diff --git a/app/Test/ComplexTypeToolTest.php b/app/Test/ComplexTypeToolTest.php index 54acead69..16d4a6273 100644 --- a/app/Test/ComplexTypeToolTest.php +++ b/app/Test/ComplexTypeToolTest.php @@ -527,10 +527,31 @@ EOT; public function testCheckFreeTextNonBreakableSpace(): void { $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2"); $this->assertCount(2, $results); $this->assertEquals('127.0.0.1', $results[0]['value']); $this->assertEquals('ip-dst', $results[0]['default_type']); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0\xc2\xa0127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + public function testCheckFreeTextControlCharToSpace(): void + { + $complexTypeTool = new ComplexTypeTool(); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\x1d127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\x1d\x1d127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); } public function testCheckFreeTextQuoted(): void diff --git a/app/Test/JSONConverterToolTest.php b/app/Test/JSONConverterToolTest.php index 5e7758148..adc47fb4a 100644 --- a/app/Test/JSONConverterToolTest.php +++ b/app/Test/JSONConverterToolTest.php @@ -1,4 +1,5 @@ assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces); - if (defined('JSON_THROW_ON_ERROR')) { - json_decode($json, true, 512, JSON_THROW_ON_ERROR); - $this->assertTrue(true); - } else { - $this->assertNotNull(json_decode($json)); - } + $this->assertTrue(JsonTool::isValid($json)); } } diff --git a/app/View/AuthKeys/view.ctp b/app/View/AuthKeys/view.ctp index 2dbac015a..5bd195ff0 100644 --- a/app/View/AuthKeys/view.ctp +++ b/app/View/AuthKeys/view.ctp @@ -3,12 +3,12 @@ $keyUsageCsv = null; if (isset($keyUsage)) { $todayString = date('Y-m-d'); $today = strtotime($todayString); - $startDate = key($keyUsage); // oldest date for sparkline + $startDate = array_key_first($keyUsage); // oldest date for sparkline $startDate = strtotime($startDate) - (3600 * 24 * 3); $keyUsageCsv = 'Date,Close\n'; for ($date = $startDate; $date <= $today; $date += (3600 * 24)) { $dateAsString = date('Y-m-d', $date); - $keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n'; + $keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n'; } } else { $lastUsed = null; diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp index 4a8e50968..5cdfae0c1 100755 --- a/app/View/Elements/global_menu.ctp +++ b/app/View/Elements/global_menu.ctp @@ -277,11 +277,6 @@ 'url' => $baseurl . '/servers/createSync', 'requirement' => $isAclSync && !$isSiteAdmin ), - array( - 'text' => __('Import Server Settings'), - 'url' => $baseurl . '/servers/import', - 'requirement' => $this->Acl->canAccess('servers', 'import'), - ), array( 'text' => __('Remote Servers'), 'url' => $baseurl . '/servers/index', @@ -292,11 +287,6 @@ 'url' => $baseurl . '/feeds/index', 'requirement' => $this->Acl->canAccess('feeds', 'index'), ), - array( - 'text' => __('Search Feed Caches'), - 'url' => $baseurl . '/feeds/searchCaches', - 'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'), - ), array( 'text' => __('SightingDB'), 'url' => $baseurl . '/sightingdb/index', @@ -313,7 +303,7 @@ 'requirement' => $this->Acl->canAccess('cerebrates', 'index'), ), array( - 'text' => __('List Taxii Servers'), + 'text' => __('TAXII Servers'), 'url' => $baseurl . '/TaxiiServers/index', 'requirement' => $this->Acl->canAccess('taxiiServers', 'index'), ), diff --git a/app/View/Elements/healthElements/files.ctp b/app/View/Elements/healthElements/files.ctp index 0221d8124..2d13b4767 100644 --- a/app/View/Elements/healthElements/files.ctp +++ b/app/View/Elements/healthElements/files.ctp @@ -1,3 +1,10 @@ +- :-
- :
- :
+ :
+ :
+ :
- :
+ :
$expectedValue): $colour = 'red'; @@ -24,7 +31,7 @@ endif; ?>
+
@@ -35,19 +42,10 @@ 1) { - $f['filesize'] = $f['filesize'] / 1024; - $sizeUnit = "KB"; - if (($f['filesize'] / 1024) > 1) { - $f['filesize'] = $f['filesize'] / 1024; - $sizeUnit = "MB"; - } - $f['filesize'] = round($f['filesize'], 1); - } ?> @@ -55,7 +53,7 @@ $ev): - if ($f['filename'] == $ev) echo h($ek) . "
"; + if ($f['filename'] == $ev) echo h($ek) . "
"; endforeach; else: echo __('N/A'); @@ -63,7 +61,7 @@ ?>- + = $humanReadableFilesize($f['filesize'], 1) ?> @@ -93,5 +91,4 @@ echo $this->Form->end(); endforeach; ?> - diff --git a/app/View/Events/automation.ctp b/app/View/Events/automation.ctp index 265b460c1..adba83846 100644 --- a/app/View/Events/automation.ctp +++ b/app/View/Events/automation.ctp @@ -64,7 +64,7 @@ "published" => __('Set whether published or unpublished events should be returned. Do not set the parameter if you want both.'), "enforceWarninglist" => __('Remove any attributes from the result that would cause a hit on a warninglist entry.'), "to_ids" => __('By default (0) all attributes are returned that match the other filter parameters, regardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. You can only use the special "exclude" setting to only return attributes that have the to_ids flag disabled.'), - "deleted" => __('If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using "only" as a parameter it will limit the returned data set to soft-deleted data only.'), + "deleted" => __('Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned.'), "includeEventUuid" => __('Instead of just including the event ID, also include the event UUID in each of the attributes.'), "event_timestamp" => __('Only return attributes from events that have received a modification after the given timestamp. The input can be a timestamp or a short-hand time description (7d or 24h for example). You can also pass a list with two values to set a time range (for example ["14d", "7d"]).'), "sgReferenceOnly" => __('If this flag is set, sharing group objects will not be included, instead only the sharing group ID is set.'), diff --git a/app/View/Events/export.ctp b/app/View/Events/export.ctp index 60e29ed79..77e3f09c9 100755 --- a/app/View/Events/export.ctp +++ b/app/View/Events/export.ctp @@ -3,6 +3,10 @@
+ ++ += __('This feature is disabled') ?>