From 7f16e27b25195d1c57df1952c28ebf4630020a17 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 27 Jul 2021 19:17:05 +0200 Subject: [PATCH 001/101] chg: [internal] Convert array to const --- app/Model/Attribute.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 77c9f3d61..fbe9559be 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -125,7 +125,7 @@ class Attribute extends AppModel 'hostname|port', ); - public $captureFields = array( + const CAPTURE_FIELDS = array( 'event_id', 'category', 'type', @@ -2885,7 +2885,7 @@ class Attribute extends AppModel $attribute['distribution'] = $this->defaultDistribution(); } $params = array( - 'fieldList' => $this->captureFields + 'fieldList' => self::CAPTURE_FIELDS, ); if (!empty($parentEvent)) { $params['parentEvent'] = $parentEvent; From 88e1c48227a5b4d2d60d988b0feb4fb9266b51a1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:32:20 +0200 Subject: [PATCH 002/101] Revert "chg: [logbehaviour] skipfields reverted to an array from a constant" This reverts commit 9d7da3103fb935c3c98c6c3c136e3a8f1a78614f. --- app/Model/Behavior/AuditLogBehavior.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index d085d50d4..f31362ab5 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -13,7 +13,7 @@ class AuditLogBehavior extends ModelBehavior private $enabled; // Hash is faster that in_array - private $skipFields = [ + const SKIP_FIELDS = [ 'id' => true, 'lastpushedid' => true, 'timestamp' => true, @@ -315,7 +315,7 @@ class AuditLogBehavior extends ModelBehavior $changedFields = []; $hasPrimaryField = isset($model->data[$model->alias][$model->primaryKey]); foreach ($model->data[$model->alias] as $key => $value) { - if (isset($this->skipFields[$key])) { + if (isset(self::SKIP_FIELDS[$key])) { continue; } if (!isset($dbFields[$key])) { From a2d6352bbb116e05fd01e069043b12b7f3ddd4e7 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:37:31 +0200 Subject: [PATCH 003/101] chg: [internal] Convert array in log to const --- app/Model/Log.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/Log.php b/app/Model/Log.php index 84da88eb0..0e9fa2097 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -3,14 +3,14 @@ App::uses('AppModel', 'Model'); class Log extends AppModel { - public $warningActions = array( + const WARNING_ACTIONS = array( 'warning', 'change_pw', 'login_fail', 'version_warning', 'auth_fail' ); - public $errorActions = array( + const ERROR_ACTIONS = array( 'error' ); public $validate = array( @@ -383,10 +383,10 @@ class Log extends AppModel if ($this->syslog) { $action = 'info'; if (isset($data['Log']['action'])) { - if (in_array($data['Log']['action'], $this->errorActions, true)) { + if (in_array($data['Log']['action'], self::ERROR_ACTIONS, true)) { $action = 'err'; } - if (in_array($data['Log']['action'], $this->warningActions, true)) { + if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) { $action = 'warning'; } } From f90edbb54029989bb9e353dc290ae7f7939414f6 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:43:19 +0200 Subject: [PATCH 004/101] chg: [internal] Convert array to const --- app/Model/GalaxyCluster.php | 4 ++-- app/Model/Organisation.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 132705880..42fad7d74 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -127,12 +127,12 @@ class GalaxyCluster extends AppModel } if (isset($result[$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) { if (isset($results[$k]['Org'])) { - $results[$k]['Org'] = $this->Org->genericMISPOrganisation; + $results[$k]['Org'] = Organisation::GENERIC_MISP_ORGANISATION; } } if (isset($result[$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) { if (isset($results[$k]['Orgc'])) { - $results[$k]['Orgc'] = $this->Org->genericMISPOrganisation; + $results[$k]['Orgc'] = Organisation::GENERIC_MISP_ORGANISATION; } } diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index 1f97788b3..a15399588 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -87,7 +87,7 @@ class Organisation extends AppModel 'User' => array('table' => 'users', 'fields' => array('org_id')) ); - public $genericMISPOrganisation = array( + const GENERIC_MISP_ORGANISATION = [ 'id' => '0', 'name' => 'MISP', 'date_created' => '', @@ -102,7 +102,7 @@ class Organisation extends AppModel 'local' => true, 'restricted_to_domain' => [], 'landingpage' => null - ); + ]; public function beforeValidate($options = array()) { From 37a5b3dc5f953fda88b902bbd66c014ec95e777f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:43:30 +0200 Subject: [PATCH 005/101] chg: [internal] Convert strings to const --- app/Model/Tag.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/Tag.php b/app/Model/Tag.php index 492e5c718..ac5e09493 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -79,8 +79,8 @@ class Tag extends AppModel ) ); - public $reGalaxy = '/misp-galaxy:[^:="]+="[^:="]+/i'; - public $reCustomGalaxy = '/misp-galaxy:[^:="]+="[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"/i'; + const RE_GALAXY = '/misp-galaxy:[^:="]+="[^:="]+/i'; + const RE_CUSTOM_GALAXY = '/misp-galaxy:[^:="]+="[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"/i'; private $tagOverrides = false; public function beforeValidate($options = array()) @@ -101,8 +101,8 @@ class Tag extends AppModel if (isset($tag['name']) && strlen($tag['name']) >= 255) { $tag['name'] = substr($tag['name'], 0, 255); } - $tag['is_galaxy'] = preg_match($this->reGalaxy, $tag['name']); - $tag['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $tag['name']); + $tag['is_galaxy'] = preg_match(self::RE_GALAXY, $tag['name']); + $tag['is_custom_galaxy'] = preg_match(self::RE_CUSTOM_GALAXY, $tag['name']); return true; } From e3375d547359963e11a4e1de914fb94064470566 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:48:24 +0200 Subject: [PATCH 006/101] chg: [internal] Convert array to const --- app/Model/Role.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/Role.php b/app/Model/Role.php index 3fe2414d2..d75345067 100644 --- a/app/Model/Role.php +++ b/app/Model/Role.php @@ -52,7 +52,7 @@ class Role extends AppModel 'permission' => "CASE WHEN (Role.perm_add AND Role.perm_modify AND Role.perm_publish) THEN '3' WHEN (Role.perm_add AND Role.perm_modify_org) THEN '2' WHEN (Role.perm_add) THEN '1' ELSE '0' END", ); - public $permissionConstants = array( + const PERMISSION_CONSTANTS = array( 'read_only' => 0, 'manage_own' => 1, 'manage_org' => 2, @@ -69,8 +69,8 @@ class Role extends AppModel } elseif (!is_numeric($this->data['Role']['permission'])) { // If a constant was passed via the API, convert it to the numeric value // For invalid entries, choose permission level 0 - if (isset($this->permissionConstants[$this->data['Role']['permission']])) { - $this->data['Role']['permission'] = $this->permissionConstants[$this->data['Role']['permission']]; + if (isset(self::PERMISSION_CONSTANTS[$this->data['Role']['permission']])) { + $this->data['Role']['permission'] = self::PERMISSION_CONSTANTS[$this->data['Role']['permission']]; } else { $this->data['Role']['permission'] = 0; } @@ -149,7 +149,7 @@ class Role extends AppModel unset($results[$key]['Role']['perm_full']); if (isset($results[$key]['Role']['permission'])) { $results[$key]['Role']['permission_description'] = - array_flip($this->permissionConstants)[$results[$key]['Role']['permission']]; + array_flip(self::PERMISSION_CONSTANTS)[$results[$key]['Role']['permission']]; } } } From a5cde82f5a31d5282da83a5f21a93d19fa0bade8 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 12:48:38 +0200 Subject: [PATCH 007/101] chg: [internal] Remove unused methods --- app/Model/WarninglistEntry.php | 6 ------ app/Model/WarninglistType.php | 6 ------ 2 files changed, 12 deletions(-) diff --git a/app/Model/WarninglistEntry.php b/app/Model/WarninglistEntry.php index 08b79c604..0bcaa4f52 100644 --- a/app/Model/WarninglistEntry.php +++ b/app/Model/WarninglistEntry.php @@ -23,10 +23,4 @@ class WarninglistEntry extends AppModel 'counterCache' => true ) ); - - public function beforeValidate($options = array()) - { - parent::beforeValidate(); - return true; - } } diff --git a/app/Model/WarninglistType.php b/app/Model/WarninglistType.php index c6aff3754..53a295e9d 100644 --- a/app/Model/WarninglistType.php +++ b/app/Model/WarninglistType.php @@ -19,10 +19,4 @@ class WarninglistType extends AppModel public $belongsTo = array( 'Warninglist' ); - - public function beforeValidate($options = array()) - { - parent::beforeValidate(); - return true; - } } From 98d75580a5804f28d1c9ac3f7826c83e5eb9c321 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 13:19:46 +0200 Subject: [PATCH 008/101] chg: [internal] Fix typo --- app/Lib/Tools/ComplexTypeTool.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index a543b8bb6..459542a00 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -3,7 +3,7 @@ require_once __DIR__ . '/TmpFileTool.php'; class ComplexTypeTool { - const REFANG_REGEX__TABLE = array( + const REFANG_REGEX_TABLE = array( array( 'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i', 'to' => 'http', @@ -35,7 +35,7 @@ class ComplexTypeTool public static function refangValue($value, $type) { - foreach (self::REFANG_REGEX__TABLE as $regex) { + foreach (self::REFANG_REGEX_TABLE as $regex) { if (in_array($type, $regex['types'], true)) { $value = preg_replace($regex['from'], $regex['to'], $value); } @@ -357,7 +357,7 @@ class ComplexTypeTool private function __refangInput($input) { $input['refanged'] = $input['raw']; - foreach (self::REFANG_REGEX__TABLE as $regex) { + foreach (self::REFANG_REGEX_TABLE as $regex) { $input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']); } $input['refanged'] = rtrim($input['refanged'], "."); From 19624a02f7bca614788e5c0ed6b3ba35e9446344 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 11 Aug 2021 13:23:16 +0200 Subject: [PATCH 009/101] chg: [internal] Convert array to const in ACLComponent --- app/Controller/Component/ACLComponent.php | 12 ++++----- app/Lib/Tools/ComplexTypeTool.php | 30 +++++++++++------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 440408f79..0a6ff5356 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -12,7 +12,7 @@ class ACLComponent extends Component // $action == array('OR' => array()) - any role in the array has access // $action == array('AND' => array()) - roles with all permissions in the array have access // If we add any new functionality to MISP and we don't add it to this list, it will only be visible to site admins. - private $__aclList = array( + const ACL_LIST = array( '*' => array( 'blackhole' => array(), 'debugACL' => array(), @@ -907,10 +907,10 @@ class ACLComponent extends Component if ($user && $user['Role']['perm_site_admin']) { return true; } - if (!isset($this->__aclList[$controller])) { + if (!isset(self::ACL_LIST[$controller])) { throw new NotFoundException('Invalid controller.'); } - $controllerAclList = array_change_key_case($this->__aclList[$controller]); + $controllerAclList = array_change_key_case(self::ACL_LIST[$controller]); if (!empty($controllerAclList[$action])) { $rules = $controllerAclList[$action]; if (in_array('*', $rules, true)) { @@ -992,8 +992,8 @@ class ACLComponent extends Component $missing = array(); foreach ($results as $controller => $functions) { foreach ($functions as $function) { - if (!isset($this->__aclList[$controller]) - || !in_array($function, array_keys($this->__aclList[$controller]))) { + if (!isset(self::ACL_LIST[$controller]) + || !in_array($function, array_keys(self::ACL_LIST[$controller]))) { $missing[$controller][] = $function; } } @@ -1027,7 +1027,7 @@ class ACLComponent extends Component { $result = array(); $fakeUser = ['Role' => $role, 'org_id' => Configure::read('MISP.host_org_id')]; - foreach ($this->__aclList as $controller => $actions) { + foreach (self::ACL_LIST as $controller => $actions) { $controllerNames = Inflector::variable($controller) === Inflector::underscore($controller) ? array(Inflector::variable($controller)) : array(Inflector::variable($controller), Inflector::underscore($controller)); diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index 459542a00..8d5145eef 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -31,6 +31,18 @@ class ComplexTypeTool ) ); + const HEX_HASH_TYPES = array( + 32 => array('single' => array('md5', 'imphash', 'x509-fingerprint-md5'), 'composite' => array('filename|md5', 'filename|imphash')), + 40 => array('single' => array('sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'), 'composite' => array('filename|sha1', 'filename|pehash')), + 56 => array('single' => array('sha224', 'sha512/224'), 'composite' => array('filename|sha224', 'filename|sha512/224')), + 64 => array('single' => array('sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'), 'composite' => array('filename|sha256', 'filename|authentihash', 'filename|sha512/256')), + 96 => array('single' => array('sha384'), 'composite' => array('filename|sha384')), + 128 => array('single' => array('sha512'), 'composite' => array('filename|sha512')) + ); + + // algorithms to run through in order, without Hashes that are checked separately + const CHECKS = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC'); + private $__tlds = null; public static function refangValue($value, $type) @@ -223,18 +235,6 @@ class ComplexTypeTool return array_values($resultArray); } - private $__hexHashTypes = array( - 32 => array('single' => array('md5', 'imphash', 'x509-fingerprint-md5'), 'composite' => array('filename|md5', 'filename|imphash')), - 40 => array('single' => array('sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'), 'composite' => array('filename|sha1', 'filename|pehash')), - 56 => array('single' => array('sha224', 'sha512/224'), 'composite' => array('filename|sha224', 'filename|sha512/224')), - 64 => array('single' => array('sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'), 'composite' => array('filename|sha256', 'filename|authentihash', 'filename|sha512/256')), - 96 => array('single' => array('sha384'), 'composite' => array('filename|sha384')), - 128 => array('single' => array('sha512'), 'composite' => array('filename|sha512')) - ); - - // algorithms to run through in order, without Hashes that are checked separately - private $__checks = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC'); - /** * @param string $raw_input Trimmed value * @return array|false @@ -263,7 +263,7 @@ class ComplexTypeTool $input = $this->__refangInput($input); $input = $this->__extractPort($input); - foreach ($this->__checks as $check) { + foreach (self::CHECKS as $check) { $result = $this->{'__checkFor' . $check}($input); if ($result) { return $result; @@ -511,8 +511,8 @@ class ComplexTypeTool private function __resolveHash($value) { $strlen = strlen($value); - if (isset($this->__hexHashTypes[$strlen]) && ctype_xdigit($value)) { - return $this->__hexHashTypes[$strlen]; + if (isset(self::HEX_HASH_TYPES[$strlen]) && ctype_xdigit($value)) { + return self::HEX_HASH_TYPES[$strlen]; } return false; } From c9e4f326eb946e6aabcd2d22544423fb903a93d0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 3 Oct 2021 14:48:23 +0200 Subject: [PATCH 010/101] chg: [internal] Convert array to const in RestResponseComponent --- app/Controller/Component/RestResponseComponent.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 6dd49a708..e19138a25 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -9,7 +9,7 @@ class RestResponseComponent extends Component public $headers = array(); - private $__convertActionToMessage = array( + const CONVERT_ACTION_TO_MESSAGE = array( 'SharingGroup' => array( 'addOrg' => 'add Organisation to', 'removeOrg' => 'remove Organisation from', @@ -473,8 +473,8 @@ class RestResponseComponent extends Component $response = array(); $action = $this->__dissectAdminRouting($action); $stringifiedAction = $action['action']; - if (isset($this->__convertActionToMessage[$controller][$action['action']])) { - $stringifiedAction = $this->__convertActionToMessage[$controller][$action['action']]; + if (isset(self::CONVERT_ACTION_TO_MESSAGE[$controller][$action['action']])) { + $stringifiedAction = self::CONVERT_ACTION_TO_MESSAGE[$controller][$action['action']]; } $response['saved'] = false; $response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller); @@ -640,7 +640,7 @@ class RestResponseComponent extends Component private function __dissectAdminRouting($action) { $admin = false; - if (strlen($action) > 6 && substr($action, 0, 6) == 'admin_') { + if (strlen($action) > 6 && substr($action, 0, 6) === 'admin_') { $action = substr($action, 6); $admin = true; } From 48f9834098de85defdd16baaf7bf7722544832bc Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 4 Oct 2021 11:01:58 +0200 Subject: [PATCH 011/101] chg: [internal] Convert array to const in Warninglist --- app/Model/Warninglist.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Model/Warninglist.php b/app/Model/Warninglist.php index 18db7f95f..6276964d1 100644 --- a/app/Model/Warninglist.php +++ b/app/Model/Warninglist.php @@ -53,7 +53,7 @@ class Warninglist extends AppModel ) ); - private $__tlds = array( + const TLDS = array( 'TLDs as known by IANA' ); @@ -725,7 +725,7 @@ class Warninglist extends AppModel public function fetchTLDLists() { $tldLists = $this->find('column', array( - 'conditions' => array('Warninglist.name' => $this->__tlds), + 'conditions' => array('Warninglist.name' => self::TLDS), 'fields' => array('Warninglist.id') )); $tlds = array(); @@ -769,7 +769,7 @@ class Warninglist extends AppModel public function missingTldLists() { $missingTldLists = array(); - foreach ($this->__tlds as $tldList) { + foreach (self::TLDS as $tldList) { $temp = $this->find('first', array( 'recursive' => -1, 'conditions' => array('Warninglist.name' => $tldList), From 03f0bc3c7868f7162740a75d07769681e5b3b5db Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 15 Oct 2021 21:32:13 +0200 Subject: [PATCH 012/101] chg: [internal] Convert array to const in QueryTool --- app/Lib/Tools/QueryTool.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/Lib/Tools/QueryTool.php b/app/Lib/Tools/QueryTool.php index 003b97261..b4451e441 100644 --- a/app/Lib/Tools/QueryTool.php +++ b/app/Lib/Tools/QueryTool.php @@ -2,24 +2,24 @@ class QueryTool { - private $__pdoMap = array( - 'integer' => PDO::PARAM_INT, - 'float' => PDO::PARAM_STR, - 'boolean' => PDO::PARAM_BOOL, - 'string' => PDO::PARAM_STR, - 'text' => PDO::PARAM_STR + const PDO_MAP = array( + 'integer' => PDO::PARAM_INT, + 'float' => PDO::PARAM_STR, + 'boolean' => PDO::PARAM_BOOL, + 'string' => PDO::PARAM_STR, + 'text' => PDO::PARAM_STR, ); public function quickDelete($table, $field, $value, $model) { $db = $model->getDataSource(); $connection = $db->getConnection(); - if ($db->config['datasource'] == 'Database/Mysql' || $db->config['datasource'] == 'Database/MysqlObserver') { + if ($db->config['datasource'] === 'Database/Mysql' || $db->config['datasource'] === 'Database/MysqlObserver') { $query = $connection->prepare('DELETE FROM ' . $table . ' WHERE ' . $field . ' = :value'); } elseif ($db->config['datasource'] == 'Database/Postgres' ) { $query = $connection->prepare('DELETE FROM "' . $table . '" WHERE "' . $field . '" = :value'); } - $query->bindValue(':value', $value, $this->__pdoMap[$db->introspectType($value)]); + $query->bindValue(':value', $value, self::PDO_MAP[$db->introspectType($value)]); $query->execute(); } } From 9ab3998b9d6fd08cd8727375a8ffdf8cf013f4e0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 15 Oct 2021 21:36:30 +0200 Subject: [PATCH 013/101] chg: [internal] Remove unused variable --- app/Model/User.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Model/User.php b/app/Model/User.php index 6b5a72312..0f7719fba 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -19,8 +19,6 @@ class User extends AppModel { public $displayField = 'email'; - public $orgField = array('Organisation', 'name'); - public $validate = array( 'role_id' => array( 'numeric' => array( From ab1e79646e9142cdbed3bd9c497c28fe46909cf7 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 15 Oct 2021 21:43:49 +0200 Subject: [PATCH 014/101] new: [internal] Use pubToZmq to check if publish to ZMQ --- app/Model/Attribute.php | 4 ++-- app/Model/Event.php | 4 ++-- app/Model/MispObject.php | 13 +++++-------- app/Model/Organisation.php | 2 +- app/Model/User.php | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index fbe9559be..ae731f91c 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -480,7 +480,7 @@ class Attribute extends AppModel $result = $this->saveAttachment($attribute); } } - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable'); + $pubToZmq = $this->pubToZmq('attribute'); $kafkaTopic = $this->kafkaTopic('attribute'); if ($pubToZmq || $kafkaTopic) { $attributeForPublish = $this->fetchAttribute($this->id); @@ -532,7 +532,7 @@ class Attribute extends AppModel // update correlation.. $this->Correlation->beforeSaveCorrelation($this->data['Attribute']); if (!empty($this->data['Attribute']['id'])) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) { + if ($this->pubToZmq('attribute')) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->attribute_save($this->data, 'delete'); } diff --git a/app/Model/Event.php b/app/Model/Event.php index cf18576b2..3a31aaa69 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -396,7 +396,7 @@ class Event extends AppModel } if (!empty($this->data['Event']['id'])) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + if ($this->pubToZmq('event')) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete'); } @@ -523,7 +523,7 @@ class Event extends AppModel $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]); } } - if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { + if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && $this->pubToZmq('event')) { $pubSubTool = $this->getPubSubTool(); $eventForZmq = $this->quickFetchEvent($event['id']); if (!empty($event)) { diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 4c45fcd08..aca87f5a0 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -300,9 +300,7 @@ class MispObject extends AppModel public function afterSave($created, $options = array()) { - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && - Configure::read('Plugin.ZeroMQ_object_notifications_enable') && - empty($this->data['Object']['skip_zmq']); + $pubToZmq = $this->pubToZmq('object') && empty($this->data['Object']['skip_zmq']); $kafkaTopic = $this->kafkaTopic('object'); $pubToKafka = $kafkaTopic && empty($this->data['Object']['skip_kafka']); if ($pubToZmq || $pubToKafka) { @@ -329,10 +327,9 @@ class MispObject extends AppModel public function beforeDelete($cascade = true) { if (!empty($this->data['Object']['id'])) { - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable'); - $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic'); - $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_notifications_enable') && !empty($kafkaTopic); - if ($pubToZmq || $pubToKafka) { + $pubToZmq = $this->pubToZmq('object'); + $kafkaTopic = $this->kafkaTopic('object'); + if ($pubToZmq || $kafkaTopic) { $object = $this->find('first', array( 'recursive' => -1, 'conditions' => array('Object.id' => $this->data['Object']['id']) @@ -341,7 +338,7 @@ class MispObject extends AppModel $pubSubTool = $this->getPubSubTool(); $pubSubTool->object_save($object, 'delete'); } - if ($pubToKafka) { + if ($kafkaTopic) { $kafkaPubTool = $this->getKafkaPubTool(); $kafkaPubTool->publishJson($kafkaTopic, $object, 'delete'); } diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index a15399588..0f9a8b80d 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -150,7 +150,7 @@ class Organisation extends AppModel public function afterSave($created, $options = array()) { - if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_organisation_notifications_enable')) { + if ($this->pubToZmq('organisation')) { $pubSubTool = $this->getPubSubTool(); $pubSubTool->modified($this->data, 'organisation'); } diff --git a/app/Model/User.php b/app/Model/User.php index 0f7719fba..bb8e14ca2 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -276,7 +276,7 @@ class User extends AppModel public function afterSave($created, $options = array()) { - $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable'); + $pubToZmq = $this->pubToZmq('user'); $kafkaTopic = $this->kafkaTopic('user'); if ($pubToZmq || $kafkaTopic) { if (!empty($this->data)) { From d57e4fd9f85222c180bcfea519f8b0b7ee7f87fd Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 18 Oct 2021 09:58:50 +0200 Subject: [PATCH 015/101] fix: [auditlog] Array converted to const --- app/Model/Behavior/AuditLogBehavior.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index f31362ab5..ebe971623 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -82,7 +82,7 @@ class AuditLogBehavior extends ModelBehavior $fieldToFetch = []; if (!empty($options['fieldList'])) { foreach ($options['fieldList'] as $field) { - if (!isset($this->skipFields[$field])) { + if (!isset(self::SKIP_FIELDS[$field])) { $fieldToFetch[] = $field; } } From dce21e7c3a63dc78ef3f28010b8ca61836ae41cc Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 17 Nov 2021 16:42:05 +0100 Subject: [PATCH 016/101] chg: allow change disable_correlation in mass edit attributes --- app/Controller/AttributesController.php | 8 +++++++- .../Attributes/ajax/attributeEditMassForm.ctp | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 7a019d06d..55bedece0 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1397,7 +1397,7 @@ class AttributesController extends AppController || ($clusters_ids_remove === null || count($clusters_ids_remove) > 0) || ($clusters_ids_add === null || count($clusters_ids_add) > 0); - $changeInAttribute = ($this->request->data['Attribute']['to_ids'] != 2) || ($this->request->data['Attribute']['distribution'] != 6) || ($this->request->data['Attribute']['comment'] != null); + $changeInAttribute = ($this->request->data['Attribute']['to_ids'] != 2) || ($this->request->data['Attribute']['distribution'] != 6) || ($this->request->data['Attribute']['comment'] != null) || ($this->request->data['Attribute']['disable_correlation'] != 2); if (!$changeInAttribute && !$changeInTagOrCluster) { return new CakeResponse(array('body'=> json_encode(array('saved' => true)), 'status' => 200, 'type' => 'json')); @@ -1435,6 +1435,12 @@ class AttributesController extends AppController } } + if ($this->request->data['Attribute']['disable_correlation'] != 2) { + foreach ($attributes as $key => $attribute) { + $attributes[$key]['Attribute']['disable_correlation'] = $this->request->data['Attribute']['disable_correlation'] === '0' ? false : true; + } + } + $date = new DateTime(); $timestamp = $date->getTimestamp(); foreach ($attributes as $key => $attribute) { diff --git a/app/View/Attributes/ajax/attributeEditMassForm.ctp b/app/View/Attributes/ajax/attributeEditMassForm.ctp index 9718cbe4e..3a1b90ad4 100644 --- a/app/View/Attributes/ajax/attributeEditMassForm.ctp +++ b/app/View/Attributes/ajax/attributeEditMassForm.ctp @@ -27,17 +27,29 @@ ?> '; echo $this->Form->input('to_ids', array( - 'options' => array(__('No'), __('Yes'), __('Do not alter current settings')), - 'data-content' => isset($attrDescriptions['signature']['formdesc']) ? $attrDescriptions['signature']['formdesc'] : $attrDescriptions['signature']['desc'], - 'label' => __('For Intrusion Detection System'), - 'selected' => 2, + 'options' => array(__('No'), __('Yes'), __('Do not alter current settings')), + 'data-content' => isset($attrDescriptions['signature']['formdesc']) ? $attrDescriptions['signature']['formdesc'] : $attrDescriptions['signature']['desc'], + 'label' => __('For Intrusion Detection System'), + 'selected' => 2 )); + echo '
'; echo $this->Form->input('is_proposal', array( 'type' => 'checkbox', 'label' => __('Create proposals'), 'checked' => false )); + echo '
'; + echo $this->Form->input('disable_correlation', array( + 'label' => __('Correlations'), + 'options' => [ + '2' => __('Do not alter current settings'), + '0' => __('Enable correlations'), + '1' => __('Disable correlations') + ], + 'selected' => '2' + )); ?>
From 0a941bd7f36a4209cd2b36329b772a8f9095774a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 11:39:10 +0100 Subject: [PATCH 017/101] fix: [internal] Remove UrlCache --- app/Config/bootstrap.default.php | 1 - app/Plugin/UrlCache/keep.txt | 1 + app/View/Helper/AppHelper.php | 15 +++++---------- travis/bootstrap.php | 1 - 4 files changed, 6 insertions(+), 12 deletions(-) create mode 100644 app/Plugin/UrlCache/keep.txt diff --git a/app/Config/bootstrap.default.php b/app/Config/bootstrap.default.php index 310a533e1..13fb9d28a 100644 --- a/app/Config/bootstrap.default.php +++ b/app/Config/bootstrap.default.php @@ -132,7 +132,6 @@ if (Configure::read('MISP.baseurl')) { CakePlugin::load('SysLog'); CakePlugin::load('Assets'); // having Logable CakePlugin::load('SysLogLogable'); -CakePlugin::load('UrlCache'); /** * Uncomment the following line to enable client SSL certificate authentication. diff --git a/app/Plugin/UrlCache/keep.txt b/app/Plugin/UrlCache/keep.txt new file mode 100644 index 000000000..2f798dbe5 --- /dev/null +++ b/app/Plugin/UrlCache/keep.txt @@ -0,0 +1 @@ +This plugin is not used anymore, but it is here to keep backward compatibility. diff --git a/app/View/Helper/AppHelper.php b/app/View/Helper/AppHelper.php index 4841ce742..e026f8473 100644 --- a/app/View/Helper/AppHelper.php +++ b/app/View/Helper/AppHelper.php @@ -20,9 +20,7 @@ * @license MIT License (http://www.opensource.org/licenses/mit-license.php) */ -App::uses('UrlCacheAppHelper', 'UrlCache.View/Helper'); - - +App::uses('Helper', 'View'); /** * Application helper @@ -32,16 +30,13 @@ App::uses('UrlCacheAppHelper', 'UrlCache.View/Helper'); * * @package app.View.Helper */ -class AppHelper extends UrlCacheAppHelper { - - public function afterLayout($layoutFile) { - } - - public function url($url = null, $full = false) { +class AppHelper extends Helper +{ + public function url($url = null, $full = false) + { if (is_array($url) && !isset($url['admin'])) { $url['admin'] = false; } return parent::url($url, $full); - } } diff --git a/travis/bootstrap.php b/travis/bootstrap.php index 630c53205..ec050e512 100644 --- a/travis/bootstrap.php +++ b/travis/bootstrap.php @@ -106,7 +106,6 @@ if (!Configure::read('MISP.baseurl')) { CakePlugin::load('SysLog'); CakePlugin::load('Assets'); // having Logable CakePlugin::load('SysLogLogable'); -CakePlugin::load('UrlCache'); /** * Uncomment the following line to enable client SSL certificate authentication. From 531676620585454c6172019719e7b3ff13235864 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Mon, 22 Nov 2021 09:49:34 +0100 Subject: [PATCH 018/101] fix: ServerShell fails if SimpleBackgroundJobs config does not exists --- app/Console/Command/ServerShell.php | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 183ff901c..21f8272b8 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -12,17 +12,8 @@ require_once 'AppShell.php'; */ class ServerShell extends AppShell { - /** @var BackgroundJobsTool */ - private $BackgroundJobsTool; - public $uses = array('Server', 'Task', 'Job', 'User', 'Feed'); - public function initialize(): void - { - parent::initialize(); - $this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs')); - } - public function list() { $servers = $this->Server->find('all', [ @@ -89,7 +80,7 @@ class ServerShell extends AppShell foreach ($servers as $serverId => $serverName) { - $backgroundJobId = $this->BackgroundJobsTool->enqueue( + $backgroundJobId = $this->Server->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ @@ -199,7 +190,7 @@ class ServerShell extends AppShell foreach ($servers as $serverId => $serverName) { - $jobId = $this->BackgroundJobsTool->enqueue( + $jobId = $this->Server->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ @@ -319,7 +310,7 @@ class ServerShell extends AppShell foreach ($servers as $serverId => $serverName) { - $jobId = $this->BackgroundJobsTool->enqueue( + $jobId = $this->Server->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ From 06107ee62202e77bde8c7d43626925727a675d65 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 12:44:33 +0100 Subject: [PATCH 019/101] fix: [internal] User ProcessTool for selfTest --- app/Model/Server.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 67f89a7d3..858d0ba5e 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3758,7 +3758,11 @@ class Server extends AppModel ]; } if (is_readable(APP . '/files/scripts/selftest.php')) { - $execResult = exec('php ' . APP . '/files/scripts/selftest.php ' . escapeshellarg(json_encode(array_keys($extensions)))); + try { + $execResult = ProcessTool::execute(['php', APP . '/files/scripts/selftest.php', json_encode(array_keys($extensions))]); + } catch (Exception $e) { + // pass + } if (!empty($execResult)) { $execResult = $this->jsonDecode($execResult); $results['cli']['phpversion'] = $execResult['phpversion']; From debb1428ea78655108c62e905ce691805c8298b1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 12:48:38 +0100 Subject: [PATCH 020/101] chg: [securityAudit] Show warning if encryption key is not set --- app/Lib/Tools/SecurityAudit.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index 6d9080e40..d71a77ae4 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -43,6 +43,10 @@ class SecurityAudit $output['Database'][] = ['warning', __('Database password is too short, should be at least %s chars long.', self::STRONG_PASSWORD_LENGTH)]; } + if (!Configure::read('Security.encryption_key')) { + $output['Database'][] = ['warning', __('Sensitive information like keys to remote server are stored in database unencrypted. Set `Security.encryption_key` to encrypt these values.')]; + } + $passwordPolicyLength = Configure::read('Security.password_policy_length') ?: $server->serverSettings['Security']['password_policy_length']['value']; if ($passwordPolicyLength < 8) { $output['Password'][] = ['error', __('Minimum password length is set to %s, it is highly advised to increase it.', $passwordPolicyLength)]; From 1bf78dc969351bec1314460f46d779b7683dbc5f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 13:02:58 +0100 Subject: [PATCH 021/101] fix: [internal] Fix checking if system is Linux --- app/Lib/Tools/SecurityAudit.php | 53 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index d71a77ae4..34a3436dc 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -216,7 +216,7 @@ class SecurityAudit if (version_compare(PHP_VERSION, '7.3.0', '<')) { $output['PHP'][] = [ 'warning', - __('PHP version %s is not supported anymore. It can be still supported by your distribution. ', PHP_VERSION), + __('PHP version %s is not supported anymore. It can be still supported by your distribution.', PHP_VERSION), 'https://www.php.net/supported-versions.php' ]; } else if (version_compare(PHP_VERSION, '7.4.0', '<')) { @@ -260,7 +260,7 @@ class SecurityAudit ]; } - $this->system($server, $output); + $this->system($output); return $output; } @@ -373,7 +373,7 @@ class SecurityAudit } } - private function system(Server $server, array &$output) + private function system(array &$output) { $kernelBuildTime = $this->getKernelBuild(); if ($kernelBuildTime) { @@ -426,27 +426,30 @@ class SecurityAudit } catch (Exception $e) { } - $ubuntuVersion = $this->getUbuntuVersion(); - if ($ubuntuVersion) { - if (in_array($ubuntuVersion, ['14.04', '19.10'])) { - $output['System'][] = [ - 'warning', - __('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $ubuntuVersion), - 'https://endoflife.date/ubuntu', - ]; - } else if (in_array($ubuntuVersion, ['16.04'])) { - $output['System'][] = [ - 'hint', - __('You are using Ubuntu %s. This version will be not supported after 02 Apr 2021.', $ubuntuVersion), - 'https://endoflife.date/ubuntu', - ]; + $linuxVersion = $this->getLinuxVersion(); + if ($linuxVersion) { + list($name, $version) = $linuxVersion; + if ($name === 'Ubuntu') { + if (in_array($version, ['14.04', '19.10'], true)) { + $output['System'][] = [ + 'warning', + __('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $version), + 'https://endoflife.date/ubuntu', + ]; + } else if (in_array($version, ['16.04'], true)) { + $output['System'][] = [ + 'hint', + __('You are using Ubuntu %s. This version will be not supported after 02 Apr 2021.', $version), + 'https://endoflife.date/ubuntu', + ]; + } } } } private function getKernelBuild() { - if (!php_uname('s') !== 'Linux') { + if (PHP_OS !== 'Linux') { return false; } @@ -458,9 +461,9 @@ class SecurityAudit return new DateTime($buildTime); } - private function getUbuntuVersion() + private function getLinuxVersion() { - if (!php_uname('s') !== 'Linux') { + if (PHP_OS !== 'Linux') { return false; } if (!is_readable('/etc/os-release')) { @@ -474,16 +477,10 @@ class SecurityAudit if ($parsed === false) { return false; } - if (!isset($parsed['NAME'])) { + if (!isset($parsed['NAME']) || !isset($parsed['VERSION_ID'])) { return false; } - if ($parsed['NAME'] !== 'Ubuntu') { - return false; - } - if (!isset($parsed['VERSION_ID'])) { - return false; - } - return $parsed['VERSION_ID']; + return [$parsed['NAME'], $parsed['VERSION_ID']]; } /** From 4663c1fff725579885e4467b450b247dce9f6ab8 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 14:55:07 +0100 Subject: [PATCH 022/101] new: [bg] Support unix socket for supervisord --- app/Lib/Tools/BackgroundJobsTool.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 42977ad56..9f2455c4a 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -605,6 +605,7 @@ class BackgroundJobsTool /** * @return \Supervisor\Supervisor + * @throws Exception */ private function createSupervisorConnection(): \Supervisor\Supervisor { @@ -618,10 +619,19 @@ class BackgroundJobsTool ]; } + $host = null; + if (substr($this->settings['supervisor_host'], 0, 5) === 'unix:') { + if (!defined('CURLOPT_UNIX_SOCKET_PATH')) { + throw new Exception("For unix socket connection, cURL is required."); + } + $httpOptions['curl'][CURLOPT_UNIX_SOCKET_PATH] = substr($this->settings['supervisor_host'], 5); + $host = 'localhost'; + } + $client = new \fXmlRpc\Client( sprintf( 'http://%s:%s/RPC2', - $this->settings['supervisor_host'], + $host ?: $this->settings['supervisor_host'], $this->settings['supervisor_port'] ), new \fXmlRpc\Transport\HttpAdapterTransport( From bd99d4866f8adfdae27f5058e6492bfb433bf7f8 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 18:30:20 +0100 Subject: [PATCH 023/101] chg: [diagnostics] Check also MISP.attachments_dir and MISP.tmpdir folders --- app/Model/Server.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/Model/Server.php b/app/Model/Server.php index 858d0ba5e..954eebfb9 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3155,6 +3155,17 @@ class Server extends AppModel APP . 'tmp' . DS . 'logs' => 0, APP . 'tmp' . DS . 'bro' => 0, ); + + $attachmentDir = Configure::read('MISP.attachments_dir'); + if ($attachmentDir && !isset($writeableDirs[$attachmentDir])) { + $writeableDirs[$attachmentDir] = 0; + } + + $tmpDir = Configure::read('MISP.tmpdir'); + if ($tmpDir && !isset($writeableDirs[$tmpDir])) { + $writeableDirs[$tmpDir] = 0; + } + foreach ($writeableDirs as $path => &$error) { if (!file_exists($path)) { // Try to create directory if not exists From 5cb4aa96971056546872f09359584fcb5034d464 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 20:10:04 +0100 Subject: [PATCH 024/101] chg: [process] No need to close pipes --- app/Lib/Tools/ProcessTool.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Lib/Tools/ProcessTool.php b/app/Lib/Tools/ProcessTool.php index 0a0d537dd..687b2058f 100644 --- a/app/Lib/Tools/ProcessTool.php +++ b/app/Lib/Tools/ProcessTool.php @@ -73,13 +73,11 @@ class ProcessTool $commandForException = self::commandFormat($command); throw new Exception("Could not get STDOUT of command '$commandForException'."); } - fclose($pipes[1]); if ($stderrToFile) { $stderr = null; } else { $stderr = stream_get_contents($pipes[2]); - fclose($pipes[2]); } $returnCode = proc_close($process); From ac59e1ee54d3c4fb234dbed5975829e095b158d9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 20:55:47 +0100 Subject: [PATCH 025/101] chg: [bg] Move logging to one place --- app/Console/Command/StartWorkerShell.php | 37 ++++++++++++++----- .../Tools/BackgroundJobs/BackgroundJob.php | 19 ++-------- app/Lib/Tools/BackgroundJobsTool.php | 21 ----------- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php index 7be815aa4..42b180a3b 100644 --- a/app/Console/Command/StartWorkerShell.php +++ b/app/Console/Command/StartWorkerShell.php @@ -62,21 +62,38 @@ class StartWorkerShell extends AppShell $this->checkMaxExecutionTime(); $job = $this->BackgroundJobsTool->dequeue($this->worker->queue()); - if ($job) { - CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}..."); - - try { - $this->BackgroundJobsTool->run($job); - } catch (Exception $exception) { - CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}"); - $job->setStatus(BackgroundJob::STATUS_FAILED); - $this->BackgroundJobsTool->update($job); - } + $this->runJob($job); } } } + /** + * @param BackgroundJob $job + */ + private function runJob(BackgroundJob $job) + { + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}..."); + + try { + $job->setStatus(BackgroundJob::STATUS_RUNNING); + CakeLog::info("[JOB ID: {$job->id()}] - started."); + $this->BackgroundJobsTool->update($job); + + $job->run(); + + if ($job->status() === BackgroundJob::STATUS_COMPLETED) { + CakeLog::info("[JOB ID: {$job->id()}] - completed."); + } else { + CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}."); + } + } catch (Exception $exception) { + CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}"); + $job->setStatus(BackgroundJob::STATUS_FAILED); + } + $this->BackgroundJobsTool->update($job); + } + /** * Checks if worker maximum execution time is reached, and exits if so. * diff --git a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php index 5bb39008d..758e55289 100644 --- a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php +++ b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php @@ -66,10 +66,8 @@ class BackgroundJob implements JsonSerializable /** * Run the job command - * - * @return self */ - public function run(): self + public function run(): void { $descriptorSpec = [ 1 => ["pipe", "w"], // stdout @@ -90,28 +88,17 @@ class BackgroundJob implements JsonSerializable ['BACKGROUND_JOB_ID' => $this->id] ); - $stdout = stream_get_contents($pipes[1]); - $this->setOutput($stdout); - fclose($pipes[1]); - - $stderr = stream_get_contents($pipes[2]); - $this->setError($stderr); - fclose($pipes[2]); + $this->output = stream_get_contents($pipes[1]); + $this->error = stream_get_contents($pipes[2]); $this->returnCode = proc_close($process); if ($this->returnCode === 0 && empty($stderr)) { $this->setStatus(BackgroundJob::STATUS_COMPLETED); $this->setProgress(100); - - CakeLog::info("[JOB ID: {$this->id()}] - completed."); } else { $this->setStatus(BackgroundJob::STATUS_FAILED); - - CakeLog::error("[JOB ID: {$this->id()}] - failed with error code {$this->returnCode}. STDERR: {$stderr}. STDOUT: {$stdout}."); } - - return $this; } public function jsonSerialize(): array diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 9f2455c4a..1687d9c3e 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -337,27 +337,6 @@ class BackgroundJobsTool ); } - /** - * Run job - * - * @param BackgroundJob $job - * - * @return integer Process return code. - */ - public function run(BackgroundJob $job): int - { - $job->setStatus(BackgroundJob::STATUS_RUNNING); - CakeLog::info("[JOB ID: {$job->id()}] - started."); - - $this->update($job); - - $job = $job->run(); - - $this->update($job); - - return $job->returnCode(); - } - /** * Start worker by name * From eb8955b220049c4e0842889083d1c1e83b94c368 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 21:57:25 +0100 Subject: [PATCH 026/101] new: [securityAudit] Check if xdebug is enabled --- app/Lib/Tools/SecurityAudit.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index 34a3436dc..e2c1e669b 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -226,6 +226,14 @@ class SecurityAudit 'https://www.php.net/supported-versions.php' ]; } + + if (extension_loaded('xdebug')) { + $output['PHP'][] = [ + 'error', + __('The xdebug extension can reveal code and data to an attacker.'), + ]; + } + if (ini_get('session.use_strict_mode') != 1) { $output['PHP'][] = [ 'warning', From acd88ade13acc788fb47a3470f9ba54ccbcbb176 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 22:19:35 +0100 Subject: [PATCH 027/101] chg: [internal] Use ProcessTool in Exports --- app/Lib/Export/OpendataExport.php | 18 +++++++++++------- app/Lib/Export/YaraExport.php | 22 ++++++++++++++++------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/app/Lib/Export/OpendataExport.php b/app/Lib/Export/OpendataExport.php index dd66b205f..2b0099d1f 100644 --- a/app/Lib/Export/OpendataExport.php +++ b/app/Lib/Export/OpendataExport.php @@ -1,4 +1,5 @@ getPythonVersion() . ' ' . $this->__scripts_dir . $this->__script_name; + $cmd = [ProcessTool::pythonBin(), $this->__scripts_dir . $this->__script_name]; if (!empty($this->__auth)) { $this->__request_object['auth'] = $this->__auth; } - if ($this->__search){ + if ($this->__search) { return $this->__search_query($cmd); } return $this->__delete ? $this->__delete_query($cmd) : $this->__add_query($cmd); @@ -83,7 +83,7 @@ class OpendataExport return ''; } - private function __add_query($cmd) + private function __add_query(array $cmd) { unset($this->__default_filters['returnFormat']); $body = json_encode($this->__default_filters); @@ -95,7 +95,9 @@ class OpendataExport $this->__request_object['setup'] = $setupFilename; $this->__request_object['misp_url'] = $this->__url; $commandFile = $this->__generateCommandFile(); - $results = shell_exec($cmd . ' --query_data ' . $commandFile); + $cmd[] = '--query_data'; + $cmd[] = $commandFile; + $results = ProcessTool::execute($cmd); unlink($commandFile); unlink($bodyFilename); unlink($setupFilename); @@ -128,13 +130,15 @@ class OpendataExport return $this->__simple_query($cmd); } - private function __simple_query($cmd) + private function __simple_query(array $cmd) { if (!empty($this->__setup['resources'])) { $this->__request_object['search'] = $this->__setup['resources']; } $commandFile = $this->__generateCommandFile(); - $results = shell_exec($cmd . ' --query_data ' . $commandFile); + $cmd[] = '--query_data'; + $cmd[] = $commandFile; + $results = ProcessTool::execute($cmd); unlink($commandFile); return $results; } diff --git a/app/Lib/Export/YaraExport.php b/app/Lib/Export/YaraExport.php index cc3658e63..3ace9d726 100644 --- a/app/Lib/Export/YaraExport.php +++ b/app/Lib/Export/YaraExport.php @@ -1,12 +1,12 @@ >' . APP . 'tmp/logs/yara_export.log'; private $__n_attributes = 0; private $__MAX_n_attributes = 15000; private $__yara_file_gen = null; @@ -35,7 +35,7 @@ class YaraExport } $this->__initialize_yara_file(); $this->__initialize_misp_file($options); - if($options['returnFormat'] === 'yara-json'){ + if($options['returnFormat'] === 'yara-json') { $this->__raw_mode = false; } return ''; @@ -132,10 +132,20 @@ class YaraExport $in = $this->__curr_input_file->path; $out1 = $this->__yara_file_gen->path; $out2 = $this->__yara_file_asis->path; - $logging = $this->__end_of_cmd; - $raw_flag = $this->__raw_mode ? '--raw' : ''; - $my_server = ClassRegistry::init('Server'); - $result = shell_exec($my_server->getPythonVersion() . " $pythonScript --input $in --out-generated $out1 --out-asis $out2 $raw_flag $logging"); + + $command = [ + ProcessTool::pythonBin(), + $pythonScript, + '--input', $in, + '--out-generated', $out1, + '--out-asis', $out2, + ]; + if ($this->__raw_mode) { + $command[] = '--raw'; + } + + ProcessTool::execute($command, null, true); + $this->__curr_input_file->close(); $this->__curr_input_file->delete(); $this->__n_attributes = 0; From bd9cecbc2c8e0e8a65ce13b84c7677ce23f2112d Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 19 Nov 2021 22:21:57 +0100 Subject: [PATCH 028/101] chg: [internal] Use ProcessTool in Sighting --- app/Model/AppModel.php | 5 ----- app/Model/Sighting.php | 3 ++- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index effa54a82..bc357c887 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1948,11 +1948,6 @@ class AppModel extends Model return true; } - public function getPythonVersion() - { - return Configure::read('MISP.python_bin') ?: 'python3'; - } - public function validateAuthkey($value) { if (empty($value['authkey'])) { diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index f9b433c56..a0c5d2e4c 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -2,6 +2,7 @@ App::uses('AppModel', 'Model'); App::uses('TmpFileTool', 'Tools'); App::uses('ServerSyncTool', 'Tools'); +App::uses('ProcessTool', 'Tools'); /** * @property Attribute $Attribute @@ -744,7 +745,7 @@ class Sighting extends AppModel $tempFile->close(); $scriptFile = APP . "files" . DS . "scripts" . DS . "stixsighting2misp.py"; // Execute the python script and point it to the temporary filename - $result = shell_exec($this->getPythonVersion() . ' ' . $scriptFile . ' ' . $randomFileName); + $result = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile, $randomFileName]); // The result of the script will be a returned JSON object with 2 variables: success (boolean) and message // If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along $result = json_decode($result, true); From f7532da589ecae66e404e07860e84204ea80eaa9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 10:20:35 +0100 Subject: [PATCH 029/101] new: [test] Exports --- tests/testlive_comprehensive_local.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/testlive_comprehensive_local.py b/tests/testlive_comprehensive_local.py index f3ffede50..76d10c389 100644 --- a/tests/testlive_comprehensive_local.py +++ b/tests/testlive_comprehensive_local.py @@ -2,6 +2,7 @@ import os import unittest import uuid +from xml.etree import ElementTree as ET from io import BytesIO import urllib3 # type: ignore @@ -565,6 +566,33 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(1, len(fetched_event.tags), fetched_event.tags) self.assertTrue(fetched_event.tags[0].local, fetched_event.tags[0]) + def test_export(self): + event = create_simple_event() + event.add_attribute("ip-src", "1.2.4.5", to_ids=True) + event = check_response(self.admin_misp_connector.add_event(event)) + + result = self._search({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]}) + ET.fromstring(result) # check if result is valid XML + self.assertTrue("1.2.4.5" in result, result) + + result = self._search({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]}) + self.assertTrue("1.2.4.5" in result, result) + self.assertTrue("GENERATED" in result, result) + self.assertTrue("AS-IS" in result, result) + + result = self._search({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]}) + self.assertIn("generated", result) + self.assertEqual(len(result["generated"]), 1, result) + self.assertIn("as-is", result) + + check_response(self.admin_misp_connector.delete_event(event)) + + def _search(self, query: dict): + response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query) + response = self.admin_misp_connector._check_response(response) + check_response(response) + return response + if __name__ == '__main__': unittest.main() From dba7d03cfb92e663eb721d4a76efcb95a18e3b62 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 11:58:14 +0100 Subject: [PATCH 030/101] chg: [stix] Simplified STIX export code --- app/Lib/Export/StixExport.php | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/app/Lib/Export/StixExport.php b/app/Lib/Export/StixExport.php index 6b64a4c23..c6669a911 100644 --- a/app/Lib/Export/StixExport.php +++ b/app/Lib/Export/StixExport.php @@ -19,7 +19,6 @@ abstract class StixExport protected $__filenames = array(); protected $__version = null; - private $__current_filename = null; private $__empty_file = null; /** @var File */ private $__tmp_file = null; @@ -27,8 +26,6 @@ abstract class StixExport public $non_restrictive_export = true; - private $Server; - public function setDefaultFilters($filters) { $sane_version = !empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions, true); @@ -56,7 +53,7 @@ abstract class StixExport } else { $this->__tmp_file->append(']}'); $this->__tmp_file->close(); - $this->__filenames[] = $this->__current_filename; + $this->__filenames[] = $this->__tmp_file->path; $this->__initialize_misp_file(); $this->__tmp_file->append($event); $this->__n_attributes = $attributesCount; @@ -88,7 +85,7 @@ abstract class StixExport } else { $this->__tmp_file->append(']}'); $this->__tmp_file->close(); - $this->__filenames[] = $this->__current_filename; + $this->__filenames[] = $this->__tmp_file->path; } $result = $this->__parse_misp_events($this->__filenames); $this->__delete_temporary_files(); @@ -118,9 +115,9 @@ abstract class StixExport private function __initialize_misp_file() { - $this->__current_filename = FileAccessTool::createTempFile(); - $this->__tmp_file = new File($this->__current_filename); - $this->__tmp_file->write('{"response": ['); + $tmpFile = FileAccessTool::createTempFile(); + $this->__tmp_file = new File($tmpFile); + $this->__tmp_file->write('{"response":['); $this->__empty_file = true; } @@ -138,11 +135,15 @@ abstract class StixExport private function getFraming() { $framingCmd = $this->__initiate_framing_params(); - $framing = json_decode(ProcessTool::execute($framingCmd, null, true), true); - if ($framing === null || isset($framing['error'])) { - throw new Exception("Could not get results from framing cmd when exporting STIX file."); + try { + $framing = JsonTool::decode(ProcessTool::execute($framingCmd, null, true)); + if (isset($framing['error'])) { + throw new Exception("Framing command error: " . $framing['error']); + } + return $framing; + } catch (Exception $e) { + throw new Exception("Could not get results from framing cmd when exporting STIX file.", 0, $e); } - return $framing; } /** From 160a26e16f9a79569d3f90fbec9f49615513c4ad Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 11:58:44 +0100 Subject: [PATCH 031/101] chg: [export] Cleanup code for OpeniocExport and YaraExport --- app/Lib/Export/OpeniocExport.php | 153 ++++++++++--------------------- app/Lib/Export/YaraExport.php | 119 ++++++++++++------------ app/Lib/Tools/TmpFileTool.php | 15 +++ 3 files changed, 122 insertions(+), 165 deletions(-) diff --git a/app/Lib/Export/OpeniocExport.php b/app/Lib/Export/OpeniocExport.php index 11137c363..890b5b321 100644 --- a/app/Lib/Export/OpeniocExport.php +++ b/app/Lib/Export/OpeniocExport.php @@ -2,92 +2,58 @@ class OpeniocExport { + const MAPPING = array( + 'composite' => array( + 'regkey|value' => array(array('Network', 'RegistryItem/KeyPath', 'string'), array('Network', 'RegistryItem/Value', 'string')), + 'filename|md5' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')), + 'filename|sha1' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha1sum', 'sha1')), + 'filename|sha256' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha256sum', 'sha256')), + 'malware-sample' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')), + 'domain|ip' => array(array('Network', 'Network/DNS', 'string'), array('PortItem', 'PortItem/remoteIP', 'IP')), + ), + 'simple' => array( + 'md5' => array('FileItem', 'FileItem/Md5sum', 'md5'), + 'sha1' => array('FileItem', 'FileItem/Sha1sum', 'sha1'), + 'sha256' => array('FileItem', 'FileItem/Sha256sum', 'sha256'), + 'filename' => array('FileItem', 'FileItem/FileName', 'string'), + 'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'), + 'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'), + 'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'), + 'email' => array('Email', 'Email/From', 'string'), + 'email-src' => array('Email', 'Email/From', 'string'), + 'email-dst' => array('Email', 'Email/To', 'string'), + 'email-subject' => array('Email', 'Email/Subject', 'string'), + 'email-attachment' => array('Email', 'Email/Attachment/Name', 'string'), + 'domain' => array('Network', 'Network/DNS', 'string'), + 'url' => array('UrlHistoryItem', 'UrlHistoryItem/URL', 'string'), + 'user-agent' => array('Network', 'Network/UserAgent', 'string'), + 'regkey' => array('Network', 'RegistryItem/KeyPath', 'string'), + 'snort' => array('Snort', 'Snort/Snort', 'string'), + 'attachment' => array('FileItem', 'FileItem/FileName', 'string'), + 'link' => array('URL', 'UrlHistoryItem/URL', 'md5') + ) + ); + public $additional_params = array( 'flatten' => 1 ); - public function buildAll($user, $data, $scope = 'event') - { - $final = ''; - if (!isset($data['Attribute'])) { - $data = array('Attribute' => $data); - } - if ($scope == 'event') { - $final = $this->generateSingleTop($data); - } else { - $final = $this->generateTop($user, $data); - } - foreach ($data['Attribute'] as $attribute) { - $final .= $this->generateAttribute($attribute); - } - $final .= $this->generateBottom(); - return $final; - } - - public function convert($attributes) - { - $final = ''; - foreach ($attributes as $attribute) { - $final .= $this->generateAttribute($attribute); - } - return $final; - } - - public function getResult() - { - return $this->__final; - } - - public $mapping = array( - 'composite' => array( - 'regkey|value' => array(array('Network', 'RegistryItem/KeyPath', 'string'), array('Network', 'RegistryItem/Value', 'string')), - 'filename|md5' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')), - 'filename|sha1' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha1sum', 'sha1')), - 'filename|sha256' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha256sum', 'sha256')), - 'malware-sample' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')), - 'domain|ip' => array(array('Network', 'Network/DNS', 'string'), array('PortItem', 'PortItem/remoteIP', 'IP')), - ), - 'simple' => array( - 'md5' => array('FileItem', 'FileItem/Md5sum', 'md5'), - 'sha1' => array('FileItem', 'FileItem/Sha1sum', 'sha1'), - 'sha256' => array('FileItem', 'FileItem/Sha256sum', 'sha256'), - 'filename' => array('FileItem', 'FileItem/FileName', 'string'), - 'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'), - 'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'), - 'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'), - 'email' => array('Email', 'Email/From', 'string'), - 'email-src' => array('Email', 'Email/From', 'string'), - 'email-dst' => array('Email', 'Email/To', 'string'), - 'email-subject' => array('Email', 'Email/Subject', 'string'), - 'email-attachment' => array('Email', 'Email/Attachment/Name', 'string'), - 'domain' => array('Network', 'Network/DNS', 'string'), - 'url' => array('UrlHistoryItem', 'UrlHistoryItem/URL', 'string'), - 'user-agent' => array('Network', 'Network/UserAgent', 'string'), - 'regkey' => array('Network', 'RegistryItem/KeyPath', 'string'), - 'snort' => array('Snort', 'Snort/Snort', 'string'), - 'attachment' => array('FileItem', 'FileItem/FileName', 'string'), - 'link' => array('URL', 'UrlHistoryItem/URL', 'md5') - ) - ); - - public function frameComposite($attribute) + private function frameComposite($attribute) { + $mapping = self::MAPPING['composite'][$attribute['type']]; $temp = ''; $values = explode('|', $attribute['value']); $temp .= ' ' . PHP_EOL; - $temp .= $this->frameIndicator($this->mapping['composite'][$attribute['type']][0], $attribute['uuid'], $values[0], true); - $temp .= $this->frameIndicator($this->mapping['composite'][$attribute['type']][1], $attribute['uuid'], $values[1], true); + $temp .= $this->frameIndicator($mapping[0], $attribute['uuid'], $values[0], true); + $temp .= $this->frameIndicator($mapping[1], $attribute['uuid'], $values[1], true); $temp .= ' ' . PHP_EOL; return $temp; } - public function frameIndicator($mapping, $uuid, $value, $extraIndent = false) + private function frameIndicator($mapping, $uuid, $value, $extraIndent = false) { $temp = ''; - $padding = 6; - if ($extraIndent) { - $padding = 8; - } + $padding = $extraIndent ? 8 : 6; $temp .= str_repeat(' ', $padding) . '' . PHP_EOL; $temp .= str_repeat(' ', ($padding + 2)) . '' . PHP_EOL; $temp .= str_repeat(' ', ($padding + 2)) . '' . h($value) . '' . PHP_EOL; @@ -95,26 +61,16 @@ class OpeniocExport return $temp; } - // This method will turn each eligible attribute into an indicator - public function generateAttribute($attribute) - { - - } - - // Simple check for valid categories and types for IOC generation - public function checkValidTypeForIOC($attribute) + private function checkValidTypeForIOC($attribute) { // categories that should be included $category = array('Payload delivery', 'Artifacts dropped', 'Payload installation', 'Persistence mechanism', 'Network activity'); - if (!in_array($attribute['category'], $category)) { - return false; - } - return true; + return in_array($attribute['category'], $category, true); } - private function __attributeHandler($attribute, $options = array()) { - $temp = ''; + private function attributeHandler($attribute, $options = array()) + { if (isset($attribute['Attribute'])) { $attribute = $attribute['Attribute']; } @@ -122,44 +78,37 @@ class OpeniocExport if (!$this->checkValidTypeForIOC($attribute) || $attribute['to_ids'] == 0) { return false; } - if ($attribute['type'] == 'malware-sample') { + if ($attribute['type'] === 'malware-sample') { $attribute['type'] = 'filename|md5'; } if (strpos($attribute['type'], '|')) { - if (isset($this->mapping['composite'][$attribute['type']])) { - $temp .= $this->frameComposite($attribute); + if (isset(self::MAPPING['composite'][$attribute['type']])) { + return $this->frameComposite($attribute); } } else { - if (isset($this->mapping['simple'][$attribute['type']])) { - $temp .= $this->frameIndicator($this->mapping['simple'][$attribute['type']], $attribute['uuid'], $attribute['value'], false); + if (isset(self::MAPPING['simple'][$attribute['type']])) { + return $this->frameIndicator(self::MAPPING['simple'][$attribute['type']], $attribute['uuid'], $attribute['value'], false); } } - return $temp; + return false; } - public function handler($data, $options = array()) { if ($options['scope'] === 'Attribute') { - return $this->__attributeHandler($data, $options); + return $this->attributeHandler($data, $options); } else if ($options['scope'] === 'Event') { $result = ''; if (!empty($data['Attribute'])) { - $first = true; foreach ($data['Attribute'] as $attribute) { - $temp = $this->__attributeHandler($attribute, $options); + $temp = $this->attributeHandler($attribute, $options); if (!empty($temp)) { - if (!$first) { - $result .= $this->separator($options); - } $result .= $temp; - $first = false; } } } return $result; } - } public function header($options = array()) @@ -179,7 +128,6 @@ class OpeniocExport $temp .= ' ' . PHP_EOL; $temp .= ' ' . PHP_EOL; return $temp; - } public function footer() @@ -195,5 +143,4 @@ class OpeniocExport { return ''; } - } diff --git a/app/Lib/Export/YaraExport.php b/app/Lib/Export/YaraExport.php index 3ace9d726..feb6a81c0 100644 --- a/app/Lib/Export/YaraExport.php +++ b/app/Lib/Export/YaraExport.php @@ -1,59 +1,57 @@ __JsonExporter = new JsonExport(); + } + private static function __count_atributes($data) { - $attributes_count = count($data['Attribute']); - // foreach ($data['Object'] as $_object) { - // $attributes_count += count($_object['Attribute']); - // } + $attributes_count = count($data['Attribute']); + foreach ($data['Object'] as $_object) { + $attributes_count += count($_object['Attribute']); + } + return $attributes_count; } public function header($options = array()) { - if($this->__JsonExporter === false){ - $this->__JsonExporter = new JsonExport(); - } - $this->__initialize_yara_file(); $this->__initialize_misp_file($options); - if($options['returnFormat'] === 'yara-json') { + + $this->__yara_file_gen = FileAccessTool::createTempFile(); + $this->__yara_file_asis = FileAccessTool::createTempFile(); + + if ($options['returnFormat'] === 'yara-json') { $this->__raw_mode = false; } return ''; } - private function __initialize_yara_file() - { - $yaraFileName = $this->generateRandomFileName(); - $this->__yara_file_gen = new File($this->__tmp_dir . $yaraFileName . '_generated', true, 0644); - $this->__yara_file_asis = new File($this->__tmp_dir . $yaraFileName . '_asis', true, 0644); - $this->__yara_file_gen->close(); - $this->__yara_file_asis->close(); - } - + /** + * @throws Exception + */ private function __initialize_misp_file($options) { - $mispFileName = $this->generateRandomFileName(); - $this->__curr_input_file = new File($this->__tmp_dir . $mispFileName, true, 0644); + $this->__curr_input_file = new File(FileAccessTool::createTempFile()); $header = $this->__JsonExporter->header($options); $this->__curr_input_file->append($header); $this->__curr_input_is_empty = true; @@ -64,11 +62,11 @@ class YaraExport // convert attribute(s) to json and write them to input queue file if ($options['scope'] === 'Attribute') { $attr_count = 1; - } else if($options['scope'] === 'Event') { - $attr_count = YaraExport::__count_atributes($data); + } else if ($options['scope'] === 'Event') { + $attr_count = $this->__count_atributes($data); } - if(!empty($data)){ - if(!$this->__curr_input_is_empty){ + if (!empty($data)) { + if (!$this->__curr_input_is_empty) { $this->separator(); // calling separator since returning '' will prevent it } $jsonData = $this->__JsonExporter->handler($data, $options); @@ -83,42 +81,48 @@ class YaraExport } $this->__n_attributes += $attr_count; // if the file exceeds the max_attributes, process it, delete it and reset the counter - if ($this->__n_attributes >= $this->__MAX_n_attributes){ + if ($this->__n_attributes >= $this->__MAX_n_attributes) { $this->__process_file($options); $this->__initialize_misp_file($options); } return ''; } + /** + * @param array $options + * @return TmpFileTool + * @throws Exception + */ public function footer($options = array()) { - if(!($this->__curr_input_is_empty)){ + if (!$this->__curr_input_is_empty) { $this->__process_file($options); } - $file = new File($this->__yara_file_gen->path); - $data_gen = $file->read(true, 'r'); - $file->close(); - $file->delete(); - $file = new File($this->__yara_file_asis->path); - $data_asis = $file->read(true, 'r'); - $file->close(); - $file->delete(); - if($this->__raw_mode){ - $output = - '// ===================================== GENERATED ===================================='. PHP_EOL . - $data_gen . PHP_EOL . - '// ===================================== AS-IS ===================================='. PHP_EOL . - $data_asis; - }else{ - $output = '{"generated":['. $data_gen .'],'. - '"as-is":[' . $data_asis . ']}'; + + $output = new TmpFileTool(); + + if ($this->__raw_mode) { + $output->write('// ===================================== GENERATED ===================================='. PHP_EOL); + $output->writeFromFile($this->__yara_file_gen); + $output->write(PHP_EOL . '// ===================================== AS-IS ===================================='. PHP_EOL); + $output->writeFromFile($this->__yara_file_asis); + } else { + $output->write('{"generated":['); + $output->writeFromFile($this->__yara_file_gen); + $output->write('],"as-is":['); + $output->writeFromFile($this->__yara_file_asis); + $output->write(']}'); } + + FileAccessTool::deleteFile($this->__yara_file_gen); + FileAccessTool::deleteFile($this->__yara_file_asis); + return $output; } public function separator() { - if(!$this->__curr_input_is_empty){ + if (!$this->__curr_input_is_empty) { $this->__curr_input_file->append(','); } return ''; @@ -128,17 +132,14 @@ class YaraExport { $footer = $this->__JsonExporter->footer($options); $this->__curr_input_file->append($footer); - $pythonScript = $this->__script_path; - $in = $this->__curr_input_file->path; - $out1 = $this->__yara_file_gen->path; - $out2 = $this->__yara_file_asis->path; + $this->__curr_input_file->close(); $command = [ ProcessTool::pythonBin(), - $pythonScript, - '--input', $in, - '--out-generated', $out1, - '--out-asis', $out2, + $this->__script_path, + '--input', $this->__curr_input_file->path, + '--out-generated', $this->__yara_file_gen, + '--out-asis', $this->__yara_file_asis, ]; if ($this->__raw_mode) { $command[] = '--raw'; @@ -146,13 +147,7 @@ class YaraExport ProcessTool::execute($command, null, true); - $this->__curr_input_file->close(); $this->__curr_input_file->delete(); $this->__n_attributes = 0; } - - public function generateRandomFileName() - { - return (new RandomTool())->random_str(false, 12); - } } diff --git a/app/Lib/Tools/TmpFileTool.php b/app/Lib/Tools/TmpFileTool.php index 86e25e5cf..4dbd2c32a 100644 --- a/app/Lib/Tools/TmpFileTool.php +++ b/app/Lib/Tools/TmpFileTool.php @@ -67,6 +67,21 @@ class TmpFileTool } } + /** + * @param string $path + * @throws Exception + */ + public function writeFromFile($path) + { + $file = fopen($path, 'r'); + if (!$file) { + throw new Exception("Could not open file $file."); + } + if (stream_copy_to_stream($file, $this->tmpfile) === false) { + throw new Exception("Could not copy content of file $file into TmpFile."); + } + } + /** * Returns generator of parsed CSV line from file. * From e13d0bd4aef7f9a9e10de58fc20073afb56fb040 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 14:32:39 +0100 Subject: [PATCH 032/101] chg: [internal] New method ProcessTool::whoami --- app/Console/Command/AdminShell.php | 7 ++----- app/Console/Command/StartWorkerShell.php | 12 ++---------- app/Lib/Tools/ProcessTool.php | 14 ++++++++++++++ app/Model/Server.php | 7 +------ 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 450bab021..6ccf9c64e 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -1,5 +1,6 @@ out('Executing all updates to bring the database up to date with the current version.'); $processId = empty($this->args[0]) ? false : $this->args[0]; diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php index 42b180a3b..e46f9e16f 100644 --- a/app/Console/Command/StartWorkerShell.php +++ b/app/Console/Command/StartWorkerShell.php @@ -3,6 +3,7 @@ declare(strict_types=1); App::uses('BackgroundJobsTool', 'Tools'); +App::uses('ProcessTool', 'Tools'); class StartWorkerShell extends AppShell { @@ -50,7 +51,7 @@ class StartWorkerShell extends AppShell [ 'pid' => getmypid(), 'queue' => $this->args[0], - 'user' => $this->whoami() + 'user' => ProcessTool::whoami(), ] ); @@ -109,13 +110,4 @@ class StartWorkerShell extends AppShell exit; } } - - private function whoami(): string - { - if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) { - return posix_getpwuid(posix_geteuid())['name']; - } else { - return trim(shell_exec('whoami')); - } - } } diff --git a/app/Lib/Tools/ProcessTool.php b/app/Lib/Tools/ProcessTool.php index 687b2058f..08b021e29 100644 --- a/app/Lib/Tools/ProcessTool.php +++ b/app/Lib/Tools/ProcessTool.php @@ -93,6 +93,20 @@ class ProcessTool return $stdout; } + /** + * Get current process user name + * @return string + * @throws ProcessException + */ + public static function whoami() + { + if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) { + return posix_getpwuid(posix_geteuid())['name']; + } else { + return rtrim(self::execute(['whoami'])); + } + } + /** * @return string */ diff --git a/app/Model/Server.php b/app/Model/Server.php index 954eebfb9..65488d0a0 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3411,12 +3411,7 @@ class Server extends AppModel $workers = $this->getWorkers(); - if (function_exists('posix_getpwuid')) { - $currentUser = posix_getpwuid(posix_geteuid()); - $currentUser = $currentUser['name']; - } else { - $currentUser = trim(ProcessTool::execute(['whoami'])); - } + $currentUser = ProcessTool::whoami(); $procAccessible = file_exists('/proc'); foreach ($workers as $pid => $worker) { $entry = ($worker['type'] == 'regular') ? $worker['queue'] : $worker['type']; From b322bd55c579275b26816251148aac8fe97808bf Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 14:41:19 +0100 Subject: [PATCH 033/101] fix: [CLI] Show error when calling methods for managing workers when SimpleBackgroundJobs are enabled --- app/Console/Command/AdminShell.php | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 6ccf9c64e..22e0dc77a 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -126,13 +126,6 @@ class AdminShell extends AppShell echo $this->Server->update($status) . PHP_EOL; } - public function restartWorkers() - { - $this->ConfigLoad->execute(); - $this->Server->restartWorkers(); - echo PHP_EOL . 'Workers restarted.' . PHP_EOL; - } - public function updateAfterPull() { $this->ConfigLoad->execute(); @@ -156,8 +149,23 @@ class AdminShell extends AppShell } } + public function restartWorkers() + { + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->error('This method does nothing when SimpleBackgroundJobs are enabled.'); + } + + $this->ConfigLoad->execute(); + $this->Server->restartWorkers(); + echo PHP_EOL . 'Workers restarted.' . PHP_EOL; + } + public function restartWorker() { + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->error('This method does nothing when SimpleBackgroundJobs are enabled.'); + } + $this->ConfigLoad->execute(); if (empty($this->args[0]) || !is_numeric($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Restart a worker'] . PHP_EOL); @@ -180,6 +188,10 @@ class AdminShell extends AppShell public function killWorker() { + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->error('This method does nothing when SimpleBackgroundJobs are enabled.'); + } + $this->ConfigLoad->execute(); if (empty($this->args[0]) || !is_numeric($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Kill a worker'] . PHP_EOL); @@ -197,6 +209,10 @@ class AdminShell extends AppShell public function startWorker() { + if (Configure::read('SimpleBackgroundJobs.enabled')) { + $this->error('This method does nothing when SimpleBackgroundJobs are enabled.'); + } + $this->ConfigLoad->execute(); if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Start a worker'] . PHP_EOL); From cc1908a7fd632308900807cef75d1553f1fb9749 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 14:49:58 +0100 Subject: [PATCH 034/101] chg: [CLI] Initialize BackgroundJobsTool just when required --- app/Console/Command/ServerShell.php | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 21f8272b8..89b535744 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -79,8 +79,7 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - - $backgroundJobId = $this->Server->getBackgroundJobsTool()->enqueue( + $backgroundJobId = $this->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ @@ -189,8 +188,7 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - - $jobId = $this->Server->getBackgroundJobsTool()->enqueue( + $jobId = $this->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ @@ -309,8 +307,7 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { - - $jobId = $this->Server->getBackgroundJobsTool()->enqueue( + $jobId = $this->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, [ @@ -320,7 +317,7 @@ class ServerShell extends AppShell ] ); - $this->out("Enqueued cacheServer from {$serverName} server as job $jobId"); + $this->out("Enqueued cacheServer from $serverName server as job $jobId"); } } @@ -605,4 +602,15 @@ class ServerShell extends AppShell } return $server; } + + /** + * @return BackgroundJobsTool + */ + private function getBackgroundJobsTool() + { + if (!$this->BackgroundJobsTool) { + $this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs')); + } + return $this->BackgroundJobsTool; + } } From 05e85ce6869066a2609ddb5b383cdc11f87f8e22 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 16:37:59 +0100 Subject: [PATCH 035/101] chg: [internal] Simplify index.php --- app/Console/cake.php | 7 ++++--- app/webroot/index.php | 10 ++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/Console/cake.php b/app/Console/cake.php index a7e737f29..209d12ad2 100644 --- a/app/Console/cake.php +++ b/app/Console/cake.php @@ -23,13 +23,14 @@ if (!defined('DS')) { $dispatcher = 'Cake' . DS . 'Console' . DS . 'ShellDispatcher.php'; if (function_exists('ini_set')) { - $root = dirname(dirname(dirname(__FILE__))); - $appDir = basename(dirname(dirname(__FILE__))); - $install = $root . DS . $appDir . DS . 'Lib' . DS . 'cakephp' . DS . 'lib'; + $root = dirname(__DIR__, 2); + $appDir = basename(dirname(__DIR__)); $composerInstall = $root . DS . $appDir . DS . 'Vendor' . DS . 'cakephp' . DS . 'cakephp' . DS . 'lib'; if (file_exists($composerInstall . DS . $dispatcher)) { $install = $composerInstall; // prefer compose install + } else { + $install = $root . DS . $appDir . DS . 'Lib' . DS . 'cakephp' . DS . 'lib'; } ini_set('include_path', $install . PATH_SEPARATOR . ini_get('include_path')); diff --git a/app/webroot/index.php b/app/webroot/index.php index 705e228e0..95090e1df 100644 --- a/app/webroot/index.php +++ b/app/webroot/index.php @@ -32,17 +32,15 @@ if (!defined('DS')) { /** * The full path to the directory which holds "app", WITHOUT a trailing DS. - * */ if (!defined('ROOT')) { - define('ROOT', dirname(dirname(dirname(__FILE__)))); + define('ROOT', dirname(__DIR__, 2)); } /** * The actual directory name for the "app". - * */ if (!defined('APP_DIR')) { - define('APP_DIR', basename(dirname(dirname(__FILE__)))); + define('APP_DIR', basename(dirname(__DIR__))); } /** @@ -73,10 +71,10 @@ if (!defined('CAKE_CORE_INCLUDE_PATH') && file_exists($vendorPath . DS . $dispat * Change at your own risk. */ if (!defined('WEBROOT_DIR')) { - define('WEBROOT_DIR', basename(dirname(__FILE__))); + define('WEBROOT_DIR', basename(__DIR__)); } if (!defined('WWW_ROOT')) { - define('WWW_ROOT', dirname(__FILE__) . DS); + define('WWW_ROOT', __DIR__ . DS); } if (!defined('CAKE_CORE_INCLUDE_PATH')) { From 82a7be8b4d8ad6e1ab50c99a93fa8ca938d9021c Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 18:27:40 +0100 Subject: [PATCH 036/101] chg: [upload] Allow to upload SVG files --- app/Model/Server.php | 62 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 65488d0a0..70542393b 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2339,47 +2339,51 @@ class Server extends AppModel public function getFileRules() { - $validItems = array( - 'orgs' => array( - 'name' => __('Organisation logos'), - 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), - 'expected' => array(), - 'valid_format' => __('48x48 pixel .png files'), - 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', - 'regex' => '.*\.(png|PNG)$', - 'regex_error' => __('Filename must be in the following format: *.png'), - 'files' => array(), - ), - 'img' => array( - 'name' => __('Additional image files'), - 'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'), - 'expected' => array( - 'MISP.footer_logo' => Configure::read('MISP.footer_logo'), - 'MISP.home_logo' => Configure::read('MISP.home_logo'), - 'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'), - 'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'), - ), - 'valid_format' => __('text/html if served inline, anything that conveys the terms of use if served as download'), - 'path' => APP . 'webroot' . DS . 'img' . DS . 'custom', - 'regex' => '.*\.(png|PNG)$', - 'regex_error' => __('Filename must be in the following format: *.png'), - 'files' => array(), + return array( + 'orgs' => array( + 'name' => __('Organisation logos'), + 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), + 'expected' => array(), + 'valid_format' => __('48x48 pixel .png files'), + 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', + 'regex' => '.*\.(png|PNG)$', + 'regex_error' => __('Filename must be in the following format: *.png'), + 'files' => array(), + ), + 'img' => array( + 'name' => __('Additional image files'), + 'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'), + 'expected' => array( + 'MISP.footer_logo' => Configure::read('MISP.footer_logo'), + 'MISP.home_logo' => Configure::read('MISP.home_logo'), + 'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'), + 'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'), ), + 'valid_format' => __('PNG or SVG file'), + 'path' => APP . 'webroot' . DS . 'img' . DS . 'custom', + 'regex' => '.*\.(png|svg)$', + 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), + 'files' => array(), + ), ); - return $validItems; } public function grabFiles() { $validItems = $this->getFileRules(); App::uses('Folder', 'Utility'); - App::uses('File', 'Utility'); foreach ($validItems as $k => $item) { $dir = new Folder($item['path']); $files = $dir->find($item['regex'], true); foreach ($files as $file) { - $f = new File($item['path'] . DS . $file); - $validItems[$k]['files'][] = array('filename' => $file, 'filesize' => $f->size(), 'read' => $f->readable(), 'write' => $f->writable(), 'execute' => $f->executable()); + $f = new SplFileInfo($item['path'] . DS . $file); + $validItems[$k]['files'][] = [ + 'filename' => $file, + 'filesize' => $f->getSize(), + 'read' => $f->isReadable(), + 'write' => $f->isWritable(), + 'execute' => $f->isExecutable(), + ]; } } return $validItems; From f895bb21e27d45345da5743ff6b97fb86d22fde3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 19:07:09 +0100 Subject: [PATCH 037/101] chg: [internal] Optimise setting --- app/Model/Server.php | 88 ++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 70542393b..ec8d487ba 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -1377,11 +1377,6 @@ class Server extends AppModel public function serverSettingReadSingle($settingObject, $settingName, $leafKey) { - // invalidate config.php from php opcode cache - if (function_exists('opcache_reset')) { - opcache_reset(); - } - $setting = Configure::read($settingName); $result = $this->__evaluateLeaf($settingObject, $leafKey, $setting); $result['setting'] = $settingName; @@ -2100,7 +2095,7 @@ class Server extends AppModel return true; } - private function __serverSettingNormaliseValue($data, $value, $setting) + private function __serverSettingNormaliseValue($data, $value) { if (!empty($data['type'])) { if ($data['type'] === 'boolean') { @@ -2112,41 +2107,39 @@ class Server extends AppModel return $value; } - public function getSettingData($setting_name) + /** + * @param string $setting_name + * @return array|false False if setting doesn't exists + */ + public function getSettingData($setting_name, $withOptions = true) { - // invalidate config.php from php opcode cache - if (function_exists('opcache_reset')) { - opcache_reset(); - } + // This is just hack to reset opcache, so for next request cache will be reloaded. + $this->opcacheResetConfig(); + if (strpos($setting_name, 'Plugin.Enrichment') !== false || strpos($setting_name, 'Plugin.Import') !== false || strpos($setting_name, 'Plugin.Export') !== false || strpos($setting_name, 'Plugin.Cortex') !== false) { $serverSettings = $this->getCurrentServerSettings(); } else { $serverSettings = $this->serverSettings; } - $setting = false; - foreach ($serverSettings as $k => $s) { - if (isset($s['branch'])) { - foreach ($s as $ek => $es) { - if ($ek != 'branch') { - if ($setting_name == $k . '.' . $ek) { - $setting = $es; - continue 2; - } - } - } + + $setting = $serverSettings; + $parts = explode('.', $setting_name); + foreach ($parts as $part) { + if (isset($setting[$part])) { + $setting = $setting[$part]; } else { - if ($setting_name == $k) { - $setting = $s; - continue; - } + $setting = false; + break; } } - if (!empty($setting)) { + + if (isset($setting['level'])) { $setting['name'] = $setting_name; + if ($withOptions && isset($setting['optionsSource'])) { + $setting['options'] = $setting['optionsSource'](); + } } - if (!empty($setting['optionsSource'])) { - $setting['options'] = $setting['optionsSource'](); - } + return $setting; } @@ -2264,19 +2257,10 @@ class Server extends AppModel throw new Exception("Could not create config backup `$backupFilePath`."); } } - $settingObject = $this->getCurrentServerSettings(); - foreach ($settingObject as $branchName => $branch) { - if (!isset($branch['level'])) { - foreach ($branch as $settingName => $settingObject) { - if ($setting === $branchName . '.' . $settingName) { - $value = $this->__serverSettingNormaliseValue($settingObject, $value, $setting); - } - } - } else { - if ($setting === $branchName) { - $value = $this->__serverSettingNormaliseValue($branch, $value, $setting); - } - } + + $settingObject = $this->getSettingData($setting, false); + if ($settingObject) { + $value = $this->__serverSettingNormaliseValue($settingObject, $value); } /** @var array $config */ @@ -2315,9 +2299,7 @@ class Server extends AppModel FileAccessTool::deleteFile($tmpFile); throw new Exception("Could not rename `$tmpFile` to config file `$configFilePath`."); } - if (function_exists('opcache_reset')) { - opcache_reset(); - } + $this->opcacheResetConfig(); chmod($configFilePath, octdec($previous_file_perm)); $config_saved = FileAccessTool::readFromFile($configFilePath); // if the saved config file is empty, restore the backup. @@ -2330,9 +2312,7 @@ class Server extends AppModel } } else { FileAccessTool::writeToFile($configFilePath, $settingsString); - if (function_exists('opcache_reset')) { - opcache_reset(); - } + $this->opcacheResetConfig(); } return true; } @@ -4616,6 +4596,16 @@ class Server extends AppModel return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); } + /** + * Invalidate config.php from php opcode cache + */ + private function opcacheResetConfig() + { + if (function_exists('opcache_invalidate')) { + opcache_invalidate(APP . 'Config' . DS . 'config.php', true); + } + } + /** * Generate just when required * @return array[] From ba71bee2937d3f20549af73e048b72d239ddb011 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 19:26:15 +0100 Subject: [PATCH 038/101] chg: [internal] testForBinExec cleanup --- app/Model/Server.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index ec8d487ba..3285662f4 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -1575,19 +1575,20 @@ class Server extends AppModel if (substr($value, 0, 7) === "phar://") { return 'Phar protocol not allowed.'; } - $finfo = finfo_open(FILEINFO_MIME_TYPE); if ($value === '') { return true; } if (is_executable($value)) { - if (finfo_file($finfo, $value) == "application/x-executable" || finfo_file($finfo, $value) == "application/x-pie-executable" || finfo_file($finfo, $value) == "application/x-sharedlib") { - finfo_close($finfo); + $finfo = finfo_open(FILEINFO_MIME_TYPE); + $type = finfo_file($finfo, $value); + finfo_close($finfo); + if ($type === "application/x-executable" || $type === "application/x-pie-executable" || $type === "application/x-sharedlib") { return true; } else { - return 'Binary file not executable. It is of type: ' . finfo_file($finfo, $value); + return 'Binary file not executable. It is of type: ' . $type; } } else { - return false; + return 'Binary file not executable.'; } } From fc16acb07098205ae54c9b2b579d71538664e465 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 20 Nov 2021 21:39:10 +0100 Subject: [PATCH 039/101] new: [securityAudit] Check expose_php setting --- app/Lib/Tools/SecurityAudit.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index e2c1e669b..d793f1c3e 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -227,6 +227,13 @@ class SecurityAudit ]; } + if (ini_get('expose_php')) { + $output['PHP'][] = [ + 'hint', + __('PHP `expose_php` setting is enabled. That means that PHP version will be send in `X-Powered-By` header. This can help attackers.'), + ]; + } + if (extension_loaded('xdebug')) { $output['PHP'][] = [ 'error', From d5d83fe26d4e8c6b7609013eea21146fa2641c4c Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 08:33:55 +0100 Subject: [PATCH 040/101] chg: [securityAudit] PHP 7.3 is not supported anymore --- app/Lib/Tools/SecurityAudit.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/app/Lib/Tools/SecurityAudit.php b/app/Lib/Tools/SecurityAudit.php index d793f1c3e..23e40cc8e 100644 --- a/app/Lib/Tools/SecurityAudit.php +++ b/app/Lib/Tools/SecurityAudit.php @@ -213,18 +213,12 @@ class SecurityAudit } } catch (RuntimeException $e) {} - if (version_compare(PHP_VERSION, '7.3.0', '<')) { + if (version_compare(PHP_VERSION, '7.4.0', '<')) { $output['PHP'][] = [ 'warning', __('PHP version %s is not supported anymore. It can be still supported by your distribution.', PHP_VERSION), 'https://www.php.net/supported-versions.php' ]; - } else if (version_compare(PHP_VERSION, '7.4.0', '<')) { - $output['PHP'][] = [ - 'hint', - __('PHP version 7.3 will not be supported after 6 Dec 2021. Even beyond that date, it can be still supported by your distribution.'), - 'https://www.php.net/supported-versions.php' - ]; } if (ini_get('expose_php')) { From 458e900a737706278181bf4f08fc80764378dd62 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 10:50:25 +0100 Subject: [PATCH 041/101] fix: [security] Disable caching of images --- app/Controller/AttributesController.php | 49 +++++++++++++++---------- app/Lib/Tools/AttachmentTool.php | 3 +- app/Model/Attribute.php | 2 +- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 7a019d06d..110506248 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -998,37 +998,46 @@ class AttributesController extends AppController { $conditions = $this->__idToConditions($id); $conditions['Attribute.type'] = 'attachment'; - $options = array( - 'conditions' => $conditions, - 'includeAllTags' => false, - 'includeAttributeUuid' => true, - 'flatten' => true, - 'deleted' => [0, 1] - ); if ($this->_isRest()) { - $options['withAttachments'] = true; + $options = array( + 'conditions' => $conditions, + 'includeAllTags' => false, + 'includeAttributeUuid' => true, + 'flatten' => true, + 'deleted' => [0, 1], + 'withAttachments' => true, + ); + $attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $options); + if (empty($attribute)) { + throw new MethodNotAllowedException('Invalid attribute'); + } + $attribute = $attribute[0]; + if (!$this->Attribute->isImage($attribute['Attribute'])) { + throw new NotFoundException("Attribute is not an image."); + } + return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type()); } - $attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $options); + $attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [ + 'conditions' => $conditions, + 'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.value'], + ]); if (empty($attribute)) { throw new MethodNotAllowedException('Invalid attribute'); } - $attribute = $attribute[0]; - if (!$this->Attribute->isImage($attribute['Attribute'])) { throw new NotFoundException("Attribute is not an image."); } - if ($this->_isRest()) { - return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type()); - } else { - $width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200; - $height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200; - $imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height); - $extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION); - return new CakeResponse(array('body' => $imageData, 'type' => strtolower($extension))); - } + $width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200; + $height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200; + $imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height); + $extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION); + + $this->response->body($imageData); + $this->response->type(strtolower($extension)); + return $this->response; } public function delete($id, $hard = false) diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index 5f6891023..b18ff9d5a 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -425,8 +425,7 @@ class AttachmentTool // Output image to string ob_start(); imagepng($imageThumbnail, null, 9); - $imageData = ob_get_contents(); - ob_end_clean(); + $imageData = ob_get_clean(); imagedestroy($imageThumbnail); return $imageData; diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index ba3282133..915cec6ac 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -952,7 +952,7 @@ class Attribute extends AppModel * @return string * @throws Exception */ - public function getPictureData(array $attribute, $thumbnail=false, $maxWidth=200, $maxHeight=200) + public function getPictureData(array $attribute, $thumbnail = false, $maxWidth = 200, $maxHeight = 200) { if ($thumbnail && extension_loaded('gd')) { if ($maxWidth == 200 && $maxHeight == 200) { From c2bdf167a64ddfe6bbe838c5e3e1efefbd64c85f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 11:13:12 +0100 Subject: [PATCH 042/101] chg: [internal] Slightly optimise CakeResponseTmp --- app/Controller/AttributesController.php | 15 ++- .../Component/RestResponseComponent.php | 27 +++-- app/Lib/Tools/CakeResponseFile.php | 107 ++++++++++++++++++ app/Lib/Tools/CakeResponseTmp.php | 40 ------- 4 files changed, 133 insertions(+), 56 deletions(-) create mode 100644 app/Lib/Tools/CakeResponseFile.php delete mode 100644 app/Lib/Tools/CakeResponseTmp.php diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 110506248..c0dddeaf0 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -315,28 +315,27 @@ class AttributesController extends AppController if (empty($attributes)) { throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.')); } - $this->__downloadAttachment($attributes[0]['Attribute']); + return $this->__downloadAttachment($attributes[0]['Attribute']); } - private function __downloadAttachment($attribute) + private function __downloadAttachment(array $attribute) { $file = $this->Attribute->getAttachmentFile($attribute); - if ('attachment' == $attribute['type']) { + if ('attachment' === $attribute['type']) { $filename = $attribute['value']; $fileExt = pathinfo($filename, PATHINFO_EXTENSION); $filename = substr($filename, 0, strlen($filename) - strlen($fileExt) - 1); - } elseif ('malware-sample' == $attribute['type']) { + } elseif ('malware-sample' === $attribute['type']) { $filenameHash = explode('|', $attribute['value']); $filename = substr($filenameHash[0], strrpos($filenameHash[0], '\\')); $fileExt = "zip"; } else { throw new NotFoundException(__('Attribute not an attachment or malware-sample')); } - $this->autoRender = false; - $this->response->type($fileExt); + $download_attachments_on_load = Configure::check('MISP.download_attachments_on_load') ? Configure::read('MISP.download_attachments_on_load') : true; - $this->response->file($file->path, array('download' => $download_attachments_on_load, 'name' => $filename . '.' . $fileExt)); + return $this->RestResponse->sendFile($file, $fileExt, $download_attachments_on_load, $filename . '.' . $fileExt); } public function add_attachment($eventId = null) @@ -1762,7 +1761,7 @@ class AttributesController extends AppController if (empty($attributes)) { throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.')); } - $this->__downloadAttachment($attributes[0]['Attribute']); + return $this->__downloadAttachment($attributes[0]['Attribute']); } // returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter. diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 6dd49a708..1b7c356c0 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -590,8 +590,8 @@ class RestResponseComponent extends Component } if ($response instanceof TmpFileTool) { - App::uses('CakeResponseTmp', 'Tools'); - $cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]); + App::uses('CakeResponseFile', 'Tools'); + $cakeResponse = new CakeResponseFile(['status' => $code, 'type' => $type]); $cakeResponse->file($response); } else { $cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type)); @@ -655,13 +655,24 @@ class RestResponseComponent extends Component return $this->__sendResponse($data, 200, $format, $raw, $download, $headers); } - public function sendFile($path, $format = false, $download = false, $name = 'download') + /** + * @param string|File|TmpFileTool $path + * @param string|null $type + * @param bool $download + * @param string $name + * @return CakeResponseFile + * @throws Exception + */ + public function sendFile($path, $type = null, $download = false, $name = 'download') { - $cakeResponse = new CakeResponse(array( - 'status' => 200, - 'type' => $format - )); - $cakeResponse->file($path, array('name' => $name, 'download' => true)); + App::uses('CakeResponseFile', 'Tools'); + $cakeResponse = new CakeResponseFile([ + 'type' => $type + ]); + $cakeResponse->file($path, ['name' => $name, 'download' => $download]); + if (Configure::read('Security.disable_browser_cache')) { + $cakeResponse->disableCache(); + } return $cakeResponse; } diff --git a/app/Lib/Tools/CakeResponseFile.php b/app/Lib/Tools/CakeResponseFile.php new file mode 100644 index 000000000..78dd737b7 --- /dev/null +++ b/app/Lib/Tools/CakeResponseFile.php @@ -0,0 +1,107 @@ +header('Content-Length', $path->size()); + $this->_clearBuffer(); + $this->_file = $path; + } else if ($path instanceof File) { + $options += array( + 'name' => null, + 'download' => null + ); + + if ($options['download']) { + $name = $options['name'] === null ? $path->name : $options['name']; + $this->download($name); + $this->header('Content-Transfer-Encoding', 'binary'); + } + + $this->header('Accept-Ranges', 'bytes'); + $httpRange = env('HTTP_RANGE'); + if (isset($httpRange)) { + $this->_fileRange($path, $httpRange); + } else { + $this->header('Content-Length', filesize($path->path)); + } + + $this->_clearBuffer(); + $this->_file = $path; + } else { + parent::file($path, $options); + } + } + + /** + * This method supports TmpFileTool and also provides optimised variant for sending file from `File` object + * @param File|TmpFileTool $file + * @param array $range + * @return bool + * @throws Exception + */ + protected function _sendFile($file, $range) + { + set_time_limit(0); + session_write_close(); + + if ($file instanceof TmpFileTool) { + foreach ($file->intoChunks() as $chunk) { + if (!$this->_isActive()) { + $file->close(); + return false; + } + echo $chunk; + } + } else { + $handler = fopen($file->path, 'rb'); + if ($handler === false) { + throw new Exception("File $file->path doesn't exists anymore or is not readable."); + } + + $end = $start = false; + if ($range && is_array($range)) { + list($start, $end) = $range; + } + if ($start !== false) { + fseek($handler, $start); + } + + $bufferSize = 8192; + while (!feof($handler)) { + if (!$this->_isActive()) { + $file->close(); + return false; + } + $offset = ftell($handler); + if ($end && $offset >= $end) { + break; + } + if ($end && $offset + $bufferSize >= $end) { + $bufferSize = $end - $offset + 1; + } + echo fread($handler, $bufferSize); + } + fclose($handler); + } + $this->_flushBuffer(); + return true; + } + + /** + * Faster version that do not do redundant check + * @return bool + */ + protected function _isActive() + { + return connection_status() === CONNECTION_NORMAL; + } +} diff --git a/app/Lib/Tools/CakeResponseTmp.php b/app/Lib/Tools/CakeResponseTmp.php deleted file mode 100644 index 12952868b..000000000 --- a/app/Lib/Tools/CakeResponseTmp.php +++ /dev/null @@ -1,40 +0,0 @@ -header('Content-Length', $path->size()); - $this->_clearBuffer(); - $this->_file = $path; - } else { - parent::file($path, $options); - } - } - - /** - * @param File|TmpFileTool $file - * @param array $range - * @return bool - * @throws Exception - */ - protected function _sendFile($file, $range) - { - if ($file instanceof TmpFileTool) { - set_time_limit(0); - session_write_close(); - - foreach ($file->intoChunks() as $chunk) { - if (!$this->_isActive()) { - $file->close(); - return false; - } - echo $chunk; - $this->_flushBuffer(); - } - return true; - } else { - return parent::_sendFile($file, $range); - } - } -} From 63d9fc7274712a17cb996c44817398725261061f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 11:59:52 +0100 Subject: [PATCH 043/101] chg: [internal] Faster sending images --- app/Controller/AttributesController.php | 6 +++++- app/Lib/Tools/AttachmentTool.php | 2 +- app/Model/Attribute.php | 9 ++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index c0dddeaf0..9c4c44177 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1031,9 +1031,13 @@ class AttributesController extends AppController $width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200; $height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200; - $imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height); $extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION); + $imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height); + if ($imageData instanceof File) { + return $this->RestResponse->sendFile($imageData, strtolower($extension)); + } + $this->response->body($imageData); $this->response->type(strtolower($extension)); return $this->response; diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index b18ff9d5a..203ce861e 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -156,7 +156,7 @@ class AttachmentTool } else { $filepath = $this->attachmentDir() . DS . $path; $file = new File($filepath); - if (!$file->exists()) { + if (!is_file($file->path)) { throw new NotFoundException("File '$filepath' does not exists."); } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 915cec6ac..64758b6ad 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -949,7 +949,7 @@ class Attribute extends AppModel * @param bool $thumbnail * @param int $maxWidth - When $thumbnail is true * @param int $maxHeight - When $thumbnail is true - * @return string + * @return string|File * @throws Exception */ public function getPictureData(array $attribute, $thumbnail = false, $maxWidth = 200, $maxHeight = 200) @@ -958,7 +958,7 @@ class Attribute extends AppModel if ($maxWidth == 200 && $maxHeight == 200) { // Return thumbnail directly if already exists try { - return $this->getAttachment($attribute['Attribute'], $path_suffix = '_thumbnail'); + return $this->loadAttachmentTool()->getFile($attribute['Attribute']['event_id'], $attribute['Attribute']['id'], $path_suffix = '_thumbnail'); } catch (NotFoundException $e) { // pass } @@ -973,11 +973,10 @@ class Attribute extends AppModel $attribute['Attribute']['data'] = $imageData; $this->saveAttachment($attribute['Attribute'], $path_suffix='_thumbnail'); } - } else { - $imageData = $this->getAttachment($attribute['Attribute']); + return $imageData; } - return $imageData; + return $this->loadAttachmentTool()->getFile($attribute['Attribute']['event_id'], $attribute['Attribute']['id']); } /** From e2a0644111bd2c38303f7ce086a3ba740db1d6b2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 12:24:48 +0100 Subject: [PATCH 044/101] chg: [internal] Move some checks to beforeRender method --- app/Controller/AppController.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index b658933ac..2214574f6 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -368,9 +368,13 @@ class AppController extends Controller } } } + } + public function beforeRender() + { // Notifications and homepage is not necessary for AJAX or REST requests - if ($user && !$this->_isRest() && !$isAjax) { + $user = $this->Auth->user(); + if ($user && !$this->_isRest() && !$this->request->is('ajax')) { $hasNotifications = $this->User->hasNotifications($user); $this->set('hasNotifications', $hasNotifications); From 5aa1e0cb3b4c476e62a5ff0825698ad1cb52190b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 18:05:51 +0100 Subject: [PATCH 045/101] chg: [internal] Element file cache --- app/Controller/AppController.php | 10 ++++++++++ app/View/AppView.php | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 app/View/AppView.php diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 2214574f6..20385a53b 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -1413,4 +1413,14 @@ class AppController extends Controller return true; } } + + /** + * Override View class + * @return AppView + */ + protected function _getViewObject() + { + App::uses('AppView', 'View'); + return new AppView($this); + } } diff --git a/app/View/AppView.php b/app/View/AppView.php new file mode 100644 index 000000000..420bc6f53 --- /dev/null +++ b/app/View/AppView.php @@ -0,0 +1,23 @@ +elementFileCache[$name])) { + return $this->elementFileCache[$name]; + } + $result = parent::_getElementFileName($name); + $this->elementFileCache[$name] = $result; + return $result; + } +} From dac6a4e1f4abe4b700d6413014c16a3a58a562c2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 18:29:40 +0100 Subject: [PATCH 046/101] chg: [internal] Slightly optimise OrgImgHelper --- app/View/Helper/OrgImgHelper.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/app/View/Helper/OrgImgHelper.php b/app/View/Helper/OrgImgHelper.php index 6da47d790..07c53e9a1 100644 --- a/app/View/Helper/OrgImgHelper.php +++ b/app/View/Helper/OrgImgHelper.php @@ -6,6 +6,9 @@ class OrgImgHelper extends AppHelper { const IMG_PATH = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS; + /** @var array */ + private $imageCache = []; + public function getNameWithImg(array $organisation, $link = null) { if (!isset($organisation['Organisation'])) { @@ -97,15 +100,26 @@ class OrgImgHelper extends AppHelper */ private function findOrgImage(array $options) { + if (isset($options['id']) && array_key_exists($options['id'], $this->imageCache)) { + return $this->imageCache[$options['id']]; + } + + $image = null; foreach (['id', 'name', 'uuid'] as $field) { if (isset($options[$field])) { foreach (['png', 'svg'] as $extensions) { if (file_exists(self::IMG_PATH . $options[$field] . '.' . $extensions)) { - return $options[$field] . '.' . $extensions; + $image = $options[$field] . '.' . $extensions; + break 2; } } } } - return null; + + if (isset($options['id'])) { + $this->imageCache[$options['id']] = $image; + } + + return $image; } } From 41db04ad47238f19fd50b41cd99f0bf8cc4dc228 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 21:16:21 +0100 Subject: [PATCH 047/101] chg: [internal] Avoid calling unnecessary method --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 20385a53b..16063b0b7 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -180,7 +180,7 @@ class AppController extends Controller Configure::write('CurrentController', $this->request->params['controller']); Configure::write('CurrentAction', $this->request->params['action']); $versionArray = $this->User->checkMISPVersion(); - $this->mispVersion = implode('.', array_values($versionArray)); + $this->mispVersion = implode('.', $versionArray); $this->Security->blackHoleCallback = 'blackHole'; // send users away that are using ancient versions of IE From ba6cdd1f7baf5be78b2507c79df9fb648ff75df9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 21:54:41 +0100 Subject: [PATCH 048/101] chg: [internal] Lazy load images --- app/View/Elements/Events/View/value_field.ctp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/View/Elements/Events/View/value_field.ctp b/app/View/Elements/Events/View/value_field.ctp index 1fdeb39a7..ff3454370 100644 --- a/app/View/Elements/Events/View/value_field.ctp +++ b/app/View/Elements/Events/View/value_field.ctp @@ -27,13 +27,12 @@ switch ($object['type']) { case 'malware-sample': if ($object['type'] === 'attachment' && isset($object['image'])) { if ($object['image'] === true) { - $img = ''; - $img .= ''; - echo $img; + $src = $baseurl . '/' . ($object['objectType'] === 'proposal' ? 'shadowAttributes' : 'attributes') . '/viewPicture/' . (int)$object['id'] . '/1'; + echo ''; } else { $extension = pathinfo($object['value'], PATHINFO_EXTENSION); $uri = 'data:image/' . strtolower(h($extension)) . ';base64,' . h($object['image']); - echo ''; + echo ''; } } else { $filenameHash = explode('|', h($object['value'])); From d20795b08cbe9f3ec4747d27ab1dd7234a918c97 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 21 Nov 2021 22:17:35 +0100 Subject: [PATCH 049/101] fix: [internal] Old style view class --- app/Controller/AppController.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 16063b0b7..bf5953785 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -1415,12 +1415,15 @@ class AppController extends Controller } /** - * Override View class - * @return AppView + * Override default View class + * @return View */ protected function _getViewObject() { - App::uses('AppView', 'View'); - return new AppView($this); + if ($this->viewClass === 'View') { + App::uses('AppView', 'View'); + return new AppView($this); + } + return parent::_getViewObject(); } } From cd9efaed5402ae0850b4ba081144df857cbce63e Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Mon, 22 Nov 2021 10:36:49 +0100 Subject: [PATCH 050/101] fix: update dep for fixing php74 build --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51b148441..bc2b3f849 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,6 +71,8 @@ jobs: LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y sudo apt-get -y install curl python3 python3-zmq python3-requests python3-pip python3-nose python3-redis python3-lxml apache2 libapache2-mod-php$php_version libfuzzy-dev sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu packages + # hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel + sudo apt install --only-upgrade libpcre2-8-0 curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - sudo chown $USER:www-data $HOME/.composer pushd app From 6a7ce569396829c1153520d5e833301e4864a61a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Nov 2021 10:27:15 +0100 Subject: [PATCH 051/101] chg: [internal] Add job ID to worker --- app/Console/Command/ServerShell.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 89b535744..23140d64e 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -79,6 +79,7 @@ class ServerShell extends AppShell )); foreach ($servers as $serverId => $serverName) { + $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'pull', "Server: $serverId", 'Pulling.'); $backgroundJobId = $this->getBackgroundJobsTool()->enqueue( BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::CMD_SERVER, @@ -86,8 +87,11 @@ class ServerShell extends AppShell 'pull', $user['id'], $serverId, - $technique - ] + $technique, + $jobId, + ], + true, + $jobId ); $this->out("Enqueued pulling from $serverName server as job $backgroundJobId"); From 328b326297ec9119c33c2093509370120887d71b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Nov 2021 15:23:50 +0100 Subject: [PATCH 052/101] fix: [internal] Deleting events --- app/Lib/Tools/AttachmentTool.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Lib/Tools/AttachmentTool.php b/app/Lib/Tools/AttachmentTool.php index 203ce861e..46d77eaae 100644 --- a/app/Lib/Tools/AttachmentTool.php +++ b/app/Lib/Tools/AttachmentTool.php @@ -281,6 +281,7 @@ class AttachmentTool $s3 = $this->loadS3Client(); $s3->deleteDirectory($eventId); } else { + App::uses('Folder', 'Utility'); $dirPath = $this->attachmentDir(); foreach (array($dirPath, $dirPath . DS . 'shadow') as $dirPath) { From 9458290b929d267ec85211b9da573fa6d7a9f104 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Nov 2021 15:28:48 +0100 Subject: [PATCH 053/101] chg: [auditLog] Fetch field required for model info --- app/Model/Behavior/AuditLogBehavior.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index b05b6f2ee..af22a2cf0 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -94,6 +94,11 @@ class AuditLogBehavior extends ModelBehavior $fieldToFetch[] = 'event_id'; } + // Fetch fields that are necessary to fill object title + if (isset($this->modelInfo[$model->name]) && is_string($this->modelInfo[$model->name]) && !in_array($this->modelInfo[$model->name], $fieldToFetch, true)) { + $fieldToFetch[] = $this->modelInfo[$model->name]; + } + if (empty($fieldToFetch)) { $this->old = null; return true; From 01410b694a30296d969bec337d08a696023ac4f1 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Nov 2021 15:36:51 +0100 Subject: [PATCH 054/101] fix: [test] Ignore beforeRender function --- app/Controller/Component/ACLComponent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 778d60f4d..77276037e 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -970,7 +970,7 @@ class ACLComponent extends Component $fileContents = preg_replace('/\/\*[^\*]+?\*\//', '', $fileContents); preg_match_all($functionFinder, $fileContents, $functionArray); foreach ($functionArray[1] as $function) { - if (substr($function, 0, 1) !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter') { + if ($function[0] !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter' && $function !== 'beforeRender') { $results[$controllerName][] = $function; } } From 776b0d046f42db24d56bccf4d459cae1e856fb63 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 22 Nov 2021 16:32:08 +0100 Subject: [PATCH 055/101] fix: [auditLog] Warning when deleting event --- app/Model/Behavior/AuditLogBehavior.php | 42 ++++++++++++++----------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php index b05b6f2ee..77f0535cb 100644 --- a/app/Model/Behavior/AuditLogBehavior.php +++ b/app/Model/Behavior/AuditLogBehavior.php @@ -4,7 +4,10 @@ App::uses('AuditLog', 'Model'); class AuditLogBehavior extends ModelBehavior { /** @var array|null */ - private $old; + private $beforeSave; + + /** @var array|null */ + private $beforeDelete; /** @var AuditLog|null */ private $AuditLog; @@ -95,19 +98,19 @@ class AuditLogBehavior extends ModelBehavior } if (empty($fieldToFetch)) { - $this->old = null; + $this->beforeSave = null; return true; } } if ($model->id) { - $this->old = $model->find('first', [ + $this->beforeSave = $model->find('first', [ 'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id], 'recursive' => -1, 'callbacks' => false, 'fields' => $fieldToFetch, ]); } else { - $this->old = null; + $this->beforeSave = null; } return true; } @@ -128,13 +131,13 @@ class AuditLogBehavior extends ModelBehavior if (isset($data['deleted'])) { if ($data['deleted']) { $action = AuditLog::ACTION_SOFT_DELETE; - } else if (isset($this->old[$model->alias]['deleted']) && $this->old[$model->alias]['deleted']) { + } else if (isset($this->beforeSave[$model->alias]['deleted']) && $this->beforeSave[$model->alias]['deleted']) { $action = AuditLog::ACTION_UNDELETE; } } } - $changedFields = $this->changedFields($model, $options['fieldList']); + $changedFields = $this->changedFields($model, $this->beforeSave, $options['fieldList']); if (empty($changedFields)) { return; } @@ -143,8 +146,8 @@ class AuditLogBehavior extends ModelBehavior $eventId = $id; } else if (isset($data['event_id'])) { $eventId = $data['event_id']; - } else if (isset($this->old[$model->alias]['event_id'])) { - $eventId = $this->old[$model->alias]['event_id']; + } else if (isset($this->beforeSave[$model->alias]['event_id'])) { + $eventId = $this->beforeSave[$model->alias]['event_id']; } else { $eventId = null; } @@ -153,11 +156,11 @@ class AuditLogBehavior extends ModelBehavior if (isset($this->modelInfo[$model->name])) { $modelTitleField = $this->modelInfo[$model->name]; if (is_callable($modelTitleField)) { - $modelTitle = $modelTitleField($data, isset($this->old[$model->alias]) ? $this->old[$model->alias] : []); + $modelTitle = $modelTitleField($data, isset($this->beforeSave[$model->alias]) ? $this->beforeSave[$model->alias] : []); } else if (isset($data[$modelTitleField])) { $modelTitle = $data[$modelTitleField]; - } else if ($this->old[$model->alias][$modelTitleField]) { - $modelTitle = $this->old[$model->alias][$modelTitleField]; + } else if ($this->beforeSave[$model->alias][$modelTitleField]) { + $modelTitle = $this->beforeSave[$model->alias][$modelTitleField]; } } @@ -196,6 +199,8 @@ class AuditLogBehavior extends ModelBehavior 'event_id' => $eventId, 'change' => $changedFields, ]]); + + $this->beforeSave = null; // cleanup } public function beforeDelete(Model $model, $cascade = true) @@ -204,7 +209,7 @@ class AuditLogBehavior extends ModelBehavior return true; } - $this->old = $model->find('first', [ + $this->beforeDelete = $model->find('first', [ 'conditions' => array($model->alias . '.' . $model->primaryKey => $model->id), 'recursive' => -1, 'callbacks' => false, @@ -217,8 +222,8 @@ class AuditLogBehavior extends ModelBehavior if (!$this->enabled) { return; } - $model->data = $this->old; - $this->old = null; + $model->data = $this->beforeDelete; + $this->beforeDelete = null; if ($model->name === 'Event') { $eventId = $model->id; } else { @@ -264,7 +269,7 @@ class AuditLogBehavior extends ModelBehavior 'model_id' => $id, 'model_title' => $modelTitle, 'event_id' => $eventId, - 'change' => $this->changedFields($model), + 'change' => $this->changedFields($model, null), ]]); } @@ -309,10 +314,11 @@ class AuditLogBehavior extends ModelBehavior /** * @param Model $model + * @param array|null $oldData Array with alias * @param array|null $fieldsToSave * @return array */ - private function changedFields(Model $model, $fieldsToSave = null) + private function changedFields(Model $model, $oldData, $fieldsToSave = null) { $dbFields = $model->schema(); $changedFields = []; @@ -328,8 +334,8 @@ class AuditLogBehavior extends ModelBehavior continue; } - if ($hasPrimaryField && isset($this->old[$model->alias][$key])) { - $old = $this->old[$model->alias][$key]; + if ($hasPrimaryField && isset($oldData[$model->alias][$key])) { + $old = $oldData[$model->alias][$key]; } else { $old = null; } From fb406a05dc55de78138c2cce2fd5f425ff9a9908 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 23 Nov 2021 14:09:25 +0100 Subject: [PATCH 056/101] fix: [backgroundjob] Support of legacy system --- app/Console/Command/StartWorkerShell.php | 2 +- app/Lib/Tools/BackgroundJobs/BackgroundJob.php | 2 +- app/Lib/Tools/BackgroundJobs/Worker.php | 2 +- app/Lib/Tools/BackgroundJobsTool.php | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php index e46f9e16f..0726f3433 100644 --- a/app/Console/Command/StartWorkerShell.php +++ b/app/Console/Command/StartWorkerShell.php @@ -16,7 +16,7 @@ class StartWorkerShell extends AppShell /** @var int */ private $maxExecutionTime; - private const DEFAULT_MAX_EXECUTION_TIME = 86400; // 1 day + const DEFAULT_MAX_EXECUTION_TIME = 86400; // 1 day public function initialize(): void { diff --git a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php index 758e55289..d50010372 100644 --- a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php +++ b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php @@ -4,7 +4,7 @@ declare(strict_types=1); class BackgroundJob implements JsonSerializable { - public const + const STATUS_WAITING = 1, STATUS_RUNNING = 2, STATUS_FAILED = 3, diff --git a/app/Lib/Tools/BackgroundJobs/Worker.php b/app/Lib/Tools/BackgroundJobs/Worker.php index 232957ec7..367751673 100644 --- a/app/Lib/Tools/BackgroundJobs/Worker.php +++ b/app/Lib/Tools/BackgroundJobs/Worker.php @@ -38,7 +38,7 @@ class Worker implements JsonSerializable */ private $status; - public const + const STATUS_RUNNING = 1, STATUS_FAILED = 2, STATUS_UNKNOWN = 3; diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 1687d9c3e..05d12bbb6 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -44,23 +44,23 @@ class BackgroundJobsTool /** @var \Supervisor\Supervisor */ private $Supervisor; - public const MISP_WORKERS_PROCESS_GROUP = 'misp-workers'; + const MISP_WORKERS_PROCESS_GROUP = 'misp-workers'; - public const + const STATUS_RUNNING = 0, STATUS_NOT_ENABLED = 1, STATUS_REDIS_NOT_OK = 2, STATUS_SUPERVISOR_NOT_OK = 3, STATUS_REDIS_AND_SUPERVISOR_NOT_OK = 4; - public const + const DEFAULT_QUEUE = 'default', EMAIL_QUEUE = 'email', CACHE_QUEUE = 'cache', PRIO_QUEUE = 'prio', UPDATE_QUEUE = 'update'; - public const VALID_QUEUES = [ + const VALID_QUEUES = [ self::DEFAULT_QUEUE, self::EMAIL_QUEUE, self::CACHE_QUEUE, @@ -68,24 +68,24 @@ class BackgroundJobsTool self::UPDATE_QUEUE, ]; - public const + const CMD_EVENT = 'event', CMD_SERVER = 'server', CMD_ADMIN = 'admin'; - public const ALLOWED_COMMANDS = [ + const ALLOWED_COMMANDS = [ self::CMD_EVENT, self::CMD_SERVER, self::CMD_ADMIN ]; - public const CMD_TO_SHELL_DICT = [ + const CMD_TO_SHELL_DICT = [ self::CMD_EVENT => 'EventShell', self::CMD_SERVER => 'ServerShell', self::CMD_ADMIN => 'AdminShell' ]; - public const JOB_STATUS_PREFIX = 'job_status'; + const JOB_STATUS_PREFIX = 'job_status'; /** @var array */ private $settings; From fd98014355882c0b69b8ea95873804cfd503d3ad Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 23 Nov 2021 14:33:42 +0100 Subject: [PATCH 057/101] fix: [tools:backgroundjob] Support of legacy systems (2) --- app/Lib/Tools/BackgroundJobsTool.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 05d12bbb6..6c01a70d9 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -212,10 +212,9 @@ class BackgroundJobsTool * Must be less than your configured `read_write_timeout` * for the redis connection. * - * @return BackgroundJob|null. * @throws Exception */ - public function dequeue($queue, int $timeout = 30): ?BackgroundJob + public function dequeue($queue, int $timeout = 30) { $this->validateQueue($queue); @@ -233,10 +232,9 @@ class BackgroundJobsTool * * @param string $jobId Background Job Id. * - * @return BackgroundJob|null job status. * */ - public function getJob(string $jobId): ?BackgroundJob + public function getJob(string $jobId) { $rawJob = $this->RedisConnection->get( self::JOB_STATUS_PREFIX . ':' . $jobId From a968150018f24c24525fe9c5c501d1b6a96c2e8b Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 23 Nov 2021 14:42:14 +0100 Subject: [PATCH 058/101] fix: [tools:backgroundjob] Support of legacy systems (3) --- app/Console/Command/StartWorkerShell.php | 6 +++--- app/Lib/Tools/BackgroundJobs/BackgroundJob.php | 10 +++++----- app/Lib/Tools/BackgroundJobs/Worker.php | 4 ++-- app/Lib/Tools/BackgroundJobsTool.php | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php index 0726f3433..5902cf498 100644 --- a/app/Console/Command/StartWorkerShell.php +++ b/app/Console/Command/StartWorkerShell.php @@ -18,7 +18,7 @@ class StartWorkerShell extends AppShell const DEFAULT_MAX_EXECUTION_TIME = 86400; // 1 day - public function initialize(): void + public function initialize() { parent::initialize(); $this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs')); @@ -45,7 +45,7 @@ class StartWorkerShell extends AppShell return $parser; } - public function main(): void + public function main() { $this->worker = new Worker( [ @@ -100,7 +100,7 @@ class StartWorkerShell extends AppShell * * @return void */ - private function checkMaxExecutionTime(): void + private function checkMaxExecutionTime() { if ($this->maxExecutionTime === 0) { return; diff --git a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php index d50010372..fcf40f13b 100644 --- a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php +++ b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php @@ -171,27 +171,27 @@ class BackgroundJob implements JsonSerializable return $this->returnCode; } - public function setStatus(int $status): void + public function setStatus(int $status) { $this->status = $status; } - public function setOutput(?string $output): void + public function setOutput(?string $output) { $this->output = $output; } - public function setError(?string $error): void + public function setError(?string $error) { $this->error = $error; } - public function setProgress(int $progress): void + public function setProgress(int $progress) { $this->progress = $progress; } - public function setUpdatedAt(int $updatedAt): void + public function setUpdatedAt(int $updatedAt) { $this->updatedAt = $updatedAt; } diff --git a/app/Lib/Tools/BackgroundJobs/Worker.php b/app/Lib/Tools/BackgroundJobs/Worker.php index 367751673..90a5c27f9 100644 --- a/app/Lib/Tools/BackgroundJobs/Worker.php +++ b/app/Lib/Tools/BackgroundJobs/Worker.php @@ -95,12 +95,12 @@ class Worker implements JsonSerializable return $this->status; } - public function setStatus(int $status): void + public function setStatus(int $status) { $this->status = $status; } - public function setUpdatedAt(int $updatedAt): void + public function setUpdatedAt(int $updatedAt) { $this->updatedAt = $updatedAt; } diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 6c01a70d9..e83b3e38b 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -324,7 +324,7 @@ class BackgroundJobsTool * * @return void */ - public function update(BackgroundJob $job): void + public function update(BackgroundJob $job) { $job->setUpdatedAt(time()); @@ -422,7 +422,7 @@ class BackgroundJobsTool * @param boolean $waitForRestart * @return void */ - public function restartWorkers(bool $waitForRestart = false): void + public function restartWorkers(bool $waitForRestart = false) { $this->getSupervisor()->stopProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); @@ -434,7 +434,7 @@ class BackgroundJobsTool * @param boolean $waitForRestart * @return void */ - public function restartDeadWorkers(bool $waitForRestart = false): void + public function restartDeadWorkers(bool $waitForRestart = false) { $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); } @@ -445,7 +445,7 @@ class BackgroundJobsTool * @param string $queue * @return void */ - public function purgeQueue(string $queue): void + public function purgeQueue(string $queue) { $this->validateQueue($queue); @@ -620,7 +620,7 @@ class BackgroundJobsTool return new \Supervisor\Supervisor($client); } - private function updateJobProcessId(int $jobId, string $processId): void + private function updateJobProcessId(int $jobId, string $processId) { $job = ClassRegistry::init('Job'); $job->id = $jobId; From 939d08a7dd4e9c1fd76125a3d20000aa494557f9 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 24 Nov 2021 14:36:08 +0100 Subject: [PATCH 059/101] chg: [internal] Log when attribute was dropped --- app/Controller/ObjectsController.php | 3 ++- app/Model/MispObject.php | 35 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index c9bd501da..678cf1c20 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -1,6 +1,7 @@ request->data['Object'] = $this->request->data; } if (isset($this->request->data['Object']['data'])) { - $this->request->data = json_decode($this->request->data['Object']['data'], true); + $this->request->data = JsonTool::decode($this->request->data['Object']['data']); } if (isset($this->request->data['Object'])) { $this->request->data = array_merge($this->request->data, $this->request->data['Object']); diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index d50ed1fa7..0672848e4 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -737,7 +737,7 @@ class MispObject extends AppModel * @throws InternalErrorException * @throws Exception */ - public function attributeCleanup($attributes) + public function attributeCleanup(array $attributes) { if (empty($attributes['Attribute'])) { return $attributes; @@ -820,7 +820,7 @@ class MispObject extends AppModel return $object; } - public function deltaMerge($object, $objectToSave, $onlyAddNewAttribute=false, $user) + public function deltaMerge(array $object, array $objectToSave, $onlyAddNewAttribute=false, array $user) { if (!isset($objectToSave['Object'])) { $dataToBackup = array('ObjectReferences', 'Attribute', 'ShadowAttribute'); @@ -851,8 +851,8 @@ class MispObject extends AppModel $object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id']; } } - $date = new DateTime(); - $object['Object']['timestamp'] = $date->getTimestamp(); + $time = time(); + $object['Object']['timestamp'] = $time; $forcedSeenOnElements = array(); if (isset($objectToSave['Object']['first_seen'])) { $forcedSeenOnElements['first_seen'] = $objectToSave['Object']['first_seen']; @@ -886,10 +886,15 @@ class MispObject extends AppModel $newAttribute['id'] = $originalAttribute['id']; $newAttribute['event_id'] = $object['Object']['event_id']; $newAttribute['object_id'] = $object['Object']['id']; - $newAttribute['timestamp'] = $date->getTimestamp(); + $newAttribute['timestamp'] = $time; $result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array('fieldList' => Attribute::EDITABLE_FIELDS)); if ($result) { $this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true); + } else { + $this->loadLog()->createLogEntry($user, 'edit', 'Attribute', $newAttribute['id'], + 'Attribute dropped due to validation for Event ' . $object['Object']['event_id'] . ' failed', + 'Validation errors: ' . json_encode($this->Event->Attribute->validationErrors) . ' Full Attribute: ' . json_encode($newAttribute) + ); } } unset($object['Attribute'][$origKey]); @@ -897,7 +902,6 @@ class MispObject extends AppModel } } } - $this->Event->Attribute->create(); $newAttribute['event_id'] = $object['Object']['event_id']; $newAttribute['object_id'] = $object['Object']['id']; // Set seen of object at attribute level @@ -914,20 +918,23 @@ class MispObject extends AppModel } } if (!isset($newAttribute['distribution'])) { - $newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($newAttribute['distribution'] == 'event') { - $newAttribute['distribution'] = 5; - } + $newAttribute['distribution'] = $this->Event->Attribute->defaultDistribution(); } + $this->Event->Attribute->create(); $saveResult = $this->Event->Attribute->save($newAttribute); if ($saveResult) { $newAttribute['id'] = $this->Event->Attribute->id; $this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true); + } else { + $this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0, + 'Attribute dropped due to validation for Event ' . $object['Object']['event_id'] . ' failed', + 'Validation errors: ' . json_encode($this->Event->Attribute->validationErrors) . ' Full Attribute: ' . json_encode($newAttribute) + ); } $attributeArrays['add'][] = $newAttribute; unset($objectToSave['Attribute'][$newKey]); } - foreach ($object['Attribute'] as $origKey => $originalAttribute) { + foreach ($object['Attribute'] as $originalAttribute) { $originalAttribute['deleted'] = 1; $this->Event->Attribute->save($originalAttribute, array('fieldList' => Attribute::EDITABLE_FIELDS)); } @@ -951,12 +958,6 @@ class MispObject extends AppModel $newAttribute['last_seen'] = $object['Object']['last_seen']; $different = true; } - if (!isset($newAttribute['distribution'])) { - $newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($newAttribute['distribution'] == 'event') { - $newAttribute['distribution'] = 5; - } - } $saveAttributeResult = $this->Attribute->saveAttributes(array($newAttribute), $user); return $saveAttributeResult ? $this->id : $this->validationErrors; } From d32c4c0c5e7785ab663d97474d21f709acfe1fa0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 24 Nov 2021 15:03:48 +0100 Subject: [PATCH 060/101] fix: [UI] Adding attributes to object --- app/Controller/ObjectsController.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index 678cf1c20..1ca8139b2 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -62,7 +62,7 @@ class ObjectsController extends AppController } $multiple_template_elements = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[multiple=true]')); $multiple_attribute_allowed = array(); - foreach ($multiple_template_elements as $k => $template_element) { + foreach ($multiple_template_elements as $template_element) { $relation_type = $template_element['object_relation'] . ':' . $template_element['type']; $multiple_attribute_allowed[$relation_type] = true; } @@ -90,6 +90,9 @@ class ObjectsController extends AppController if (isset($this->request->data['Attribute'])) { foreach ($this->request->data['Attribute'] as &$attribute) { + if (empty($attribute['uuid'])) { + $attribute['uuid'] = CakeText::uuid(); + } $validation = $this->MispObject->Attribute->validateAttribute($attribute, false); if ($validation !== true) { $attribute['validation'] = $validation; From d20a68dc38bc9fa27e2eefa9de571dd2c3f1a9c0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 25 Nov 2021 10:10:04 +0100 Subject: [PATCH 061/101] fix: [internal] Fixes #7961 --- app/Model/Attribute.php | 42 ++++++++++++++++++++++++---------------- app/Model/Event.php | 4 ++-- app/Model/MispObject.php | 11 ++--------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index b439a1af4..42009f8b4 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -9,6 +9,7 @@ App::uses('AttachmentTool', 'Tools'); App::uses('TmpFileTool', 'Tools'); App::uses('ComplexTypeTool', 'Tools'); App::uses('AttributeValidationTool', 'Tools'); +App::uses('JsonTool', 'Tools'); /** * @property Event $Event @@ -1720,7 +1721,6 @@ class Attribute extends AppModel } } - public function checkTemplateAttributes($template, $data, $event_id) { $result = array(); @@ -2874,10 +2874,7 @@ class Attribute extends AppModel if (!$this->Warninglist->filterWarninglistAttribute($attribute)) { $this->validationErrors['warninglist'] = 'Attribute could not be saved as it trips over a warninglist and enforceWarninglist is enforced.'; $validationErrors = $this->validationErrors['warninglist']; - $this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0, - 'Attribute dropped due to validation for Event ' . $eventId . ' failed', - 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute) - ); + $this->logDropped($user, $attribute); return $attribute; } } @@ -2901,12 +2898,8 @@ class Attribute extends AppModel unset($attribute['sharing_group_id']); } } - if (!$this->save($attribute, $params)) { - $attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A'); - $this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0, - 'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short, - 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute) - ); + if (!$this->save(['Attribute' => $attribute], $params)) { + $this->logDropped($user, $attribute); } else { if (!empty($attribute['AttributeTag'])) { $toSave = []; @@ -3023,12 +3016,8 @@ class Attribute extends AppModel $fieldList[] = 'object_id'; $fieldList[] = 'object_relation'; } - if (!$this->save($attribute, ['fieldList' => $fieldList, 'parentEvent' => $event])) { - $attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A'); - $this->loadLog()->createLogEntry($user, 'edit', 'Attribute', 0, - 'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short, - 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute) - ); + if (!$this->save(['Attribute' => $attribute], ['fieldList' => $fieldList, 'parentEvent' => $event])) { + $this->logDropped($user, $attribute, 'edit'); return $this->validationErrors; } if (!empty($attribute['Sighting'])) { @@ -3458,6 +3447,25 @@ class Attribute extends AppModel return $distribution; } + /** + * Log when attribute was dropped due to validation errors. + * + * @param array $user + * @param array $attribute + * @param string $action + * @throws JsonException + */ + public function logDropped(array $user, array $attribute, $action = 'add') + { + $attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A'); + $eventId = $attribute['event_id']; + $modelId = $action === 'add' ? 0 : $this->id; + $this->loadLog()->createLogEntry($user, 'add', 'Attribute', $modelId, + "Attribute dropped due to validation for Event $eventId failed: $attribute_short", + 'Validation errors: ' . JsonTool::encode($this->validationErrors) . ' Full Attribute: ' . JsonTool::encode($attribute) + ); + } + public function __isset($name) { if ($name === 'typeDefinitions' || $name === 'categoryDefinitions') { diff --git a/app/Model/Event.php b/app/Model/Event.php index 79956b8c3..77c341a4e 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3963,7 +3963,7 @@ class Event extends AppModel $server['Server']['internal'] = false; } // If the event exists... - if (count($existingEvent)) { + if (!empty($existingEvent)) { $data['Event']['id'] = $existingEvent['Event']['id']; $id = $existingEvent['Event']['id']; // Conditions affecting all: @@ -4008,7 +4008,7 @@ class Event extends AppModel $changed = false; // If a field is not set in the request, just reuse the old value // Also, compare the event to the existing event and see whether this is a meaningful change - $recoverFields = array('analysis', 'threat_level_id', 'info', 'distribution', 'date'); + $recoverFields = array('analysis', 'threat_level_id', 'info', 'distribution', 'date', 'org_id'); foreach ($recoverFields as $rF) { if (!isset($data['Event'][$rF])) { $data['Event'][$rF] = $existingEvent['Event'][$rF]; diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 80863c123..4c35972ed 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -888,10 +888,7 @@ class MispObject extends AppModel if ($result) { $this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true); } else { - $this->loadLog()->createLogEntry($user, 'edit', 'Attribute', $newAttribute['id'], - 'Attribute dropped due to validation for Event ' . $object['Object']['event_id'] . ' failed', - 'Validation errors: ' . json_encode($this->Event->Attribute->validationErrors) . ' Full Attribute: ' . json_encode($newAttribute) - ); + $this->Event->Attribute->logDropped($user, $newAttribute, 'edit'); } } unset($object['Attribute'][$origKey]); @@ -923,12 +920,8 @@ class MispObject extends AppModel $newAttribute['id'] = $this->Event->Attribute->id; $this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true); } else { - $this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0, - 'Attribute dropped due to validation for Event ' . $object['Object']['event_id'] . ' failed', - 'Validation errors: ' . json_encode($this->Event->Attribute->validationErrors) . ' Full Attribute: ' . json_encode($newAttribute) - ); + $this->Event->Attribute->logDropped($user, $newAttribute, 'add'); } - $attributeArrays['add'][] = $newAttribute; unset($objectToSave['Attribute'][$newKey]); } foreach ($object['Attribute'] as $originalAttribute) { From cab5262d65ce25b8ddb945d2d93ec17baf3c33a5 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 25 Nov 2021 10:29:16 +0100 Subject: [PATCH 062/101] fix: improve error handling when supervisor is not available or connection settings are wrong --- app/Lib/Tools/BackgroundJobsTool.php | 26 +++++++++++++++++--- app/Model/Server.php | 10 ++++++++ app/View/Elements/healthElements/workers.ctp | 9 ++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index e83b3e38b..faeab945f 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -278,9 +278,19 @@ class BackgroundJobsTool */ public function getWorkers(): array { - $workers = []; - $procs = $this->getSupervisor()->getAllProcesses(); + try { + $procs = $this->getSupervisor()->getAllProcesses(); + } catch (\Exception $exception) { + CakeLog::error( + "An error occured when getting the workers statuses via Supervisor API: {$exception->getMessage()}", + 0, + $exception + ); + return []; + } + + $workers = []; foreach ($procs as $proc) { if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) { if ($proc->offsetGet('pid') > 0) { @@ -471,7 +481,7 @@ class BackgroundJobsTool } try { - $supervisorStatus = $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING; + $supervisorStatus = $this->getSupervisorStatus(); } catch (Exception $exception) { CakeLog::error("SimpleBackgroundJobs Supervisor error: {$exception->getMessage()}"); $supervisorStatus = false; @@ -488,6 +498,16 @@ class BackgroundJobsTool } } + /** + * Return true if Supervisor process is running. + * + * @return boolean + */ + public function getSupervisorStatus(): bool + { + return $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING; + } + /** * Validate queue * diff --git a/app/Model/Server.php b/app/Model/Server.php index 3285662f4..0f448c227 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3443,6 +3443,16 @@ class Server extends AppModel if (Configure::check('MISP.manage_workers')) { $worker_array['controls'] = Configure::read('MISP.manage_workers'); } + + if (Configure::read('SimpleBackgroundJobs.enabled')) { + try { + $worker_array['supervisord_status'] = $this->getBackgroundJobsTool()->getSupervisorStatus(); + } catch (Exception $exception) { + $this->logException('Error getting supervisor status.', $exception); + $worker_array['supervisord_status'] = false; + } + } + return $worker_array; } diff --git a/app/View/Elements/healthElements/workers.ctp b/app/View/Elements/healthElements/workers.ctp index b8d34c924..5f19f6b85 100644 --- a/app/View/Elements/healthElements/workers.ctp +++ b/app/View/Elements/healthElements/workers.ctp @@ -1,10 +1,17 @@
-
:
+
:
+
:
From 08c66ed16546af0738a682a1f37728299dcdd3c0 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 25 Nov 2021 14:34:10 +0100 Subject: [PATCH 063/101] fix: wrong params --- app/Lib/Tools/BackgroundJobsTool.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index faeab945f..6eb369014 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -281,12 +281,7 @@ class BackgroundJobsTool try { $procs = $this->getSupervisor()->getAllProcesses(); } catch (\Exception $exception) { - CakeLog::error( - "An error occured when getting the workers statuses via Supervisor API: {$exception->getMessage()}", - 0, - $exception - ); - + CakeLog::error("An error occured when getting the workers statuses via Supervisor API: {$exception->getMessage()}"); return []; } From 8d7df612f323a85ae50ab32c7a970033724829eb Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 25 Nov 2021 18:15:22 +0100 Subject: [PATCH 064/101] new: [CLI] Get authkey info by `cake user authkey` --- app/Console/Command/UserShell.php | 52 +++++++++++++++++++++++++++++++ app/Model/AuthKey.php | 25 +++++++++------ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/app/Console/Command/UserShell.php b/app/Console/Command/UserShell.php index e7983fb7a..fdbe9c7dc 100644 --- a/app/Console/Command/UserShell.php +++ b/app/Console/Command/UserShell.php @@ -19,6 +19,14 @@ class UserShell extends AppShell ], ] ]); + $parser->addSubcommand('authkey', [ + 'help' => __('Get information about given authkey.'), + 'parser' => [ + 'arguments' => [ + 'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')], + ], + ] + ]); $parser->addSubcommand('block', [ 'help' => __('Immediately block user.'), 'parser' => [ @@ -111,6 +119,50 @@ class UserShell extends AppShell } } + public function authkey() + { + if (isset($this->args[0])) { + $authkey = $this->args[0]; + } else { + $authkey = fgets(STDIN); // read line from STDIN + } + $authkey = trim($authkey); + if (strlen($authkey) !== 40) { + $this->error('Authkey has not valid format.'); + } + if (Configure::read('Security.advanced_authkeys')) { + $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey, true); + if (empty($user)) { + $this->error("Given authkey doesn't belong to any user."); + } + + $isExpired = $user['authkey_expiration'] && $user['authkey_expiration'] < time(); + + $this->out($this->json([ + 'user_id' => $user['id'], + 'email' => $user['email'], + 'org_id' => $user['org_id'], + 'authkey_id' => $user['authkey_id'], + 'authkey_expiration' => $user['authkey_expiration'], + 'authkey_expired' => $isExpired, + 'allowed_ips' => $user['allowed_ips'], + 'authkey_read_only' => $user['authkey_read_only'], + ])); + + $this->_stop($isExpired ? 2 : 0); + } else { + $user = $this->User->getAuthUserByAuthkey($authkey); + if (empty($user)) { + $this->error("Given authkey doesn't belong to any user."); + } + $this->out($this->json([ + 'user_id' => $user['id'], + 'email' => $user['email'], + 'org_id' => $user['org_id'], + ])); + } + } + public function block() { list($userId) = $this->args; diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php index 3c98512b7..63990e97f 100644 --- a/app/Model/AuthKey.php +++ b/app/Model/AuthKey.php @@ -129,23 +129,30 @@ class AuthKey extends AppModel /** * @param string $authkey + * @param bool $includeExpired * @return array|false */ - public function getAuthUserByAuthKey($authkey) + public function getAuthUserByAuthKey($authkey, $includeExpired = false) { $start = substr($authkey, 0, 4); $end = substr($authkey, -4); + + $conditions = [ + 'authkey_start' => $start, + 'authkey_end' => $end, + ]; + + if (!$includeExpired) { + $conditions['OR'] = [ + 'expiration >' => time(), + 'expiration' => 0 + ]; + } + $possibleAuthkeys = $this->find('all', [ 'recursive' => -1, 'fields' => ['id', 'authkey', 'user_id', 'expiration', 'allowed_ips', 'read_only'], - 'conditions' => [ - 'OR' => [ - 'expiration >' => time(), - 'expiration' => 0 - ], - 'authkey_start' => $start, - 'authkey_end' => $end, - ] + 'conditions' => $conditions, ]); $passwordHasher = $this->getHasher(); foreach ($possibleAuthkeys as $possibleAuthkey) { From 019bba81af4a07e06db7f5baa3643818f0376f9f Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Fri, 26 Nov 2021 11:45:10 +0100 Subject: [PATCH 065/101] fix: show error message instead of fatal error when diagnostics tool fails to run --- app/Model/Server.php | 28 ++++++++++++++++--- .../Elements/healthElements/diagnostics.ctp | 12 ++++++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 0f448c227..24b330ada 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3204,9 +3204,18 @@ class Server extends AppModel public function yaraDiagnostics(&$diagnostic_errors) { $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'yaratest.py'; - $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); - $scriptResult = json_decode($scriptResult, true); - return array('operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara']); + try { + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); + $scriptResult = json_decode($scriptResult, true); + } catch (Exception $exception) { + $this->logException('Failed to run yara diagnostics.', $exception); + return array( + 'operational' => 0, + 'plyara' => 0, + 'test_run' => false + ); + } + return array('operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara'], 'test_run' => true); } public function stixDiagnostics(&$diagnostic_errors) @@ -3214,7 +3223,17 @@ class Server extends AppModel $expected = array('stix' => '>1.2.0.11', 'cybox' => '>2.1.0.21', 'mixbox' => '>1.0.5', 'maec' => '>4.1.0.17', 'stix2' => '>3.0.0', 'pymisp' => '>2.4.120'); // check if the STIX and Cybox libraries are working using the test script stixtest.py $scriptFile = APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'; - $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); + try { + $scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]); + } catch (Exception $exception) { + $this->logException('Failed to run STIX diagnostics.', $exception); + return [ + 'operational' => 0, + 'invalid_version' => false, + 'test_run' => false + ]; + } + try { $scriptResult = $this->jsonDecode($scriptResult); } catch (Exception $e) { @@ -3237,6 +3256,7 @@ class Server extends AppModel $result = [ 'operational' => $scriptResult['operational'], 'invalid_version' => false, + 'test_run' => true ]; foreach ($expected as $package => $expectedVersion) { $result[$package]['version'] = $scriptResult[$package]; diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index 19a0e7ec6..0c851269c 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -335,7 +335,9 @@ : - + + + @@ -369,10 +371,14 @@
' . $message . ''; ?> From 1f79573fd7059f8b9ad073e50c634b6e0d598107 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 30 Nov 2021 12:55:08 +0100 Subject: [PATCH 066/101] chg: [feeds] Support for sharing groups with feeds, fixes #5758 --- app/Controller/FeedsController.php | 6 +- app/Model/Feed.php | 61 +++++++++++++------ .../Elements/Feeds/View/row_attribute.ctp | 9 +++ app/View/Elements/Feeds/View/row_object.ctp | 1 + app/View/Elements/Feeds/eventattribute.ctp | 1 + app/View/Feeds/preview_event.ctp | 9 +++ 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index f46b17145..7af81bb0c 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -124,7 +124,9 @@ class FeedsController extends AppController 'menuItem' => 'index' ]); $this->loadModel('Event'); - $this->set('distributionLevels', $this->Event->distributionLevels); + $distributionLevels = $this->Event->distributionLevels; + $distributionLevels[5] = 'Inherit from feed'; + $this->set('distributionLevels', $distributionLevels); $this->set('scope', $scope); } @@ -286,6 +288,7 @@ class FeedsController extends AppController $this->loadModel('Event'); $sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $distributionLevels = $this->Event->distributionLevels; + $distributionLevels[5] = 'Inherit from feed'; if (empty($sharingGroups)) { unset($distributionLevels[4]); } @@ -424,6 +427,7 @@ class FeedsController extends AppController $this->loadModel('Event'); $sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $distributionLevels = $this->Event->distributionLevels; + $distributionLevels[5] = 'Inherit from feed'; if (empty($sharingGroups)) { unset($distributionLevels[4]); } diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 575b155c2..f85c038ae 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -915,11 +915,49 @@ class Feed extends AppModel $event['Event']['Orgc'] = $event['Event']['orgc']; unset($event['Event']['orgc']); } - $event['Event']['distribution'] = $feed['Feed']['distribution']; - $event['Event']['sharing_group_id'] = $feed['Feed']['sharing_group_id']; - if (!empty($event['Event']['Attribute'])) { - foreach ($event['Event']['Attribute'] as $key => $attribute) { - $event['Event']['Attribute'][$key]['distribution'] = 5; + if (5 == $feed['Feed']['distribution'] && isset($event['Event']['distribution'])) { + // We inherit the distribution from the feed and should not rewrite the distributions. + // MISP magically parses the Sharing Group info and creates the SG if it didn't exist. + } else { + // rewrite the distributions to the one configured by the Feed settings + // overwrite Event distribution + if (5 == $feed['Feed']['distribution']) { + // We said to inherit the distribution from the feed, but the feed does not contain distribution + // rewrite the event to My org only distribution, and set all attributes to inherit the event distribution + $event['Event']['distribution'] = 0; + $event['Event']['sharing_group_id'] = 0; + } else { + $event['Event']['distribution'] = $feed['Feed']['distribution']; + $event['Event']['sharing_group_id'] = $feed['Feed']['sharing_group_id']; + if ($feed['Feed']['sharing_group_id']) { + $sg = $this->SharingGroup->find('first', array( + 'recursive' => -1, + 'conditions' => array('SharingGroup.id' => $feed['Feed']['sharing_group_id']) + )); + if (!empty($sg)) { + $event['Event']['SharingGroup'] = $sg['SharingGroup']; + } else { + // We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back. + $event['Event']['distribution'] = 0; + $event['Event']['sharing_group_id'] = 0; + } + } + } + // overwrite Attributes and Objects distribution to Inherit + if (!empty($event['Event']['Attribute'])) { + foreach ($event['Event']['Attribute'] as $key => $attribute) { + $event['Event']['Attribute'][$key]['distribution'] = 5; + } + } + if (!empty($event['Event']['Object'])) { + foreach ($event['Event']['Object'] as $key => $object) { + $event['Event']['Object'][$key]['distribution'] = 5; + if (!empty($event['Event']['Object'][$key]['Attribute'])) { + foreach ($event['Event']['Object'][$key]['Attribute'] as $a_key => $attribute) { + $event['Event']['Object'][$key]['Attribute'][$a_key]['distribution'] = 5; + } + } + } } } if ($feed['Feed']['tag_id']) { @@ -940,19 +978,6 @@ class Feed extends AppModel } } } - if ($feed['Feed']['sharing_group_id']) { - $sg = $this->SharingGroup->find('first', array( - 'recursive' => -1, - 'conditions' => array('SharingGroup.id' => $feed['Feed']['sharing_group_id']) - )); - if (!empty($sg)) { - $event['Event']['SharingGroup'] = $sg['SharingGroup']; - } else { - // We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back. - $event['Event']['distribution'] = 0; - $event['Event']['sharing_group_id'] = 0; - } - } if (!$this->__checkIfEventBlockedByFilter($event, $filterRules)) { return 'blocked'; } diff --git a/app/View/Elements/Feeds/View/row_attribute.ctp b/app/View/Elements/Feeds/View/row_attribute.ctp index 97b7c4476..2d82418dd 100755 --- a/app/View/Elements/Feeds/View/row_attribute.ctp +++ b/app/View/Elements/Feeds/View/row_attribute.ctp @@ -80,6 +80,15 @@   + + +
diff --git a/app/View/Elements/Feeds/View/row_object.ctp b/app/View/Elements/Feeds/View/row_object.ctp index afd29ed35..22bb5bfda 100644 --- a/app/View/Elements/Feeds/View/row_object.ctp +++ b/app/View/Elements/Feeds/View/row_object.ctp @@ -37,6 +37,7 @@     +   Paginator->sort('comment');?> + Paginator->sort('to_ids', 'IDS');?> $eventDescriptions['analysis']['desc'], 'value' => $analysisLevels[$event['Event']['analysis']], ]; +if ($event['Event']['distribution'] == 4) { + $distributionText = $event['SharingGroup']['name']; +} else { + $distributionText = $distributionLevels[$event['Event']['distribution']]; +} +$tableData[] = [ + 'key' => __('Distribution'), + 'value' => $distributionText +]; $tableData[] = [ 'key' => __('Info'), 'value' => $event['Event']['info'] From e905817768e0825d1551415d71b909e23337c728 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 30 Nov 2021 15:04:19 +0100 Subject: [PATCH 067/101] fix: [feeds] preview attribute distribution - escape sharing group name --- app/View/Elements/Feeds/View/row_attribute.ctp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/View/Elements/Feeds/View/row_attribute.ctp b/app/View/Elements/Feeds/View/row_attribute.ctp index 2d82418dd..c942a493a 100755 --- a/app/View/Elements/Feeds/View/row_attribute.ctp +++ b/app/View/Elements/Feeds/View/row_attribute.ctp @@ -83,9 +83,9 @@ From 175a0b204f857633d35ad3f805588f09667fd7b5 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 30 Nov 2021 15:04:48 +0100 Subject: [PATCH 068/101] fix: [feeds] i18n some strings --- app/Controller/FeedsController.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index 7af81bb0c..90616ac36 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -125,7 +125,7 @@ class FeedsController extends AppController ]); $this->loadModel('Event'); $distributionLevels = $this->Event->distributionLevels; - $distributionLevels[5] = 'Inherit from feed'; + $distributionLevels[5] = __('Inherit from feed'); $this->set('distributionLevels', $distributionLevels); $this->set('scope', $scope); } @@ -288,7 +288,7 @@ class FeedsController extends AppController $this->loadModel('Event'); $sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $distributionLevels = $this->Event->distributionLevels; - $distributionLevels[5] = 'Inherit from feed'; + $distributionLevels[5] = __('Inherit from feed'); if (empty($sharingGroups)) { unset($distributionLevels[4]); } @@ -427,7 +427,7 @@ class FeedsController extends AppController $this->loadModel('Event'); $sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $distributionLevels = $this->Event->distributionLevels; - $distributionLevels[5] = 'Inherit from feed'; + $distributionLevels[5] = __('Inherit from feed'); if (empty($sharingGroups)) { unset($distributionLevels[4]); } From 72548fd9a432607676d22b5b82653f2b2f99e57d Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 2 Dec 2021 14:03:20 +0100 Subject: [PATCH 069/101] fix: [UI] Ajax forms lose persistence - generic Form builder now has the persistence baked in - capture all form fields' data before submiting as expected --- app/Controller/AppController.php | 2 +- app/View/Elements/genericElements/Form/genericForm.ctp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index bf5953785..92713e757 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -33,7 +33,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '131'; + private $__queryVersion = '132'; public $pyMispVersion = '2.4.151'; public $phpmin = '7.2'; public $phprec = '7.4'; diff --git a/app/View/Elements/genericElements/Form/genericForm.ctp b/app/View/Elements/genericElements/Form/genericForm.ctp index 7f87f7634..2b9e63820 100644 --- a/app/View/Elements/genericElements/Form/genericForm.ctp +++ b/app/View/Elements/genericElements/Form/genericForm.ctp @@ -39,8 +39,10 @@ $fieldsArrayForPersistence = array(); $formOptions = isset($formOptions) ? $formOptions : array(); $formOptions = array_merge(['class' => 'genericForm'], $formOptions); $formCreate = $this->Form->create($modelForForm, $formOptions); +$fieldsArray = []; if (!empty($data['fields'])) { foreach ($data['fields'] as $fieldData) { + $fieldsArray[] = $modelForForm . Inflector::camelize($fieldData['field']); if (isset($fieldData['requirements']) && !$fieldData['requirements']) { continue; } @@ -124,6 +126,7 @@ if (!empty($ajax)) { ?> \ No newline at end of file diff --git a/app/View/Elements/serverRuleElements/rules_widget.ctp b/app/View/Elements/serverRuleElements/rules_widget.ctp index 5136227a1..df8854926 100644 --- a/app/View/Elements/serverRuleElements/rules_widget.ctp +++ b/app/View/Elements/serverRuleElements/rules_widget.ctp @@ -4,6 +4,7 @@ $pickerDisplayed = false; ?>
+
@@ -28,11 +29,13 @@ $pickerDisplayed = false;
+
+ + + +
@@ -110,6 +126,7 @@ $pickerDisplayed = false;
+
@@ -134,14 +151,17 @@ $pickerDisplayed = false;
+
\ No newline at end of file diff --git a/app/View/Elements/serverRuleElements/rules_filtering_type.ctp b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp new file mode 100644 index 000000000..742495094 --- /dev/null +++ b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp @@ -0,0 +1,57 @@ +
+ + + + +
\ No newline at end of file diff --git a/app/View/Servers/edit.ctp b/app/View/Servers/edit.ctp index 5b258331f..4119f75d1 100644 --- a/app/View/Servers/edit.ctp +++ b/app/View/Servers/edit.ctp @@ -157,14 +157,20 @@ + + + +


+ +

array('colour' => 'green', 'text' => 'allowed'), 'NOT' => array('colour' => 'red', 'text' => 'blocked')); $ruleDescription = array('pull' => '', 'push' => ''); foreach ($syncOptions as $syncOption) { From 3db4a4636bae6589026303628d0d355f20bb7f28 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 09:52:07 +0100 Subject: [PATCH 083/101] chg: [server:synchronisation] Usage of template_uuid instead of the object name --- app/Controller/ServersController.php | 11 +++++++---- app/Model/Event.php | 2 +- app/Model/Server.php | 2 +- app/View/Elements/serverRuleElements/push.ctp | 2 +- .../serverRuleElements/rules_filtering_type.ctp | 2 +- app/View/Servers/edit.ctp | 4 ++-- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 7965255bf..2c4a0219f 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -640,13 +640,16 @@ class ServersController extends AppController $allTypes = []; $this->loadModel('Attribute'); $this->loadModel('ObjectTemplate'); - $objects = $this->ObjectTemplate->find('list', [ - 'fields' => ['name'], - 'group' => ['name', 'id'], + $objects = $this->ObjectTemplate->find('all', [ + 'recursive' => -1, + 'fields' => ['uuid', 'name'], + 'group' => ['uuid', 'name', 'id'], ]); $allTypes = [ 'attribute' => array_unique(Hash::extract(Hash::extract($this->Attribute->categoryDefinitions, '{s}.types'), '{n}.{n}')), - 'object' => $objects + 'object' => Hash::map($objects, '{n}.ObjectTemplate', function ($item) { + return ['id' => $item['uuid'], 'name' => $item['name']]; + }) ]; $oldRemoteSetting = 0; diff --git a/app/Model/Event.php b/app/Model/Event.php index c4478e4c8..001c21b98 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1149,7 +1149,7 @@ class Event extends AppModel $data['Object'][$key] = $this->__prepareAttributesForSync($data['Object'][$key], $server, $pushRules); } } - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['name'], $pushRules['type_objects']['NOT'])) { + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pushRules['type_objects']['NOT'])) { unset($data['Object'][$key]); } $data['Object'] = array_values($data['Object']); diff --git a/app/Model/Server.php b/app/Model/Server.php index e51c4e5fa..3499d2684 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -295,7 +295,7 @@ class Server extends AppModel } if (isset($event['Event']['Object'])) { foreach ($event['Event']['Object'] as $i => $object) { - if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['name'], $pullRules['type_objects']['NOT'])) { + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pullRules['type_objects']['NOT'])) { unset($event['Event']['Object'][$i]); continue; } diff --git a/app/View/Elements/serverRuleElements/push.ctp b/app/View/Elements/serverRuleElements/push.ctp index 6858b8671..94f057169 100644 --- a/app/View/Elements/serverRuleElements/push.ctp +++ b/app/View/Elements/serverRuleElements/push.ctp @@ -63,7 +63,7 @@ ]); ?> - element('serverRuleElements/rules_filtering_type', [ 'technique' => 'push', diff --git a/app/View/Elements/serverRuleElements/rules_filtering_type.ctp b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp index 742495094..d23fd7d1c 100644 --- a/app/View/Elements/serverRuleElements/rules_filtering_type.ctp +++ b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp @@ -45,7 +45,7 @@ 'technique' => $technique, 'allowEmptyOptions' => true, 'options' => $allObjectTypes, - 'optionNoValue' => true, + 'optionNoValue' => false, 'initAllowOptions' => [], 'initBlockOptions' => $objectTypeBlockRules, 'disableAllow' => true, diff --git a/app/View/Servers/edit.ctp b/app/View/Servers/edit.ctp index 4119f75d1..4ca10cfe1 100644 --- a/app/View/Servers/edit.ctp +++ b/app/View/Servers/edit.ctp @@ -159,7 +159,7 @@ - +


@@ -169,7 +169,7 @@ - +

From 7d87fef8ead9727eac3dcfa25dfa6e5fd8052992 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 10:02:49 +0100 Subject: [PATCH 084/101] fix: [events:synchronisation] debug and typos --- app/Model/Event.php | 3 --- app/Model/Server.php | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 001c21b98..bffd6a1b7 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1201,9 +1201,6 @@ class Event extends AppModel $event['Event']['distribution'] = 1; } - debug($event); - throw new Exception("Error Processing Request", 1); - return $event; } diff --git a/app/Model/Server.php b/app/Model/Server.php index 3499d2684..f7d772437 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -5674,9 +5674,9 @@ class Server extends AppModel ], 'enable_synchronisation_filtering_on_type' => [ 'level' => self::SETTING_OPTIONAL, - 'description' => __('Allows server synchronisation to have filtering on Attribute type or Object name. Warning: This feature has some shortcoming and might introduce inconsistency in your MISP instance and those with whom you are synchronising with.'), + 'description' => __('Allows server synchronisation to have filtering on Attribute type or Object name. Warning: This feature has some shortcoming and might introduce inconsistencies in your MISP instance and those with whom you are synchronising with.'), 'value' => false, - 'test' => 'testBool', + 'test' => 'testBoolFalse', 'type' => 'boolean', 'null' => true, ], From 6479f841a18c73526d6ea919426cef6e66e7bcf8 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 10:07:37 +0100 Subject: [PATCH 085/101] chg: [servers:index] Improved UI Only show blocked attribute types/objects if setting is turned on --- app/View/Servers/index.ctp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/View/Servers/index.ctp b/app/View/Servers/index.ctp index 4d35a7d22..2f3b00640 100644 --- a/app/View/Servers/index.ctp +++ b/app/View/Servers/index.ctp @@ -41,7 +41,10 @@ foreach ($servers as $server): $rules['push'] = json_decode($server['Server']['push_rules'], true); $rules['pull'] = json_decode($server['Server']['pull_rules'], true); $syncOptions = array('pull', 'push'); - $fieldOptions = array('tags', 'orgs', 'type_attributes', 'type_objects'); + $fieldOptions = array('tags', 'orgs'); + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) { + $fieldOptions = array_merge($fieldOptions, ['type_attributes', 'type_objects']); + } $typeOptions = array('OR' => array('colour' => 'green', 'text' => 'allowed'), 'NOT' => array('colour' => 'red', 'text' => 'blocked')); $ruleDescription = array('pull' => '', 'push' => ''); foreach ($syncOptions as $syncOption) { From fee5563c5a4221b21e624a9e185dc00eb54e671f Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 10:56:45 +0100 Subject: [PATCH 086/101] chg: [server:pull] Do not log empty event entries if it was cause by the rules --- app/Model/Server.php | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index f7d772437..43ccc2b38 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -240,7 +240,7 @@ class Server extends AppModel * @param array $server * @param array $user */ - private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules) + private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false) { // we have an Event array // The event came from a pull, so it should be locked. @@ -270,6 +270,7 @@ class Server extends AppModel } if (isset($event['Event']['Attribute'])) { + $originalCount = count($event['Event']['Attribute']); foreach ($event['Event']['Attribute'] as $key => $attribute) { if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Attribute'][$key]); @@ -292,8 +293,12 @@ class Server extends AppModel } } } + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) { + $pullRulesEmptiedEvent = true; + } } if (isset($event['Event']['Object'])) { + $originalObjectCount = count($event['Event']['Object']); foreach ($event['Event']['Object'] as $i => $object) { if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pullRules['type_objects']['NOT'])) { unset($event['Event']['Object'][$i]); @@ -308,6 +313,7 @@ class Server extends AppModel break; } if (isset($object['Attribute'])) { + $originalAttributeCount = count($object['Attribute']); foreach ($object['Attribute'] as $j => $a) { if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) { unset($event['Event']['Object'][$i]['Attribute'][$j]['Tag'][$k]); @@ -330,8 +336,14 @@ class Server extends AppModel } } } + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && count($object['Attribute']) == 0) { + $pullRulesEmptiedEvent = true; + } } } + if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) { + $pullRulesEmptiedEvent = true; + } } if (isset($event['Event']['EventReport'])) { foreach ($event['Event']['EventReport'] as $key => $r) { @@ -454,10 +466,13 @@ class Server extends AppModel if ($this->__checkIfEventIsBlockedBeforePull($event)) { return false; } - $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules()); + $pullRulesEmptiedEvent = false; + $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent); if (!$this->__checkIfEventSaveAble($event)) { - $fails[$eventId] = __('Empty event detected.'); + if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure + $fails[$eventId] = __('Empty event detected.'); + } } else { $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force); } From 0bb4f372ffcd64839f0f6f825c8673a09d12a4d1 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 13:47:39 +0100 Subject: [PATCH 087/101] fix: [server:pull] Typo while unsetting attribute blocked by filtering rule --- app/Model/Server.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 43ccc2b38..0fe2290fc 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -316,7 +316,7 @@ class Server extends AppModel $originalAttributeCount = count($object['Attribute']); foreach ($object['Attribute'] as $j => $a) { if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) { - unset($event['Event']['Object'][$i]['Attribute'][$j]['Tag'][$k]); + unset($event['Event']['Object'][$i]['Attribute'][$j]); continue; } switch ($a['distribution']) { From 666ffcbe256d7de4626cf1ccb4b034c6b6fd3454 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Dec 2021 15:34:17 +0100 Subject: [PATCH 088/101] fix: [server:edit] Typo synchronisation --- app/View/Elements/serverRuleElements/rules_filtering_type.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/serverRuleElements/rules_filtering_type.ctp b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp index d23fd7d1c..100248c2f 100644 --- a/app/View/Elements/serverRuleElements/rules_filtering_type.ctp +++ b/app/View/Elements/serverRuleElements/rules_filtering_type.ctp @@ -12,7 +12,7 @@
    • -
    • +
@@ -54,4 +54,4 @@ ?>
-
\ No newline at end of file +