diff --git a/PyMISP b/PyMISP index a2566f028..e20a9c753 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit a2566f0282b9f3f83b7785e9fdac3f7aa95fd88b +Subproject commit e20a9c753957c2582789b85ca3176f27da089232 diff --git a/VERSION.json b/VERSION.json index 859814823..04b46f16c 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":175} +{"major":2, "minor":4, "hotfix":176} diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php index 4cee17115..a3b1fe37d 100644 --- a/app/Console/Command/TrainingShell.php +++ b/app/Console/Command/TrainingShell.php @@ -209,6 +209,11 @@ class TrainingShell extends AppShell { $this->createRemoteServersFromConfig($createdOrgs, $createdUsers); } + public function deleteAllSyncs() + { + $this->Server->deleteAll(['Server.id' > 0]); + } + private function __createOrgFromBlueprint($id) { $org = str_replace('$ID', $id, $this->__config['org_blueprint']); diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 3550c0fec..715457783 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -34,7 +34,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); private $__queryVersion = '155'; - public $pyMispVersion = '2.4.175'; + public $pyMispVersion = '2.4.176'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; @@ -962,6 +962,14 @@ class AppController extends Controller return $user; } + private function __captureParam($data, $param, $value) + { + if ($this->modelClass->checkParam($param)) { + $data[$param] = $value; + } + return $data; + } + /** * generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params * @param array $options @@ -982,9 +990,21 @@ class AppController extends Controller return false; } else { if (isset($request->data['request'])) { - $data = array_merge($data, $request->data['request']); + $temp = $request->data['request']; } else { - $data = array_merge($data, $request->data); + $temp = $request->data; + } + if (empty($options['paramArray'])) { + foreach ($options['paramArray'] as $param => $value) { + $data = $this->__captureParam($data, $param, $value); + } + $data = array_merge($data, $temp); + } else { + foreach ($options['paramArray'] as $param) { + if (isset($temp[$param])) { + $data[$param] = $temp[$param]; + } + } } } } diff --git a/app/Controller/Component/IndexFilterComponent.php b/app/Controller/Component/IndexFilterComponent.php index 15fc73924..064a0bb87 100644 --- a/app/Controller/Component/IndexFilterComponent.php +++ b/app/Controller/Component/IndexFilterComponent.php @@ -54,7 +54,7 @@ class IndexFilterComponent extends Component private function __massageData($data, $request, $paramArray) { $data = array_filter($data, function($paramName) use ($paramArray) { - return in_array($paramName, $paramArray); + return in_array($paramName, $paramArray, true); }, ARRAY_FILTER_USE_KEY); if (!empty($paramArray)) { diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index b86db15cb..1d540ae48 100644 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -285,6 +285,12 @@ class LogsController extends AppController $filters['model'] = $this->request->data['Log']['model']; $filters['model_id'] = $this->request->data['Log']['model_id']; $filters['title'] = $this->request->data['Log']['title']; + if (!empty ($this->request->data['Log']['from'])) { + $filters['from'] = $this->request->data['Log']['from']; + } + if (!empty ($this->request->data['Log']['to'])) { + $filters['to'] = $this->request->data['Log']['to']; + } $filters['change'] = $this->request->data['Log']['change']; if (Configure::read('MISP.log_client_ip')) { $filters['ip'] = $this->request->data['Log']['ip']; @@ -297,6 +303,8 @@ class LogsController extends AppController $this->set('modelSearch', $filters['model']); $this->set('model_idSearch', $filters['model_id']); $this->set('titleSearch', $filters['title']); + $this->set('fromSearch', $filters['from'] ?? null); + $this->set('toSearch', $filters['to'] ?? null); $this->set('changeSearch', $filters['change']); if (Configure::read('MISP.log_client_ip')) { $this->set('ipSearch', $filters['ip']); @@ -329,6 +337,8 @@ class LogsController extends AppController $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']); $this->Session->write('paginate_conditions_log_title', $filters['title']); $this->Session->write('paginate_conditions_log_change', $filters['change']); + $this->Session->write('paginate_conditions_log_change', $filters['from'] ?? null); + $this->Session->write('paginate_conditions_log_change', $filters['to'] ?? null); if (Configure::read('MISP.log_client_ip')) { $this->Session->write('paginate_conditions_log_ip', $filters['ip']); } @@ -345,6 +355,8 @@ class LogsController extends AppController $filters['model_id'] = $this->Session->read('paginate_conditions_log_model_id'); $filters['title'] = $this->Session->read('paginate_conditions_log_title'); $filters['change'] = $this->Session->read('paginate_conditions_log_change'); + $filters['change'] = $this->Session->read('paginate_conditions_log_from') ?? null; + $filters['change'] = $this->Session->read('paginate_conditions_log_to') ?? null; if (Configure::read('MISP.log_client_ip')) { $filters['ip'] = $this->Session->read('paginate_conditions_log_ip'); } @@ -357,6 +369,8 @@ class LogsController extends AppController $this->set('model_idSearch', $filters['model_id']); $this->set('titleSearch', $filters['title']); $this->set('changeSearch', $filters['change']); + $this->set('changeSearch', $filters['from'] ?? null); + $this->set('changeSearch', $filters['to'] ?? null); if (Configure::read('MISP.log_client_ip')) { $this->set('ipSearch', $filters['ip']); } @@ -449,6 +463,12 @@ class LogsController extends AppController if (isset($filters['change']) && !empty($filters['change'])) { $conditions['LOWER(Log.change) LIKE'] = '%' . strtolower($filters['change']) . '%'; } + if (isset($filters['from']) && !empty($filters['from'])) { + $conditions['Log.created >='] = $filters['from']; + } + if (isset($filters['to']) && !empty($filters['to'])) { + $conditions['Log.created <='] = $filters['to']; + } if (Configure::read('MISP.log_client_ip') && isset($filters['ip']) && !empty($filters['ip'])) { $conditions['Log.ip LIKE'] = '%' . $filters['ip'] . '%'; } diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 3aed0deae..597110166 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -301,6 +301,7 @@ class UsersController extends AppController // What fields should be saved (allowed to be saved) $user['User']['change_pw'] = 0; $user['User']['password'] = $this->request->data['User']['password']; + $user['User']['last_pw_change'] = time(); if ($this->_isRest()) { $user['User']['confirm_password'] = $this->request->data['User']['password']; } else { @@ -475,7 +476,8 @@ class UsersController extends AppController 'last_api_access', 'force_logout', 'date_created', - 'date_modified' + 'date_modified', + 'last_pw_change' ), 'contain' => array( 'Organisation' => array('id', 'name'), @@ -687,6 +689,7 @@ class UsersController extends AppController } } $this->request->data['User']['date_created'] = time(); + $this->request->data['User']['last_pw_change'] = $this->request->data['User']['date_created']; if (!array_key_exists($this->request->data['User']['role_id'], $syncRoles)) { $this->request->data['User']['server_id'] = 0; } @@ -758,7 +761,7 @@ class UsersController extends AppController $this->Flash->error(__('The user could not be saved. Invalid organisation.')); } } else { - $fieldList = array('password', 'email', 'external_auth_required', 'external_auth_key', 'enable_password', 'confirm_password', 'org_id', 'role_id', 'authkey', 'nids_sid', 'server_id', 'gpgkey', 'certif_public', 'autoalert', 'contactalert', 'disabled', 'invited_by', 'change_pw', 'termsaccepted', 'newsread', 'date_created', 'date_modified'); + $fieldList = array('password', 'email', 'external_auth_required', 'external_auth_key', 'enable_password', 'confirm_password', 'org_id', 'role_id', 'authkey', 'nids_sid', 'server_id', 'gpgkey', 'certif_public', 'autoalert', 'contactalert', 'disabled', 'invited_by', 'change_pw', 'termsaccepted', 'newsread', 'date_created', 'date_modified', 'last_pw_change'); if ($this->User->save($this->request->data, true, $fieldList)) { $notification_message = ''; if (!empty($this->request->data['User']['notify'])) { @@ -953,6 +956,8 @@ class UsersController extends AppController $this->__canChangePassword() ) { $fields[] = 'password'; + $fields[] = 'last_pw_change'; + $this->request->data['User']['last_pw_change'] = time(); if ($this->_isRest() && !isset($this->request->data['User']['confirm_password'])) { $this->request->data['User']['confirm_password'] = $this->request->data['User']['password']; $fields[] = 'confirm_password'; diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 974c97828..b90001e90 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -85,7 +85,7 @@ class AppModel extends Model 93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false, 99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false, 105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false, - 111 => false, 112 => false, 113 => true, 114 => false + 111 => false, 112 => false, 113 => true, 114 => false, 115 => false ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -1973,6 +1973,10 @@ class AppModel extends Model case 114: $indexArray[] = ['object_references', 'uuid']; break; + case 115: + $sqlArray[] = "ALTER TABLE `users` ADD COLUMN `last_pw_change` BIGINT(20) NULL DEFAULT NULL;"; + $sqlArray[] = "UPDATE `users` SET last_pw_change=date_modified WHERE last_pw_change IS NULL"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -3288,26 +3292,32 @@ class AppModel extends Model foreach ($filters as $f) { if ($f === -1) { foreach ($keys as $key) { - $temp['OR'][$key][] = -1; + if ($this->checkParam($key)) { + $temp['OR'][$key][] = -1; + } } continue; } // split the filter params into two lists, one for substring searches one for exact ones if (is_string($f) && ($f[strlen($f) - 1] === '%' || $f[0] === '%')) { foreach ($keys as $key) { - if ($operator === 'NOT') { - $temp[] = array($key . ' NOT LIKE' => $f); - } else { - $temp[] = array($key . ' LIKE' => $f); - $temp[] = array($key => $f); + if ($this->checkParam($key)) { + if ($operator === 'NOT') { + $temp[] = array($key . ' NOT LIKE' => $f); + } else { + $temp[] = array($key . ' LIKE' => $f); + $temp[] = array($key => $f); + } } } } else { foreach ($keys as $key) { - if ($operator === 'NOT') { - $temp[$key . ' !='][] = $f; - } else { - $temp['OR'][$key][] = $f; + if ($this->checkParam($key)) { + if ($operator === 'NOT') { + $temp[$key . ' !='][] = $f; + } else { + $temp['OR'][$key][] = $f; + } } } } @@ -4017,4 +4027,9 @@ class AppModel extends Model return false; } + + public function checkParam($param) + { + return preg_match('/^[\w\_\-\. ]+$/', $param); + } } diff --git a/app/Model/User.php b/app/Model/User.php index f2a5403fb..afbec075e 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -984,6 +984,7 @@ class User extends AppModel if ($result) { $this->id = $user['User']['id']; $this->saveField('password', $password); + $this->saveField('last_pw_change', time()); $this->updateField($user['User'], 'change_pw', 1); if ($simpleReturn) { return true; diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index 0ced5e5c9..948bd038a 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -94,15 +94,11 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {

-

+

- - You are using a MISP installation method that does not support or recommend using the MISP self-update, such as a Docker container. Please update using the appropriate update mechanism. -

-

@@ -110,6 +106,9 @@ $humanReadableFilesize = function ($bytes, $dec = 2) { + + You are using a MISP installation method that does not support or recommend using the MISP self-update, such as a Docker container. Please update using the appropriate update mechanism. +

@@ -604,13 +603,13 @@ $humanReadableFilesize = function ($bytes, $dec = 2) { + + diff --git a/app/View/Logs/admin_search.ctp b/app/View/Logs/admin_search.ctp index 76c08c885..c986b72e0 100755 --- a/app/View/Logs/admin_search.ctp +++ b/app/View/Logs/admin_search.ctp @@ -19,6 +19,9 @@ 'label' => __('Title'), 'div' => 'input clear')); echo $this->Form->input('change', array('label' => __('Change'))); + echo '
'; + echo $this->Form->input('from', array('label' => __('From'), 'class' => 'datepicker form-control')); + echo $this->Form->input('to', array('label' => __('To'), 'class' => 'datepicker form-control')); ?> __('Created'), 'html' => $user['User']['date_created'] ? $this->Time->time($user['User']['date_created']) : __('N/A') ); + $table_data[] = array( + 'key' => __('Last password change'), + 'html' => $user['User']['last_pw_change'] ? $this->Time->time($user['User']['last_pw_change']) : __('N/A') + ); if ($admin_view) { $table_data[] = array( 'key' => __('News read at'), diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 34b86e4ab..f80bcdd97 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 34b86e4abc47d3dfbafaa813f01e22be0387168a +Subproject commit f80bcdd97fdf7a841ab57036cb30a314467fa3ba diff --git a/app/files/misp-objects b/app/files/misp-objects index 8b6489815..364f747e9 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 8b648981573f77c9526df5322c52902ae1a81859 +Subproject commit 364f747e9d64c4f390bed2f63f74a0863097c4f6 diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix index 6771e5cd9..7de99b136 160000 --- a/app/files/scripts/misp-stix +++ b/app/files/scripts/misp-stix @@ -1 +1 @@ -Subproject commit 6771e5cd9ec22d0d24ec9f657d78d385a3c5ef80 +Subproject commit 7de99b1369a3f6649e10f7913fccda15f5f0e134 diff --git a/app/files/warninglists b/app/files/warninglists index 07a1e6609..db5de32d3 160000 --- a/app/files/warninglists +++ b/app/files/warninglists @@ -1 +1 @@ -Subproject commit 07a1e66092a8216574b103c650b423e816a1091a +Subproject commit db5de32d3deec9d4eba80e11a862e20c5ddf67a3 diff --git a/db_schema.json b/db_schema.json index 1fca6ae7c..139503142 100644 --- a/db_schema.json +++ b/db_schema.json @@ -8612,6 +8612,17 @@ "column_type": "int(11)", "column_default": "NULL", "extra": "" + }, + { + "column_name": "last_pw_change", + "is_nullable": "YES", + "data_type": "bigint", + "character_maximum_length": null, + "numeric_precision": "19", + "collation_name": null, + "column_type": "bigint(20)", + "column_default": "NULL", + "extra": "" } ], "user_settings": [ @@ -9549,5 +9560,5 @@ "uuid": false } }, - "db_version": "114" + "db_version": "115" } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 168d3a9d8..e3ed30756 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,19 @@ -pyzmq -coveralls -codecov -stix -pymisp -requests-mock -pip -nose +cybox>=2.1.0.21 jsonschema -plyara >= 2.0.2 +lief>=0.13.1 +maec>=4.1.0.17 +misp-lib-stix2>=3.0.0 +mixbox>=1.0.5 +plyara>=2.0.2 +pydeep2>=0.5.1 +pymisp==2.4.175 +python-magic +pyzmq +redis +stix>=1.2.0.11 +# test dependencies +codecov +coveralls +nose +pip +requests-mock diff --git a/tests/testlive_comprehensive_local.py b/tests/testlive_comprehensive_local.py index 417015b4f..138593def 100644 --- a/tests/testlive_comprehensive_local.py +++ b/tests/testlive_comprehensive_local.py @@ -10,6 +10,7 @@ import time from xml.etree import ElementTree as ET from io import BytesIO import urllib3 # type: ignore +from datetime import datetime, timedelta import logging logging.disable(logging.CRITICAL) @@ -974,5 +975,113 @@ class TestComprehensive(unittest.TestCase): return response +class TestLastPwChange(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.admin_misp_connector = PyMISP(url, key) + + organisation = MISPOrganisation() + organisation.name = 'Test org for last pw change tests' + cls.test_org = cls.admin_misp_connector.add_organisation(organisation, pythonify=True) + check_response(cls.test_org) + cls.test_org_id = cls.test_org.id + + @classmethod + def tearDownClass(cls) -> None: + cls.admin_misp_connector.delete_organisation(cls.test_org) + + def setUp(self) -> None: + self.admin_misp_connector = type(self).admin_misp_connector + + # Create a user + user = MISPUser() + user.email = 'testusr_last_pw_change@user' + gen_random_id() + '.local' # make name always unique + user.org_id = type(self).test_org_id + user.role_id = 3 # User role + user.password = str(uuid.uuid4()) + self.test_usr = self.admin_misp_connector.add_user(user, pythonify=True) + check_response(self.test_usr) + self.test_usr_misp_connector = PyMISP(url, self.test_usr.authkey) + + def tearDown(self) -> None: + # Delete Authkey and user + body = { + "authkey_start": self.test_usr.authkey[0:4], + "authkey_end": self.test_usr.authkey[-4:], + "User.id": self.test_usr.id + } + auth_key = type(self).admin_misp_connector.direct_call('auth_keys', body) + check_response(auth_key) + if len(auth_key) == 1 and "AuthKey" in auth_key[0]: + type(self).admin_misp_connector.direct_call(f'auth_keys/delete/{auth_key[0]["AuthKey"]["id"]}', {}) + + type(self).admin_misp_connector.delete_user(self.test_usr) + + def test_new_user_last_pw_change_is_date_created(self): + self.assertEqual(self.test_usr.last_pw_change, self.test_usr.date_created) + + def test_admin_edit_password_updates_last_pw_change(self): + old_last_pw_change = self.test_usr.last_pw_change + + # edit user password + self.test_usr.password = uuid.uuid4() + time_just_before_update = datetime.now() + self.updated_test_usr = self.admin_misp_connector.update_user(self.test_usr, pythonify=True) + time_just_after_update = datetime.now() + check_response(self.updated_test_usr) + + self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update) + + def test_user_change_password_updates_last_pw_change(self): + old_last_pw_change = self.test_usr.last_pw_change + + # edit user password + time_just_before_update = datetime.now() + change_password_result = self.test_usr_misp_connector.change_user_password(uuid.uuid4()) + time_just_after_update = datetime.now() + check_response(change_password_result) + self.updated_test_usr = self.test_usr_misp_connector.get_user(pythonify=True) + check_response(self.updated_test_usr) + + self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update) + + def test_reset_user_password_updates_last_pw_change(self): + old_last_pw_change = self.test_usr.last_pw_change + + # reset user password + time_just_before_update = datetime.now() + self.admin_misp_connector.direct_call(f'users/initiatePasswordReset/{self.test_usr.id}', {}) + time.sleep(1) + time_just_after_update = datetime.now() + self.updated_test_usr = self.test_usr_misp_connector.get_user(pythonify=True) + check_response(self.updated_test_usr) + + self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update) + + def last_pw_change_almost_equal_to_date_modified(self): + date_modified = datetime.fromtimestamp(int(self.updated_test_usr.date_modified)) + last_pw_change = datetime.fromtimestamp(int(self.updated_test_usr.last_pw_change)) + return date_modified - last_pw_change < timedelta(milliseconds=5) + + def last_pw_change_time_is_in_expected_range(self, time_just_before_update, time_just_after_update): + timediff_last_pw_change_now = datetime.fromtimestamp(int(self.updated_test_usr.last_pw_change)) - time_just_before_update + max_accepted_timediff = time_just_after_update - time_just_before_update + return timediff_last_pw_change_now <= max_accepted_timediff + + def check_last_pw_change_timestamp(self, old_last_pw_change, time_just_before_update, time_just_after_update): + # check if new last_pw_change timestamp looks okay, starting with fact that it should be newer than previous one + self.assertGreater(self.updated_test_usr.last_pw_change, old_last_pw_change) + + # last pw change should be set to timestamp sometime between time_just_before_update and time_just_after_update + self.assertTrue(self.last_pw_change_time_is_in_expected_range(time_just_before_update, time_just_after_update)) + + # last_pw_change should be relatively close to date_modified + self.assertTrue(self.last_pw_change_almost_equal_to_date_modified()) + + +def gen_random_id() -> str: + return str(uuid.uuid4()).split("-")[0] + + if __name__ == '__main__': unittest.main()