Merge pull request #9495 from JakubOnderka/cleanup-php74

chg: [internal] PHP 7.4 is required, so we can remove hacks for older…
pull/9496/head
Jakub Onderka 2024-01-14 17:58:16 +01:00 committed by GitHub
commit 4a3f0990ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 192 additions and 156 deletions

View File

@ -140,6 +140,16 @@ jobs:
sudo chmod +x /home
sudo chmod +x /
- name: Python setup
run: |
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
- name: DB Update
run: |
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
@ -148,7 +158,7 @@ jobs:
- name: Configure MISP
run: |
sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
sudo -u $USER app/Console/cake User init | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
@ -166,9 +176,6 @@ jobs:
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
- name: Configure ZMQ
run: |
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
@ -192,7 +199,7 @@ jobs:
run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- name: Turn MISP live
run: sudo -E su $USER -c 'app/Console/cake Live 1'
run: sudo -E su $USER -c 'app/Console/cake Admin live 1'
- name: Check if Redis is ready
run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
@ -202,19 +209,6 @@ jobs:
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
- name: Python setup
run: |
sudo chmod 777 ./key.txt
sudo chmod -R 777 ./tests
# Start workers
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
- name: Test if apache is working
run: |
sudo systemctl status apache2 --no-pager -l
@ -234,7 +228,7 @@ jobs:
- name: Run PHP tests
run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
- name: Clone test files

View File

@ -46,11 +46,11 @@ class AdminShell extends AppShell
'help' => __('Update the JSON definition of taxonomies.'),
));
$parser->addSubcommand('setSetting', [
'help' => __('Set setting in PHP config file.'),
'help' => __('Set setting in MISP config'),
'parser' => [
'arguments' => [
'name' => ['help' => __('Setting name'), 'required' => true],
'value' => ['help' => __('Setting value'), 'required' => true],
'value' => ['help' => __('Setting value')],
],
'options' => [
'force' => [
@ -507,32 +507,47 @@ class AdminShell extends AppShell
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
$this->out($this->json($result));
}
public function setSetting()
{
list($setting_name, $value) = $this->args;
if ($value === 'false') {
$value = 0;
} elseif ($value === 'true') {
$value = 1;
}
if ($this->params['null']) {
list($settingName) = $this->args;
if ($this->params['null'] && isset($this->args[1])) {
$this->error(__('Trying to set setting to null value, but value was provided.'));
} else if ($this->params['null']) {
$value = null;
} elseif (isset($this->args[1])) {
$value = $this->args[1];
} else {
$this->error(__('No setting value provided.'));
}
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || ($value === null && !$this->params['null'])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
}
$setting = $this->Server->getSettingData($setting_name);
$setting = $this->Server->getSettingData($settingName);
if (empty($setting)) {
$message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$this->error(__('Setting change rejected.'), $message);
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
// Convert value to boolean or to int
if ($value !== null) {
if ($setting['type'] === 'boolean') {
$value = $this->toBoolean($value);
} else if ($setting['type'] === 'numeric') {
if (is_numeric($value)) {
$value = (int)$value;
} elseif ($value === 'true' || $value === 'false') {
$value = $value === 'true' ? 1 : 0; // special case for `debug` setting
} else {
$this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
}
}
}
$result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
if ($result === true) {
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
$this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
} else {
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
$this->error(__('Setting change rejected.'), $message);

View File

@ -90,7 +90,7 @@ abstract class AppShell extends Shell
*/
protected function deprecated($newCommand)
{
$this->err("<warning>This method is deprecated. Next time please use `$newCommand`.</warning>");
$this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
}
/**

View File

@ -1,7 +1,13 @@
<?php
/**
* @deprecated
*/
class UserInitShell extends AppShell {
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
public function main() {
$this->deprecated('cake user init');
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));

View File

@ -22,6 +22,9 @@ class UserShell extends AppShell
],
]
]);
$parser->addSubcommand('init', [
'help' => __('Create default role, organisation and user when not exists.'),
]);
$parser->addSubcommand('authkey', [
'help' => __('Get information about given authkey.'),
'parser' => [
@ -121,7 +124,7 @@ class UserShell extends AppShell
public function list()
{
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions = ['OR' => [
'User.id' => $userId,
@ -163,6 +166,21 @@ class UserShell extends AppShell
}
}
public function init()
{
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
}
$authKey = $this->User->init();
if ($authKey === null) {
$this->err('Script aborted: MISP instance already initialised.');
} else {
$this->out($authKey);
}
}
public function authkey()
{
if (isset($this->args[0])) {

View File

@ -1244,8 +1244,6 @@ class UsersController extends AppController
// login was successful, do everything that is needed such as logging and more:
$this->_postlogin();
} else {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
// don't display authError before first login attempt
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
$this->Session->delete('Message.auth');
@ -1260,73 +1258,7 @@ class UsersController extends AppController
}
}
//
// Actions needed for the first access, when the database is not populated yet.
//
// populate the DB with the first role (site admin) if it's empty
if (!$this->User->Role->hasAny()) {
$siteAdmin = array('Role' => array(
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
));
$this->User->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->User->Role->query($sql);
}
}
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
$this->User->runUpdates();
$date = date('Y-m-d H:i:s');
$org = array('Organisation' => array(
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'uuid' => CakeText::uuid(),
'local' => 1,
'date_created' => $date,
'sector' => '',
'nationality' => ''
));
$this->User->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->User->Organisation->query($sql);
}
$org_id = $this->User->Organisation->id;
}
// populate the DB with the first user if it's empty
if (!$this->User->hasAny()) {
if (!isset($org_id)) {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
$this->User->runUpdates();
$this->User->createInitialUser($org_id);
}
$this->User->init();
}
}

View File

@ -153,7 +153,7 @@ class JSONConverterTool
return;
}
yield '{"Event":{';
$firstKey = key($event['Event']);
$firstKey = array_key_first($event['Event']);
foreach ($event['Event'] as $key => $value) {
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";

View File

@ -75,7 +75,7 @@ class JsonTool
*/
public static function escapeNonUnicode($string)
{
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
if (mb_check_encoding($string, 'UTF-8')) {
return $string; // string is valid unicode
}

View File

@ -2374,9 +2374,9 @@ class AppModel extends Model
}
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
public function valueNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!empty($value)) {
return true;
@ -2384,27 +2384,27 @@ class AppModel extends Model
return ucfirst($field) . ' cannot be empty.';
}
public function valueIsJson($value)
public function valueIsJson(array $value)
{
$value = array_values($value)[0];
$value = current($value);
if (!JsonTool::isValid($value)) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsID($value)
public function valueIsID(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
return true;
}
public function stringNotEmpty($value)
public function stringNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
@ -3713,7 +3713,7 @@ class AppModel extends Model
if (!$isRule) {
$args = func_get_args();
$fields = $args[1];
$or = isset($args[2]) ? $args[2] : true;
$or = $args[2] ?? true;
}
}
if (!is_array($fields)) {
@ -3858,8 +3858,7 @@ class AppModel extends Model
protected function isMysql()
{
$dataSource = ConnectionManager::getDataSource('default');
$dataSourceName = $dataSource->config['datasource'];
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
return $dataSource instanceof Mysql;
}
/**
@ -3995,21 +3994,21 @@ class AppModel extends Model
");
}
public function findOrder($order, $order_model, $valid_order_fields)
public function findOrder($order, $orderModel, $validOrderFields)
{
if (!is_array($order)) {
$order_rules = explode(' ', strtolower($order));
$order_field = explode('.', $order_rules[0]);
$order_field = end($order_field);
if (in_array($order_field, $valid_order_fields)) {
$orderRules = explode(' ', strtolower($order));
$orderField = explode('.', $orderRules[0]);
$orderField = end($orderField);
if (in_array($orderField, $validOrderFields, true)) {
$direction = 'asc';
if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') {
if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') {
$direction = 'desc';
}
} else {
return null;
}
return $order_model . '.' . $order_field . ' ' . $direction;
return $orderModel . '.' . $orderField . ' ' . $direction;
}
return null;
}

View File

@ -434,7 +434,7 @@ class Attribute extends AppModel
public function afterSave($created, $options = array())
{
// Passing event in `parentEvent` field will speed up correlation
$passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false;
$passedEvent = $options['parentEvent'] ?? false;
$attribute = $this->data['Attribute'];
@ -808,7 +808,7 @@ class Attribute extends AppModel
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
$seen = current($fields);
if ($seen === null) {
return true;
}

View File

@ -2371,23 +2371,21 @@ class Server extends AppModel
return $setting;
}
public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false)
/**
* @param array|string $user
* @param array $setting
* @param mixed $value
* @param bool $forceSave
* @return mixed|string|true|null
* @throws Exception
*/
public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false)
{
if (isset($setting['beforeHook'])) {
$beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value));
$beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value);
if ($beforeResult !== true) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->saveOrFailSilently(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $user['email'],
'action' => 'serverSettingsEdit',
'user_id' => $user['id'],
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
));
$change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.';
$this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
return $beforeResult;
}
}
@ -2396,7 +2394,7 @@ class Server extends AppModel
if ($setting['type'] === 'boolean') {
$value = (bool)$value;
} else if ($setting['type'] === 'numeric') {
$value = (int)($value);
$value = (int)$value;
}
if (isset($setting['test'])) {
if ($setting['test'] instanceof Closure) {
@ -2437,7 +2435,7 @@ class Server extends AppModel
if ($setting['afterHook'] instanceof Closure) {
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
} else {
$afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue));
$afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue);
}
if ($afterResult !== true) {
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;

View File

@ -46,7 +46,7 @@ class SystemSetting extends AppModel
{
/** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) {
if (!$systemSetting->tableExists()) {
return;
}
$settings = $systemSetting->getSettings();
@ -58,7 +58,7 @@ class SystemSetting extends AppModel
}
}
public function databaseExists()
private function tableExists()
{
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true);

View File

@ -1105,13 +1105,18 @@ class User extends AppModel
return $hashed;
}
public function createInitialUser($org_id)
/**
* @param int $orgId
* @return string User auth key
* @throws Exception
*/
public function createInitialUser($orgId)
{
$authKey = $this->generateAuthKey();
$admin = array('User' => array(
'id' => 1,
'email' => 'admin@admin.test',
'org_id' => $org_id,
'org_id' => $orgId,
'password' => 'admin',
'confirm_password' => 'admin',
'authkey' => $authKey,
@ -1123,7 +1128,6 @@ class User extends AppModel
$this->validator()->remove('password'); // password is too simple, remove validation
$this->save($admin);
if (!empty(Configure::read("Security.advanced_authkeys"))) {
$this->AuthKey = ClassRegistry::init('AuthKey');
$newKey = [
'authkey' => $authKey,
'user_id' => 1,
@ -2156,7 +2160,7 @@ class User extends AppModel
if (!ctype_alnum($token)) {
return false;
}
$redis = $this->setupRedis();
$redis = RedisTool::init();
$userId = $redis->get('misp:forgot:' . $token);
if (empty($userId)) {
return false;
@ -2167,8 +2171,78 @@ class User extends AppModel
public function purgeForgetToken($token)
{
$redis = $this->setupRedis();
$userId = $redis->del('misp:forgot:' . $token);
$redis = RedisTool::init();
$redis->del('misp:forgot:' . $token);
return true;
}
/**
* Create default Role, Organisation and User
* @return string|null Created user auth key
* @throws Exception
*/
public function init()
{
if (!$this->Role->hasAny()) {
$siteAdmin = ['Role' => [
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
]];
$this->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->Role->query($sql);
}
}
if (!$this->Organisation->hasAny(['Organisation.local' => true])) {
$this->runUpdates();
$org = ['Organisation' => [
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'date_created' => date('Y-m-d H:i:s'),
'local' => 1,
]];
$this->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->Organisation->query($sql);
}
$orgId = $this->Organisation->id;
}
if (!$this->hasAny()) {
if (!isset($orgId)) {
$hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$orgId = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$orgId = $firstOrg['Organisation']['id'];
}
}
$this->runUpdates();
return $this->createInitialUser($orgId);
}
return null;
}
}

View File

@ -3,12 +3,12 @@ $keyUsageCsv = null;
if (isset($keyUsage)) {
$todayString = date('Y-m-d');
$today = strtotime($todayString);
$startDate = key($keyUsage); // oldest date for sparkline
$startDate = array_key_first($keyUsage); // oldest date for sparkline
$startDate = strtotime($startDate) - (3600 * 24 * 3);
$keyUsageCsv = 'Date,Close\n';
for ($date = $startDate; $date <= $today; $date += (3600 * 24)) {
$dateAsString = date('Y-m-d', $date);
$keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n';
$keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n';
}
} else {
$lastUsed = null;