MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/7984/head
iglocska 2021-11-18 23:06:21 +01:00
parent d9ee59c87b a371194689
commit bb685e0bf9
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
12 changed files with 226 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/main.yml vendored
View File

@ -54,7 +54,7 @@ jobs:
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php }}
extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd
extensions: mysql, mbstring, json, xml, opcache, readline, redis, gd, apcu
- name: Initialize variables
run: |

5
app/Config/core.default.php
View File

@ -255,8 +255,9 @@ Configure::write('Acl.database', 'default');
* and their setttings.
*/
$engine = 'File';
if (extension_loaded('apc') && function_exists('apc_dec') && (php_sapi_name() !== 'cli' || ini_get('apc.enable_cli'))) {
$engine = 'Apc';
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
$engine = 'Apcu'; // faster version of ApcEngine
}
// In development mode, caches should expire quickly.

17
app/Console/Command/StartWorkerShell.php
View File

@ -17,8 +17,6 @@ class StartWorkerShell extends AppShell
private const DEFAULT_MAX_EXECUTION_TIME = 86400; // 1 day
public $tasks = ['ConfigLoad'];
public function initialize(): void
{
parent::initialize();
@ -30,17 +28,14 @@ class StartWorkerShell extends AppShell
$parser = parent::getOptionParser();
$parser
->addArgument('queue', [
'help' => sprintf(
'Name of the queue to process. Must be one of [%]',
implode(', ', $this->BackgroundJobsTool->getQueues())
),
'help' => 'Name of the queue to process.',
'choices' => $this->BackgroundJobsTool->getQueues(),
'required' => true
])
->addOption(
'maxExecutionTime',
[
'help' => 'Worker maximum execution time (seconds) before it self-destruct.',
'help' => 'Worker maximum execution time (seconds) before it self-destruct. Zero means unlimited.',
'default' => self::DEFAULT_MAX_EXECUTION_TIME,
'required' => false
]
@ -64,18 +59,17 @@ class StartWorkerShell extends AppShell
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
while (true) {
$this->ConfigLoad->execute();
$this->checkMaxExecutionTime();
$job = $this->BackgroundJobsTool->dequeue($this->worker->queue());
if ($job) {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with id: {$job->id()} ...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
try {
$this->BackgroundJobsTool->run($job);
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job id: {$job->id()} failed with exception: {$exception->getMessage()}");
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
$job->setStatus(BackgroundJob::STATUS_FAILED);
$this->BackgroundJobsTool->update($job);
}
@ -90,6 +84,9 @@ class StartWorkerShell extends AppShell
*/
private function checkMaxExecutionTime(): void
{
if ($this->maxExecutionTime === 0) {
return;
}
if ((time() - $this->worker->createdAt()) > $this->maxExecutionTime) {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - worker max execution time reached, exiting gracefully worker...");
exit;

6
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -2,8 +2,6 @@
declare(strict_types=1);
App::uses('Model', 'Model');
class BackgroundJob implements JsonSerializable
{
public const
@ -87,7 +85,9 @@ class BackgroundJob implements JsonSerializable
$this->args()
),
$descriptorSpec,
$pipes
$pipes,
null,
['BACKGROUND_JOB_ID' => $this->id]
);
$stdout = stream_get_contents($pipes[1]);

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -2,8 +2,6 @@
declare(strict_types=1);
App::uses('Model', 'Model');
class Worker implements JsonSerializable
{
/** @var integer|null */

42
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -116,7 +116,6 @@ class BackgroundJobsTool
if ($this->settings['enabled'] === true) {
$this->RedisConnection = $this->createRedisConnection();
$this->Supervisor = $this->createSupervisorConnection();
}
}
@ -128,8 +127,8 @@ class BackgroundJobsTool
* @param array $args Arguments passed to the job.
* @param boolean|null $trackStatus Whether to track the status of the job.
* @param int|null $jobId Id of the relational database record representing the job.
* @return string Background Job Id.
* @param array $metadata Related to the job.
* @return string Background Job ID.
* @throws InvalidArgumentException
*/
public function enqueue(
@ -223,11 +222,7 @@ class BackgroundJobsTool
$rawJob = $this->RedisConnection->blpop($queue, $timeout);
if (!empty($rawJob)) {
try {
return new BackgroundJob($rawJob[1]);
} catch (Exception $exception) {
CakeLog::error("Failed to parse job, invalid format: {$rawJob[1]}. exception: {$exception->getMessage()}");
}
return new BackgroundJob($rawJob[1]);
}
return null;
@ -236,7 +231,7 @@ class BackgroundJobsTool
/**
* Get the job status.
*
* @param string $jobId Backgroun Job Id.
* @param string $jobId Background Job Id.
*
* @return BackgroundJob|null job status.
*
@ -286,7 +281,7 @@ class BackgroundJobsTool
public function getWorkers(): array
{
$workers = [];
$procs = $this->Supervisor->getAllProcesses();
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) {
@ -374,7 +369,7 @@ class BackgroundJobsTool
{
$this->validateWorkerName($name);
return $this->Supervisor->startProcess(
return $this->getSupervisor()->startProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
@ -395,13 +390,13 @@ class BackgroundJobsTool
{
$this->validateQueue($queue);
$procs = $this->Supervisor->getAllProcesses();
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) {
$name = explode("_", $proc->offsetGet('name'))[0];
if ($name === $queue && $proc->offsetGet('state') != \Supervisor\Process::RUNNING) {
return $this->Supervisor->startProcess(
return $this->getSupervisor()->startProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
@ -434,7 +429,7 @@ class BackgroundJobsTool
$this->validateWorkerName($name);
return $this->Supervisor->stopProcess(
return $this->getSupervisor()->stopProcess(
sprintf(
'%s:%s',
self::MISP_WORKERS_PROCESS_GROUP,
@ -452,8 +447,8 @@ class BackgroundJobsTool
*/
public function restartWorkers(bool $waitForRestart = false): void
{
$this->Supervisor->stopProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
$this->Supervisor->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
$this->getSupervisor()->stopProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
@ -464,7 +459,7 @@ class BackgroundJobsTool
*/
public function restartDeadWorkers(bool $waitForRestart = false): void
{
$this->Supervisor->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
@ -499,7 +494,7 @@ class BackgroundJobsTool
}
try {
$supervisorStatus = ($this->Supervisor->getState()['statecode'] === \Supervisor\Supervisor::RUNNING);
$supervisorStatus = $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING;
} catch (Exception $exception) {
CakeLog::error("SimpleBackgroundJobs Supervisor error: {$exception->getMessage()}");
$supervisorStatus = false;
@ -597,6 +592,17 @@ class BackgroundJobsTool
return $redis;
}
/**
* @return \Supervisor\Supervisor
*/
private function getSupervisor()
{
if (!$this->Supervisor) {
$this->Supervisor = $this->createSupervisorConnection();
}
return $this->Supervisor;
}
/**
* @return \Supervisor\Supervisor
*/
@ -644,7 +650,7 @@ class BackgroundJobsTool
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{
$procs = $this->Supervisor->getAllProcesses();
$procs = $this->getSupervisor()->getAllProcesses();
foreach ($procs as $proc) {
if (

2
app/Model/Galaxy.php
View File

@ -383,7 +383,7 @@ class Galaxy extends AppModel
throw new NotFoundException(__('Invalid %s.', $target_type));
}
$target = $target[0];
$local_only = $cluster['Galaxy']['local_only'];
$local_only = $cluster['GalaxyCluster']['Galaxy']['local_only'];
if ($local_only && !$local) {
throw new MethodNotAllowedException(__("This Cluster can only be attached in a local scope"));
}

8
app/Model/Server.php
View File

@ -3157,8 +3157,12 @@ class Server extends AppModel
);
foreach ($writeableDirs as $path => &$error) {
if (!file_exists($path)) {
$error = 1;
} else if (!is_writable($path)) {
// Try to create directory if not exists
if (!mkdir($path, 0700, true)) {
$error = 1;
}
}
if (!is_writable($path)) {
$error = 2;
}
if ($error !== 0) {

10
app/Model/SystemSetting.php
View File

@ -42,7 +42,11 @@ class SystemSetting extends AppModel
*/
public static function setGlobalSetting()
{
/** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) {
return;
}
$settings = $systemSetting->getSettings();
foreach ($settings as $settingName => $settingValue) {
$firstPart = explode('.', $settingName)[0];
@ -52,6 +56,12 @@ class SystemSetting extends AppModel
}
}
public function databaseExists()
{
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true);
}
/**
* @return array
* @throws JsonException

167
app/Plugin/ApcuCache/Engine/ApcuEngine.php Normal file
View File

@ -0,0 +1,167 @@
<?php
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/CacheEngine.php';
/**
* APC storage engine for cache. Faster version of original ApcEngine
*/
class ApcuEngine extends CacheEngine {
/**
* Contains the compiled group names
* (prefixed with the global configuration prefix)
*
* @var array
*/
protected $_compiledGroupNames = array();
/**
* Initialize the Cache Engine
*
* Called automatically by the cache frontend
* To reinitialize the settings call Cache::engine('EngineName', [optional] settings = array());
*
* @param array $settings array of setting for the engine
* @return bool True if the engine has been successfully initialized, false if not
* @see CacheEngine::__defaults
*/
public function init($settings = array()) {
if (!isset($settings['prefix'])) {
$settings['prefix'] = Inflector::slug(APP_DIR) . '_';
}
$settings += array('engine' => 'Apc');
parent::init($settings);
return function_exists('apcu_dec');
}
/**
* Write data for key into cache
*
* @param string $key Identifier for the data
* @param mixed $value Data to be cached
* @param int $duration How long to cache the data, in seconds
* @return bool True if the data was successfully cached, false on failure
*/
public function write($key, $value, $duration) {
return apcu_store($key, $value, $duration);
}
/**
* Read a key from the cache
*
* @param string $key Identifier for the data
* @return mixed The cached data, or false if the data doesn't exist, has expired, or if there was an error fetching it
*/
public function read($key) {
return apcu_fetch($key);
}
/**
* Increments the value of an integer cached key
*
* @param string $key Identifier for the data
* @param int $offset How much to increment
* @return false|int New incremented value, false otherwise
*/
public function increment($key, $offset = 1) {
return apcu_inc($key, $offset);
}
/**
* Decrements the value of an integer cached key
*
* @param string $key Identifier for the data
* @param int $offset How much to subtract
* @return false|int New decremented value, false otherwise
*/
public function decrement($key, $offset = 1) {
return apcu_dec($key, $offset);
}
/**
* Delete a key from the cache
*
* @param string $key Identifier for the data
* @return bool True if the value was successfully deleted, false if it didn't exist or couldn't be removed
*/
public function delete($key) {
return apcu_delete($key);
}
/**
* Delete all keys from the cache. This will clear every cache config using APC.
*
* @param bool $check If true, nothing will be cleared, as entries are removed
* from APC as they expired. This flag is really only used by FileEngine.
* @return bool True Returns true.
*/
public function clear($check) {
if ($check) {
return true;
}
$iterator = new APCUIterator(
'/^' . preg_quote($this->settings['prefix'], '/') . '/',
APC_ITER_NONE
);
apcu_delete($iterator);
return true;
}
/**
* Returns the `group value` for each of the configured groups
* If the group initial value was not found, then it initializes
* the group accordingly.
*
* @return array
*/
public function groups() {
if (empty($this->_compiledGroupNames)) {
foreach ($this->settings['groups'] as $group) {
$this->_compiledGroupNames[] = $this->settings['prefix'] . $group;
}
}
$groups = apcu_fetch($this->_compiledGroupNames);
if (count($groups) !== count($this->settings['groups'])) {
foreach ($this->_compiledGroupNames as $group) {
if (!isset($groups[$group])) {
apcu_store($group, 1);
$groups[$group] = 1;
}
}
ksort($groups);
}
$result = array();
$groups = array_values($groups);
foreach ($this->settings['groups'] as $i => $group) {
$result[] = $group . $groups[$i];
}
return $result;
}
/**
* Increments the group value to simulate deletion of all keys under a group
* old values will remain in storage until they expire.
*
* @param string $group The group to clear.
* @return bool success
*/
public function clearGroup($group) {
apcu_inc($this->settings['prefix'] . $group, 1, $success);
return $success;
}
/**
* Write data for key into cache if it doesn't exist already.
* If it already exists, it fails and returns false.
*
* @param string $key Identifier for the data.
* @param mixed $value Data to be cached.
* @param int $duration How long to cache the data, in seconds.
* @return bool True if the data was successfully cached, false on failure.
* @link http://php.net/manual/en/function.apc-add.php
*/
public function add($key, $value, $duration) {
return apc_add($key, $value, $duration);
}
}

1
app/composer.json
View File

@ -30,6 +30,7 @@
"ext-ssdeep": "For ssdeep hashes correlation",
"ext-bcmath": "For faster validating IBAN numbers",
"ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system",
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3",
"jumbojett/openid-connect-php": "For OIDC authentication",

5
travis/core.php
View File

@ -244,8 +244,9 @@
* and their setttings.
*/
$engine = 'File';
if (extension_loaded('apc') && function_exists('apc_dec') && (php_sapi_name() !== 'cli' || ini_get('apc.enable_cli'))) {
$engine = 'Apc';
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
$engine = 'Apcu';
}
// In development mode, caches should expire quickly.