MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9491/head
Sami Mokaddem 2024-01-12 13:33:49 +01:00
parent aa7c8cc621 9616e07e95
commit b7d11d3772
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
19 changed files with 760 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
app/Console/Command/AdminShell.php
View File

@ -72,7 +72,7 @@ class AdminShell extends AppShell
'help' => __('Set if MISP instance is live and accessible for users.'),
'parser' => [
'arguments' => [
'state' => ['help' => __('Set Live state')],
'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
],
],
]);
@ -109,6 +109,17 @@ class AdminShell extends AppShell
$parser->addSubcommand('configLint', [
'help' => __('Check if settings has correct value.'),
]);
$parser->addSubcommand('scanAttachment', [
'help' => __('Scan attachments with AV.'),
'parser' => [
'arguments' => [
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
'attributeId' => ['help' => __('ID to scan.')],
'jobId' => ['help' => __('Job ID')],
],
],
]);
return $parser;
}
@ -839,8 +850,8 @@ class AdminShell extends AppShell
public function scanAttachment()
{
$input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
$jobId = isset($this->args[2]) ? $this->args[2] : null;
$attributeId = $this->args[1] ?? null;
$jobId = $this->args[2] ?? null;
$this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
@ -951,7 +962,7 @@ class AdminShell extends AppShell
$newStatus = $this->toBoolean($this->args[0]);
$overallSuccess = false;
try {
$redis = $this->Server->setupRedisWithException();
$redis = RedisTool::init();
if ($newStatus) {
$redis->del('misp:live');
$this->out('Set live status to True in Redis.');
@ -980,7 +991,7 @@ class AdminShell extends AppShell
} else {
$this->out('Current status:');
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
$newStatus = $this->Server->setupRedisWithException()->get('misp:live');
$newStatus = RedisTool::init()->get('misp:live');
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
}
}

3
app/Console/Command/LiveShell.php
View File

@ -1,8 +1,9 @@
<?php
/*
/**
* Enable/disable misp
*
* arg0 = [0|1]
* @deprecated Use AdminShell::live instead
*/
class LiveShell extends AppShell {

26
app/Console/Command/StartWorkerShell.php
View File

@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
public function main()
{
$pid = getmypid();
if ($pid === false) {
throw new RuntimeException("Could not get current process ID");
}
$this->worker = new Worker(
[
'pid' => getmypid(),
'pid' => $pid,
'queue' => $this->args[0],
'user' => ProcessTool::whoami(),
]
);
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
$queue = $this->worker->queue();
$backgroundJobTool = $this->getBackgroundJobsTool();
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
while (true) {
$this->checkMaxExecutionTime();
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
$job = $backgroundJobTool->dequeue($queue);
if ($job) {
$this->runJob($job);
$backgroundJobTool->removeFromRunning($this->worker, $job);
}
}
}
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
*/
private function runJob(BackgroundJob $job)
{
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
try {
$job->setStatus(BackgroundJob::STATUS_RUNNING);
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
$this->getBackgroundJobsTool()->update($job);
$job->run();
$start = microtime(true);
$job->run(function (array $status) use ($job) {
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job);
});
$duration = number_format(microtime(true) - $start, 3, '.', '');
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
} else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
}
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

112
app/Console/Command/WorkerShell.php Normal file
View File

@ -0,0 +1,112 @@
<?php
declare(strict_types=1);
/**
* @property Job $Job
*/
class WorkerShell extends AppShell
{
public $uses = ['Job'];
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser->addSubcommand('showQueues', [
'help' => __('Show jobs in worker queues'),
]);
$parser->addSubcommand('flushQueue', [
'help' => __('Flush jobs in given queue'),
'parser' => [
'arguments' => [
'queue' => ['help' => __('Queue name'), 'required' => true],
],
],
]);
$parser->addSubcommand('showJobStatus', [
'help' => __('Show job status'),
'parser' => [
'arguments' => [
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
],
],
]);
return $parser;
}
public function showQueues()
{
$tool = $this->getBackgroundJobsTool();
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
foreach ($tool->runningJobs($queue) as $jobId) {
$this->out(" - $jobId");
}
}
}
public function flushQueue()
{
$queue = $this->args[0];
try {
$this->getBackgroundJobsTool()->clearQueue($queue);
} catch (InvalidArgumentException $e) {
$this->error($e->getMessage());
}
}
public function showJobStatus()
{
$processId = $this->args[0];
if (is_numeric($processId)) {
$job = $this->Job->find('first', [
'conditions' => ['Job.id' => $processId],
'recursive' => -1,
]);
if (!$job) {
$this->error('Job not found', "Job with ID {$processId} not found");
}
$this->out($this->json($job['Job']));
$processId = $job['Job']['process_id'];
}
if (!Validation::uuid($processId)) {
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
}
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
if (!$jobStatus) {
$this->error('Job not found', "Job with UUID {$processId} not found");
}
$jobStatus = $jobStatus->jsonSerialize();
foreach (['createdAt', 'updatedAt'] as $timeField) {
if (isset($jobStatus[$timeField])) {
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
}
}
if (isset($jobStatus['status'])) {
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
}
$this->out($this->json($jobStatus));
}
private function jobStatusToString(int $jobStatus)
{
switch ($jobStatus) {
case Job::STATUS_WAITING:
return 'waiting';
case Job::STATUS_RUNNING:
return 'running';
case Job::STATUS_FAILED:
return 'failed';
case Job::STATUS_COMPLETED:
return 'completed';
}
throw new InvalidArgumentException("Invalid job status $jobStatus");
}
}

4
app/Controller/AppController.php
View File

@ -230,6 +230,10 @@ class AppController extends Controller
$this->Security->csrfCheck = false;
$loginByAuthKeyResult = $this->__loginByAuthKey();
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
if ($this->IndexFilter->isXhr()) {
throw new ForbiddenException('Authentication failed.');
}
if ($loginByAuthKeyResult === null) {
$this->loadModel('Log');
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");

14
app/Controller/Component/IndexFilterComponent.php
View File

@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
{
/** @var Controller */
public $Controller;
public $isRest = null;
/** @var bool|null */
private $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
}
}
public function isXhr()
{
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
}
public function isJson()
{
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
}
public function isXml()
{
}
/**
* @param string $controller
* @param string $action

2
app/Controller/ServersController.php
View File

@ -1861,7 +1861,7 @@ class ServersController extends AppController
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
$this->Server->getBackgroundJobsTool()->clearQueue($worker);
} else {
// CakeResque
$worker_array = array('cache', 'default', 'email', 'prio');

22
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
/**
* Run the job command
* @param callable|null $runningCallback
*/
public function run(): void
public function run(callable $runningCallback = null): void
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id]
);
$this->pool($process, $pipes);
$this->pool($process, $pipes, $runningCallback);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
}
}
private function pool($process, array $pipes)
/**
* @param resource $process
* @param array $pipes
* @param callable|null $runningCallback
* @return void
*/
private function pool($process, array $pipes, callable $runningCallback = null)
{
stream_set_blocking($pipes[1], false);
stream_set_blocking($pipes[2], false);
@ -118,6 +125,12 @@ class BackgroundJob implements JsonSerializable
$this->error .= stream_get_contents($pipes[2]);
}
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
if ($runningCallback) {
$runningCallback($status);
}
if (!$status['running']) {
// Just in case read rest data from stream
$this->output .= stream_get_contents($pipes[1]);
@ -153,6 +166,9 @@ class BackgroundJob implements JsonSerializable
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
}
/**
* @return string Background job ID in UUID format
*/
public function id(): string
{
return $this->id;

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -65,7 +65,7 @@ class Worker implements JsonSerializable
];
}
public function pid(): ?int
public function pid(): int
{
return $this->pid;
}

62
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -91,7 +91,8 @@ class BackgroundJobsTool
];
const JOB_STATUS_PREFIX = 'job_status',
DATA_CONTENT_PREFIX = 'data_content';
DATA_CONTENT_PREFIX = 'data_content',
RUNNING_JOB_PREFIX = 'running';
/** @var array */
private $settings;
@ -277,6 +278,49 @@ class BackgroundJobsTool
return null;
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function markAsRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->setex($key, 60, $worker->pid());
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function removeFromRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->del($key);
}
/**
* Return current running jobs
* @param string $queue
* @return string[] Background jobs IDs
* @throws RedisException
*/
public function runningJobs(string $queue): array
{
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':' . $queue . ':*');
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
$jobIds = [];
foreach ($keys as $key) {
$parts = explode(':', $key);
$jobIds[] = end($parts);
}
return $jobIds;
}
/**
* Get the job status.
*
@ -500,19 +544,6 @@ class BackgroundJobsTool
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/**
* Return Background Jobs status
*
@ -728,8 +759,7 @@ class BackgroundJobsTool
*
* @param integer $pid
* @return \Supervisor\Process
*
* @throws NotFoundException
* @throws NotFoundException|Exception
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{

376
app/Lib/Tools/CurlClient.php Normal file
View File

@ -0,0 +1,376 @@
<?php
App::uses('HttpSocketExtended', 'Tools');
class CurlClient extends HttpSocketExtended
{
/** @var resource */
private $ch;
/** @var int */
private $timeout = 30;
/** @var string|null */
private $caFile;
/** @var string|null */
private $localCert;
/** @var int */
private $cryptoMethod;
/** @var bool */
private $allowSelfSigned;
/** @var bool */
private $verifyPeer;
/** @var bool */
private $compress = true;
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
*/
public function __construct(array $params)
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
}
if (isset($params['ssl_local_cert'])) {
$this->localCert = $params['ssl_local_cert'];
}
if (isset($params['compress'])) {
$this->compress = $params['compress'];
}
if (isset($params['ssl_crypto_method'])) {
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
}
if (isset($params['ssl_allow_self_signed'])) {
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
}
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function head($uri = null, $query = [], $request = [])
{
return $this->internalRequest('HEAD', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function get($uri = null, $query = [], $request = [])
{
return $this->internalRequest('GET', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function post($uri = null, $data = [], $request = [])
{
return $this->internalRequest('POST', $uri, $data, $request);
}
/**
* @param string $uri
* @param array$data
* @param $request
* @return HttpSocketResponseExtended
*/
public function put($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PUT', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function patch($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PATCH', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function delete($uri = null, $data = array(), $request = array())
{
return $this->internalRequest('DELETE', $uri, $data, $request);
}
public function url($url = null, $uriTemplate = null)
{
throw new Exception('Not implemented');
}
public function request($request = array())
{
throw new Exception('Not implemented');
}
public function setContentResource($resource)
{
throw new Exception('Not implemented');
}
public function getMetaData()
{
return null; // not supported by curl extension
}
/**
* @param string $host
* @param int $port
* @param string $method
* @param string $user
* @param string $pass
* @return void
*/
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
{
if (empty($host)) {
$this->proxy = [];
return;
}
if (is_array($host)) {
$this->proxy = $host + ['host' => null];
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
}
/**
* @param string $method
* @param string $url
* @param array|string $query
* @param array $request
* @return HttpSocketResponseExtended
*/
private function internalRequest($method, $url, $query, $request)
{
if (empty($url)) {
throw new InvalidArgumentException("No URL provided.");
}
if (!$this->ch) {
// Share handle between requests to allow keep connection alive between requests
$this->ch = curl_init();
if (!$this->ch) {
throw new \RuntimeException("Could not initialize cURL");
}
} else {
// Reset options, so we can do another request
curl_reset($this->ch);
}
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->defaultOptions;
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
$options[CURLOPT_POSTFIELDS] = $query;
}
if (!empty($request['header'])) {
$headers = [];
foreach ($request['header'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$headers[] = "$key: $value";
}
$options[CURLOPT_HTTPHEADER] = $headers;
}
// Parse response headers
$responseHeaders = [];
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
$len = strlen($header);
$header = explode(':', $header, 2);
if (count($header) < 2) { // ignore invalid headers
return $len;
}
$key = strtolower(trim($header[0]));
$value = trim($header[1]);
if (isset($responseHeaders[$key])) {
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
} else {
$responseHeaders[$key] = $value;
}
return $len;
};
if (!curl_setopt_array($this->ch, $options)) {
throw new \RuntimeException('cURL error: Could not set options');
}
// Download the given URL, and return output
$output = curl_exec($this->ch);
if ($output === false) {
$errorMessage = curl_error($this->ch);
if (!empty($errorMessage)) {
$errorMessage = ": $errorMessage";
}
throw new SocketException('cURL error ' . curl_strerror(curl_errno($this->ch)) . $errorMessage);
}
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
return $this->constructResponse($output, $responseHeaders, $code);
}
public function disconnect()
{
if ($this->ch) {
curl_close($this->ch);
$this->ch = null;
}
}
/**
* @param string $body
* @param array $headers
* @param int $code
* @return HttpSocketResponseExtended
*/
private function constructResponse($body, array $headers, $code)
{
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
if (!function_exists('zstd_uncompress')) {
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
}
$body = zstd_uncompress($body);
if ($body === false) {
throw new SocketException('Could not decode zstd encoded response.');
}
}
$response = new HttpSocketResponseExtended();
$response->code = $code;
$response->body = $body;
$response->headers = $headers;
return $response;
}
/**
* @param int $cryptoMethod
* @return int
*/
private function convertCryptoMethod($cryptoMethod)
{
switch ($cryptoMethod) {
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1;
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_1;
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_2;
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_3;
default:
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
}
}
/**
* @return array
*/
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
CURLOPT_MAXREDIRS => 10,
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
CURLOPT_HEADER => false, // Include header in result?
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
];
if ($this->caFile) {
$options[CURLOPT_CAINFO] = $this->caFile;
}
if ($this->localCert) {
$options[CURLOPT_SSLCERT] = $this->localCert;
}
if ($this->cryptoMethod) {
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
}
if ($this->compress) {
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
}
if ($this->allowSelfSigned) {
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
$options[CURLOPT_SSL_VERIFYHOST] = 0;
}
if (!empty($this->proxy)) {
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
}
}
return $options;
}
/**
* @return string
*/
private function supportedEncodings()
{
$encodings = [];
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
if (function_exists('zstd_uncompress')) {
$encodings[] = 'zstd';
}
// brotli and gzip is supported by curl itself if it is compiled with these features
$info = curl_version();
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
$encodings[] = 'br';
}
if ($info['features'] & CURL_VERSION_LIBZ) {
$encodings[] = 'gzip, deflate';
}
return implode(', ', $encodings);
}
}

21
app/Lib/Tools/RedisTool.php
View File

@ -57,24 +57,37 @@ class RedisTool
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @return Generator<string>
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
public static function keysByPattern(Redis $redis, $pattern)
{
if (is_string($pattern)) {
$pattern = [$pattern];
}
$allKeys = [];
foreach ($pattern as $p) {
$iterator = null;
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
foreach ($keys as $key) {
$allKeys[] = $key;
yield $key;
}
}
}
}
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
{
$allKeys = [];
foreach (self::keysByPattern($redis, $pattern) as $key) {
$allKeys[] = $key;
}
if (empty($allKeys)) {
return 0;

11
app/Lib/Tools/SyncTool.php
View File

@ -1,5 +1,4 @@
<?php
class SyncTool
{
@ -84,8 +83,14 @@ class SyncTool
$params['ssl_crypto_method'] = $version;
}
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
}
$proxy = Configure::read('Proxy');
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

26
app/Model/AttachmentScan.php
View File

@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
/** @var Job $job */
$job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) {
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
$jobId = null;
}
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
if ($infected === true) {
$virusFound++;
$scanned++;
} else if ($infected === false) {
$scanned++;
}
$scanned++;
} catch (NotFoundException $e) {
// skip if file doesn't exists
} catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
$fails++;
}
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
Job::WORKER_PRIO,
'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'scanAttachment',
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
}
/**
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
*
* @param string $type
* @param array $attribute
* @param array $moduleInfo
* @return bool|null Return true if attachment is infected.
* @return bool|null
* @throws Exception
*/
private function scanAttachment($type, array $attribute, array $moduleInfo)
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
return false; // empty file is automatically considered as not infected
}
/* if ($file->size() > 50 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
return false;
}*/
if ($fileSize > 25 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
return null;
}
$fileContent = $file->read();
if ($fileContent === false) {

23
app/Model/Attribute.php
View File

@ -545,6 +545,28 @@ class Attribute extends AppModel
return $result;
}
/**
* This method is called after all data are successfully saved into database
* @return void
* @throws Exception
*/
private function afterDatabaseSave(array $data)
{
$attribute = $data['Attribute'];
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
}
public function save($data = null, $validate = true, $fieldList = array())
{
$result = parent::save($data, $validate, $fieldList);
if ($result) {
$this->afterDatabaseSave($result);
}
return $result;
}
public function beforeDelete($cascade = true)
{
// delete attachments from the disk
@ -881,7 +903,6 @@ class Attribute extends AppModel
}
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
// Clean thumbnail cache
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
$redis = RedisTool::init();

48
app/Model/Galaxy.php
View File

@ -287,24 +287,42 @@ class Galaxy extends AppModel
if (empty($galaxy['uuid'])) {
return false;
}
$existingGalaxy = $this->find('first', array(
$existingGalaxy = $this->find('first', [
'recursive' => -1,
'conditions' => array('Galaxy.uuid' => $galaxy['uuid'])
));
if (empty($existingGalaxy)) {
if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) {
$this->create();
unset($galaxy['id']);
$this->save($galaxy);
$existingGalaxy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Galaxy.id' => $this->id)
));
} else {
return false;
'conditions' => ['Galaxy.uuid' => $galaxy['uuid']],
]);
unset($galaxy['id']);
if (!empty($existingGalaxy)) {
// check if provided galaxy has the same fields as galaxy that are saved in database
$fieldsToSave = [];
foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) {
if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) {
$fieldsToSave[$key] = $galaxy[$key];
}
}
} else {
$fieldsToSave = $galaxy;
}
return $existingGalaxy;
if (empty($fieldsToSave) && !empty($existingGalaxy)) {
return $existingGalaxy; // galaxy already exists and galaxy fields are the same
}
if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) {
return false; // user has no permission to modify galaxy
}
if (empty($existingGalaxy)) {
$this->create();
}
$this->save($fieldsToSave);
return $this->find('first', [
'recursive' => -1,
'conditions' => ['Galaxy.id' => $this->id],
]);
}
/**

48
app/Model/Module.php
View File

@ -50,6 +50,8 @@ class Module extends AppModel
)
);
private $httpSocket = [];
public function validateIPField($value)
{
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
@ -309,16 +311,9 @@ class Module extends AppModel
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
@ -422,4 +417,37 @@ class Module extends AppModel
return false;
}
/**
* @param string $moduleFamily
* @param int $timeout
* @return HttpSocketExtended|CurlClient
*/
private function initHttpSocket($moduleFamily, $timeout)
{
$unique = "$moduleFamily:$timeout";
if (isset($this->httpSocket[$unique])) {
return $this->httpSocket[$unique];
}
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$httpSocket = new CurlClient($httpSocketSetting);
} else {
App::uses('HttpSocketExtended', 'Tools');
$httpSocket = new HttpSocketExtended($httpSocketSetting);
}
return $this->httpSocket[$unique] = $httpSocket;
}
}

34
app/Model/Server.php
View File

@ -472,7 +472,21 @@ class Server extends AppModel
return false;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false)
/**
* @param array $event
* @param int|string $eventId
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param array $server
* @param array $user
* @param int $jobId
* @param bool $force
* @param HttpSocketResponseExtended $response
* @return false|void
* @throws Exception
*/
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', [
@ -485,7 +499,7 @@ class Server extends AppModel
if (!$existingEvent) {
// add data for newly imported events
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
return false;
}
@ -505,7 +519,7 @@ class Server extends AppModel
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
if ($existingEvent['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
}
}
@ -549,10 +563,8 @@ class Server extends AppModel
$params['excludeLocalTags'] = 1;
}
try {
$event = $serverSync->fetchEvent($eventId, $params);
$headers = $event->headers;
$body = $event->body;
$event = $event->json();
$response = $serverSync->fetchEvent($eventId, $params);
$event = $response->json();
} catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e);
$fails[$eventId] = __('failed downloading the event');
@ -568,7 +580,7 @@ class Server extends AppModel
}
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
return true;
}
@ -4793,11 +4805,11 @@ class Server extends AppModel
$results = [
__('User') => $user['User']['email'],
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'),
__('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
];
if (isset($response->headers['X-Auth-Key-Expiration'])) {
$date = new DateTime($response->headers['X-Auth-Key-Expiration']);
if ($response->getHeader('X-Auth-Key-Expiration')) {
$date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
}
return $results;

1
app/composer.json
View File

@ -36,6 +36,7 @@
"ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system",
"ext-simdjson": "To decode JSON structures faster",
"ext-curl": "For faster remote requests",
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3",
"jakub-onderka/openid-connect-php": "For OIDC authentication",