MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #9480 from JakubOnderka/attachment-scan 

Attachment scan
pull/9491/head
Jakub Onderka 2024-01-12 09:57:19 +01:00 committed by GitHub
parent 0e0eed218f 57459063cd
commit f6db5e7e6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 268 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Command/AdminShell.php
View File

@ -72,7 +72,7 @@ class AdminShell extends AppShell
'help' => __('Set if MISP instance is live and accessible for users.'),
'parser' => [
'arguments' => [
'state' => ['help' => __('Set Live state')],
'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
],
],
]);
@ -951,7 +951,7 @@ class AdminShell extends AppShell
$newStatus = $this->toBoolean($this->args[0]);
$overallSuccess = false;
try {
$redis = $this->Server->setupRedisWithException();
$redis = RedisTool::init();
if ($newStatus) {
$redis->del('misp:live');
$this->out('Set live status to True in Redis.');
@ -980,7 +980,7 @@ class AdminShell extends AppShell
} else {
$this->out('Current status:');
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
$newStatus = $this->Server->setupRedisWithException()->get('misp:live');
$newStatus = RedisTool::init()->get('misp:live');
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
}
}

3
app/Console/Command/LiveShell.php
View File

@ -1,8 +1,9 @@
<?php
/*
/**
* Enable/disable misp
*
* arg0 = [0|1]
* @deprecated Use AdminShell::live instead
*/
class LiveShell extends AppShell {

26
app/Console/Command/StartWorkerShell.php
View File

@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
public function main()
{
$pid = getmypid();
if ($pid === false) {
throw new RuntimeException("Could not get current process ID");
}
$this->worker = new Worker(
[
'pid' => getmypid(),
'pid' => $pid,
'queue' => $this->args[0],
'user' => ProcessTool::whoami(),
]
);
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
$queue = $this->worker->queue();
$backgroundJobTool = $this->getBackgroundJobsTool();
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
while (true) {
$this->checkMaxExecutionTime();
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
$job = $backgroundJobTool->dequeue($queue);
if ($job) {
$this->runJob($job);
$backgroundJobTool->removeFromRunning($this->worker, $job);
}
}
}
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
*/
private function runJob(BackgroundJob $job)
{
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
try {
$job->setStatus(BackgroundJob::STATUS_RUNNING);
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
$this->getBackgroundJobsTool()->update($job);
$job->run();
$start = microtime(true);
$job->run(function (array $status) use ($job) {
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job);
});
$duration = number_format(microtime(true) - $start, 3, '.', '');
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
} else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
}
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

112
app/Console/Command/WorkerShell.php Normal file
View File

@ -0,0 +1,112 @@
<?php
declare(strict_types=1);
/**
* @property Job $Job
*/
class WorkerShell extends AppShell
{
public $uses = ['Job'];
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser->addSubcommand('showQueues', [
'help' => __('Show jobs in worker queues'),
]);
$parser->addSubcommand('flushQueue', [
'help' => __('Flush jobs in given queue'),
'parser' => [
'arguments' => [
'queue' => ['help' => __('Queue name'), 'required' => true],
],
],
]);
$parser->addSubcommand('showJobStatus', [
'help' => __('Show job status'),
'parser' => [
'arguments' => [
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
],
],
]);
return $parser;
}
public function showQueues()
{
$tool = $this->getBackgroundJobsTool();
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
foreach ($tool->runningJobs($queue) as $jobId) {
$this->out(" - $jobId");
}
}
}
public function flushQueue()
{
$queue = $this->args[0];
try {
$this->getBackgroundJobsTool()->clearQueue($queue);
} catch (InvalidArgumentException $e) {
$this->error($e->getMessage());
}
}
public function showJobStatus()
{
$processId = $this->args[0];
if (is_numeric($processId)) {
$job = $this->Job->find('first', [
'conditions' => ['Job.id' => $processId],
'recursive' => -1,
]);
if (!$job) {
$this->error('Job not found', "Job with ID {$processId} not found");
}
$this->out($this->json($job['Job']));
$processId = $job['Job']['process_id'];
}
if (!Validation::uuid($processId)) {
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
}
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
if (!$jobStatus) {
$this->error('Job not found', "Job with UUID {$processId} not found");
}
$jobStatus = $jobStatus->jsonSerialize();
foreach (['createdAt', 'updatedAt'] as $timeField) {
if (isset($jobStatus[$timeField])) {
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
}
}
if (isset($jobStatus['status'])) {
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
}
$this->out($this->json($jobStatus));
}
private function jobStatusToString(int $jobStatus)
{
switch ($jobStatus) {
case Job::STATUS_WAITING:
return 'waiting';
case Job::STATUS_RUNNING:
return 'running';
case Job::STATUS_FAILED:
return 'failed';
case Job::STATUS_COMPLETED:
return 'completed';
}
throw new InvalidArgumentException("Invalid job status $jobStatus");
}
}

4
app/Controller/AppController.php
View File

@ -230,6 +230,10 @@ class AppController extends Controller
$this->Security->csrfCheck = false;
$loginByAuthKeyResult = $this->__loginByAuthKey();
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
if ($this->IndexFilter->isXhr()) {
throw new ForbiddenException('Authentication failed.');
}
if ($loginByAuthKeyResult === null) {
$this->loadModel('Log');
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");

14
app/Controller/Component/IndexFilterComponent.php
View File

@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
{
/** @var Controller */
public $Controller;
public $isRest = null;
/** @var bool|null */
private $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
}
}
public function isXhr()
{
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
}
public function isJson()
{
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
}
public function isXml()
{
}
/**
* @param string $controller
* @param string $action

2
app/Controller/ServersController.php
View File

@ -1861,7 +1861,7 @@ class ServersController extends AppController
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
$this->Server->getBackgroundJobsTool()->clearQueue($worker);
} else {
// CakeResque
$worker_array = array('cache', 'default', 'email', 'prio');

22
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
/**
* Run the job command
* @param callable|null $runningCallback
*/
public function run(): void
public function run(callable $runningCallback = null): void
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id]
);
$this->pool($process, $pipes);
$this->pool($process, $pipes, $runningCallback);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
}
}
private function pool($process, array $pipes)
/**
* @param resource $process
* @param array $pipes
* @param callable|null $runningCallback
* @return void
*/
private function pool($process, array $pipes, callable $runningCallback = null)
{
stream_set_blocking($pipes[1], false);
stream_set_blocking($pipes[2], false);
@ -118,6 +125,12 @@ class BackgroundJob implements JsonSerializable
$this->error .= stream_get_contents($pipes[2]);
}
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
if ($runningCallback) {
$runningCallback($status);
}
if (!$status['running']) {
// Just in case read rest data from stream
$this->output .= stream_get_contents($pipes[1]);
@ -153,6 +166,9 @@ class BackgroundJob implements JsonSerializable
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
}
/**
* @return string Background job ID in UUID format
*/
public function id(): string
{
return $this->id;

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -65,7 +65,7 @@ class Worker implements JsonSerializable
];
}
public function pid(): ?int
public function pid(): int
{
return $this->pid;
}

62
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -91,7 +91,8 @@ class BackgroundJobsTool
];
const JOB_STATUS_PREFIX = 'job_status',
DATA_CONTENT_PREFIX = 'data_content';
DATA_CONTENT_PREFIX = 'data_content',
RUNNING_JOB_PREFIX = 'running';
/** @var array */
private $settings;
@ -277,6 +278,49 @@ class BackgroundJobsTool
return null;
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function markAsRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->setex($key, 60, $worker->pid());
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function removeFromRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->del($key);
}
/**
* Return current running jobs
* @param string $queue
* @return string[] Background jobs IDs
* @throws RedisException
*/
public function runningJobs(string $queue): array
{
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':' . $queue . ':*');
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
$jobIds = [];
foreach ($keys as $key) {
$parts = explode(':', $key);
$jobIds[] = end($parts);
}
return $jobIds;
}
/**
* Get the job status.
*
@ -500,19 +544,6 @@ class BackgroundJobsTool
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/**
* Return Background Jobs status
*
@ -728,8 +759,7 @@ class BackgroundJobsTool
*
* @param integer $pid
* @return \Supervisor\Process
*
* @throws NotFoundException
* @throws NotFoundException|Exception
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{

21
app/Lib/Tools/RedisTool.php
View File

@ -57,24 +57,37 @@ class RedisTool
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @return Generator<string>
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
public static function keysByPattern(Redis $redis, $pattern)
{
if (is_string($pattern)) {
$pattern = [$pattern];
}
$allKeys = [];
foreach ($pattern as $p) {
$iterator = null;
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
foreach ($keys as $key) {
$allKeys[] = $key;
yield $key;
}
}
}
}
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
{
$allKeys = [];
foreach (self::keysByPattern($redis, $pattern) as $key) {
$allKeys[] = $key;
}
if (empty($allKeys)) {
return 0;

26
app/Model/AttachmentScan.php
View File

@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
/** @var Job $job */
$job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) {
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
$jobId = null;
}
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
if ($infected === true) {
$virusFound++;
$scanned++;
} else if ($infected === false) {
$scanned++;
}
$scanned++;
} catch (NotFoundException $e) {
// skip if file doesn't exists
} catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
$fails++;
}
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
Job::WORKER_PRIO,
'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'scanAttachment',
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
}
/**
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
*
* @param string $type
* @param array $attribute
* @param array $moduleInfo
* @return bool|null Return true if attachment is infected.
* @return bool|null
* @throws Exception
*/
private function scanAttachment($type, array $attribute, array $moduleInfo)
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
return false; // empty file is automatically considered as not infected
}
/* if ($file->size() > 50 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
return false;
}*/
if ($fileSize > 25 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
return null;
}
$fileContent = $file->read();
if ($fileContent === false) {

23
app/Model/Attribute.php
View File

@ -545,6 +545,28 @@ class Attribute extends AppModel
return $result;
}
/**
* This method is called after all data are successfully saved into database
* @return void
* @throws Exception
*/
private function afterDatabaseSave(array $data)
{
$attribute = $data['Attribute'];
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
}
public function save($data = null, $validate = true, $fieldList = array())
{
$result = parent::save($data, $validate, $fieldList);
if ($result) {
$this->afterDatabaseSave($result);
}
return $result;
}
public function beforeDelete($cascade = true)
{
// delete attachments from the disk
@ -881,7 +903,6 @@ class Attribute extends AppModel
}
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
// Clean thumbnail cache
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
$redis = RedisTool::init();