MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #6485 from JakubOnderka/module-timeout 

chg: [module] Allow to specify module timeout
pull/6498/head
Jakub Onderka 2020-10-24 18:06:11 +02:00 committed by GitHub
parent 87445dcc5f b0c28858c0
commit b97e96ffc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 68 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/AttributesController.php
View File

@ -2456,7 +2456,7 @@ class AttributesController extends AppController
} else {
$data[$attribute[0]['Attribute']['type']] = $attribute[0]['Attribute']['value'];
}
$result = $this->Module->queryModuleServer('/query', $data, true);
$result = $this->Module->queryModuleServer($data, true);
if ($result) {
if (!is_array($result)) {
$resultArray[$type] = ['error' => $result];

6
app/Controller/EventsController.php
View File

@ -4762,7 +4762,7 @@ class EventsController extends AppController
if (!empty($options)) {
$data['config'] = $options;
}
$result = $this->Module->queryModuleServer('/query', $data, false, $type);
$result = $this->Module->queryModuleServer($data, false, $type);
if (!$result) {
throw new MethodNotAllowedException(__('%s service not reachable.', $type));
}
@ -4807,7 +4807,7 @@ class EventsController extends AppController
if (!empty($options)) {
$data['config'] = $options;
}
$result = $this->Module->queryModuleServer('/query', $data, false, $type);
$result = $this->Module->queryModuleServer($data, false, $type);
if (!$result) {
throw new MethodNotAllowedException(__('%s service not reachable.', $type));
}
@ -4997,7 +4997,7 @@ class EventsController extends AppController
if (!empty($filename)) {
$modulePayload['filename'] = $filename;
}
$result = $this->Module->queryModuleServer('/query', $modulePayload, false, $moduleFamily = 'Import');
$result = $this->Module->queryModuleServer($modulePayload, false, $moduleFamily = 'Import');
if (!$result) {
throw new Exception(__('Import service not reachable.'));
}

2
app/Controller/ModulesController.php
View File

@ -42,7 +42,7 @@ class ModulesController extends AppController
}
// Query
$result = $this->Module->queryModuleServer('/query', $data, true);
$result = $this->Module->queryModuleServer($data, true);
if (!$result) {
$result = array('error' => 'Something went wrong, no response from module.');
}

5
app/Model/AttachmentScan.php
View File

@ -390,14 +390,17 @@ class AttachmentScan extends AppModel
*/
private function sendToModule(array $attribute, array $moduleConfig)
{
// How long we will wait for scan result
$timeout = Configure::read('MISP.attachment_scan_timeout') ?: 30;
$data = [
'module' => $this->attachmentScanModuleName,
'attribute' => $attribute,
'event_id' => $attribute['event_id'],
'config' => $moduleConfig,
'timeout' => $timeout, // module internal timeout
];
$results = $this->moduleModel()->queryModuleServer('/query', $data, false, 'Enrichment', true);
$results = $this->moduleModel()->sendRequest('/query', $timeout + 1, $data, 'Enrichment');
if (isset($results['error'])) {
throw new Exception("{$this->attachmentScanModuleName} module returns error: " . $results['error']);
}

4
app/Model/Event.php
View File

@ -5755,7 +5755,7 @@ class Event extends AppModel
}
}
$modulePayload['data'] = $events;
$result = $this->Module->queryModuleServer('/query', $modulePayload, false, 'Export');
$result = $this->Module->queryModuleServer($modulePayload, false, 'Export');
return array(
'data' => $result['data'],
'extension' => $module['mispattributes']['outputFileExtension'],
@ -6139,7 +6139,7 @@ class Event extends AppModel
} else {
$data[$attribute['type']] = $attribute['value'];
}
$result = $this->Module->queryModuleServer('/query', $data, false, 'Enrichment');
$result = $this->Module->queryModuleServer($data, false, 'Enrichment');
if (!$result) {
throw new MethodNotAllowedException(h($module['name']) . ' service not reachable.');
}

105
app/Model/Module.php
View File

@ -90,11 +90,15 @@ class Module extends AppModel
*/
public function getModules($moduleFamily = 'Enrichment', $throwException = false)
{
$modules = $this->queryModuleServer('/modules', false, false, $moduleFamily, $throwException);
if ($modules === false) { // not possible when $throwException is true
try {
// Wait just one second to not block loading pages when modules are not reachable
return $this->sendRequest('/modules', 1, null, $moduleFamily);
} catch (Exception $e) {
if ($throwException) {
throw $e;
}
return 'Module service not reachable.';
}
return $modules;
}
public function getEnabledModules($user, $type = false, $moduleFamily = 'Enrichment')
@ -195,44 +199,57 @@ class Module extends AppModel
}
/**
* @param string $uri
* @param array|false $post
* Send request to `/query` module endpoint.
*
* @param array $postData
* @param bool $hover
* @param string $moduleFamily
* @param bool $throwException
* @return array|false
* @throws JsonException
*/
public function queryModuleServer($uri, $post = false, $hover = false, $moduleFamily = 'Enrichment', $throwException = false)
public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false)
{
if ($hover) {
$timeout = Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5;
} else {
$timeout = Configure::read('Plugin.' . $moduleFamily . '_timeout') ?: 10;
}
try {
return $this->sendRequest('/query', $timeout, $postData, $moduleFamily);
} catch (Exception $e) {
if ($throwException) {
throw $e;
}
$this->logException('Failed to query module ' . $moduleFamily, $e);
return false;
}
}
/**
* Low-level way how to send request to module.
*
* @param string $uri
* @param int $timeout
* @param array|null $postData
* @param string $moduleFamily
* @return array
* @throws JsonException
*/
public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
{
$url = $this->__getModuleServer($moduleFamily);
if (!$url) {
if ($throwException) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
return false;
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocket', 'Network/Http');
if ($hover) {
$settings = array(
'timeout' => Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5
);
} else {
$settings = array(
'timeout' => Configure::read('Plugin.' . $moduleFamily . '_timeout') ?: 10
);
}
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
$settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
}
}
// let's set a low timeout for the introspection so that we don't block the loading of pages due to a misconfigured modules
if ($uri == '/modules') {
$settings['timeout'] = 1;
}
$httpSocket = new HttpSocket($settings);
$httpSocket = new HttpSocket(['timeout' => $timeout]);
$request = array(
'header' => array(
'Content-Type' => 'application/json',
@ -243,33 +260,25 @@ class Module extends AppModel
$request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
}
}
try {
if ($post) {
if (!is_array($post)) {
throw new InvalidArgumentException("Post data must be array, " . gettype($post) . " given.");
}
$post = json_encode($post);
$response = $httpSocket->post($url . $uri, $post, $request);
} else {
if ($moduleFamily == 'Cortex') {
unset($request['header']['Content-Type']);
}
$response = $httpSocket->get($url . $uri, false, $request);
if ($postData) {
if (!is_array($postData)) {
throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
}
if (!$response->isOk()) {
if ($httpSocket->lastError()) {
throw new Exception("Failed to get response from $moduleFamily module " . $httpSocket->lastError());
}
throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
$post = json_encode($postData);
$response = $httpSocket->post($url . $uri, $post, $request);
} else {
if ($moduleFamily == 'Cortex') {
unset($request['header']['Content-Type']);
}
return $this->jsonDecode($response->body);
} catch (Exception $e) {
if ($throwException) {
throw $e;
}
$this->logException('Failed to query module ' . $moduleFamily, $e);
return false;
$response = $httpSocket->get($url . $uri, false, $request);
}
if (!$response->isOk()) {
if ($httpSocket->lastError()) {
throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
}
throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
}
return $this->jsonDecode($response->body);
}
/**