MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [workflow] Better module loading and execution errors get propagated to the caller for blocking path

pull/8530/head
Sami Mokaddem 2022-05-30 14:34:45 +02:00
parent 1a2d6e130d
commit 9e29830dfa
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
11 changed files with 132 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/Model/AppModel.php
View File

@ -3333,4 +3333,26 @@ class AppModel extends Model
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSource instanceof Mysql;
}
/**
* executeTrigger
*
* @param string $trigger_id
* @param array $data Data to be passed to the workflow
* @param array $errors
* @return boolean If the execution for the blocking path was a success
*/
public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[]): bool
{
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
if (
$this->Workflow->checkTriggerEnabled($trigger_id) &&
$this->Workflow->checkTriggerListenedTo($trigger_id)
) {
return $this->Workflow->executeWorkflowsForTrigger($trigger_id, $data, $blockingErrors);
}
return true;
}
}

103
app/Model/Workflow.php
View File

@ -3,6 +3,7 @@ App::uses('AppModel', 'Model');
App::uses('WorkflowGraphTool', 'Tools');
class WorkflowDuplicatedModuleIDException extends Exception {}
class TriggerNotFoundException extends Exception {}
class Workflow extends AppModel
{
@ -58,8 +59,10 @@ class Workflow extends AppModel
// 'User'
];
public $loaded_modules = [];
public $loaded_classes = [];
private $loaded_modules = [];
private $loaded_classes = [];
private $module_initialized = false;
const CAPTURE_FIELDS = ['name', 'description', 'timestamp', 'data'];
@ -73,7 +76,6 @@ class Workflow extends AppModel
{
parent::__construct($id, $table, $ds);
$this->workflowGraphTool = new WorkflowGraphTool();
$this->loadAllWorkflowModules(); // TODO Clever loading to avoid doing this task at every trigger check
}
public function beforeValidate($options = array())
@ -132,6 +134,19 @@ class Workflow extends AppModel
$this->updateListeningTriggers($this->workflowToDelete);
}
protected function checkTriggerEnabled($trigger_id)
{
$filename = sprintf('Module_%s.php', preg_replace('/[^a-zA-Z0-9_]/', '_', Inflector::underscore($trigger_id)));
$module_config = $this->__getClassFromModuleFiles('trigger', [$filename])['classConfigs'];
// FIXME: Merge global configuration!
return empty($module_config['disabled']);
}
protected function checkTriggerListenedTo($trigger_id)
{
return !empty($this->__getWorkflowsIDPerTrigger($trigger_id));
}
public function rebuildRedis($user)
{
$redis = $this->setupRedisWithException();
@ -441,25 +456,43 @@ class Workflow extends AppModel
return true;
}
public function executeWorkflowsForTrigger($trigger_id, array $data)
/**
* executeWorkflowsForTrigger
*
* @param string $trigger_id
* @param array $data
* @param array $errors
* @return boolean True if the execution for the blocking path was a success
* @throws TriggerNotFoundException
*/
public function executeWorkflowsForTrigger($trigger_id, array $data, array &$blockingErrors=[]): bool
{
$this->loadAllWorkflowModules();
$user = ['Role' => ['perm_site_admin' => true]];
if (empty($this->loaded_modules['trigger'][$trigger_id])) {
return false;
throw new TriggerNotFoundException(__('Unknown trigger `%s`', $trigger_id));
}
$trigger = $this->loaded_modules['trigger'][$trigger_id];
if (!empty($trigger['disabled'])) {
return true;
}
$blockingPathExecutionSuccess = true;
$workflowExecutionOrder = $this->getExecutionOrderForTrigger($user, $trigger, true);
$orderedBlockingWorkflows = $workflowExecutionOrder['blocking'];
$orderedDeferredWorkflows = $workflowExecutionOrder['non-blocking'];
foreach ($orderedBlockingWorkflows as $workflow) {
$continueExecution = $this->walkGraph($workflow, $trigger_id, 'blocking', $data);
$continueExecution = $this->walkGraph($workflow, $trigger_id, 'blocking', $data, $blockingErrors);
if (!$continueExecution) {
$blockingPathExecutionSuccess = false;
break;
}
}
foreach ($orderedDeferredWorkflows as $workflow) {
$this->walkGraph($workflow, $trigger_id, 'non-blocking');
}
return $blockingPathExecutionSuccess;
}
public function executeWorkflow($id, array $data=[])
@ -479,9 +512,11 @@ class Workflow extends AppModel
* @param array $workflow The worflow to walk
* @param string $trigger_id The ID of the trigger to start from
* @param bool|null $for_path If provided, execute the workflow for the provided path. If not provided, execute the worflow regardless of the path
* @param array $data
* @param array $errors
* @return boolean If all module returned a successful response
*/
private function walkGraph(array $workflow, $trigger_id, $for_path=null, $data=[]): bool
private function walkGraph(array $workflow, $trigger_id, $for_path=null, array $data=[], array &$errors=[]): bool
{
$workflowUser = $this->User->getAuthUser($workflow['Workflow']['user_id'], true);
$roamingData = $this->workflowGraphTool->getRoamingData($workflowUser, $data);
@ -494,7 +529,7 @@ class Workflow extends AppModel
foreach ($graphWalker as $graphNode) {
$node = $graphNode['node'];
$path_type = $graphNode['path_type'];
$success = $this->__executeNode($node, $roamingData);
$success = $this->__executeNode($node, $roamingData, $errors);
if (empty($success) && $path_type == 'blocking') {
return false; // Node stopped execution for blocking path
}
@ -503,12 +538,12 @@ class Workflow extends AppModel
return true;
}
public function __executeNode($node, WorkflowRoamingData $roamingData): bool
public function __executeNode($node, WorkflowRoamingData $roamingData, array &$errors): bool
{
$moduleClass = $this->getModuleClass($node);
if (!is_null($moduleClass)) {
try {
$success = $moduleClass->exec($node, $roamingData);
$success = $moduleClass->exec($node, $roamingData, $errors);
} catch (Exception $e) {
$message = sprintf(__('Error while executing module: %s'), $e->getMessage());
$this->__logError($node['data']['id'], $message);
@ -520,6 +555,7 @@ class Workflow extends AppModel
public function getModuleClass($node)
{
$this->loadAllWorkflowModules();
$moduleClass = $this->loaded_classes[$node['data']['module_type']][$node['data']['id']] ?? null;
return $moduleClass;
}
@ -558,7 +594,10 @@ class Workflow extends AppModel
public function loadAllWorkflowModules()
{
$phpModuleFiles = $this->__listPHPModuleFiles();
if ($this->module_initialized) {
return;
}
$phpModuleFiles = Workflow::__listPHPModuleFiles();
foreach ($phpModuleFiles as $type => $files) {
$classModuleFromFiles = $this->__getClassFromModuleFiles($type, $files);
foreach ($classModuleFromFiles['classConfigs'] as $i => $config) {
@ -577,6 +616,28 @@ class Workflow extends AppModel
}
$this->loaded_modules['action'] = array_merge($this->loaded_modules['action'], $misp_module_configs);
$this->loaded_classes['action'] = array_merge($this->loaded_classes['action'], $misp_module_class);
$this->__mergeGlobalConfigIntoLoadedModules();
$this->module_initialized = true;
}
private function __mergeGlobalConfigIntoLoadedModules()
{
/* FIXME: Delete `disabled` entry. This is for testing while we wait for module settings */
array_walk($this->loaded_modules['trigger'], function (&$trigger) {
$module_enabled = !in_array($trigger['id'], ['publish', 'new-attribute']);
$trigger['html_template'] = !empty($trigger['html_template']) ? $trigger['html_template'] : 'trigger';
$trigger['disabled'] = $module_enabled;
$this->loaded_classes['trigger'][$trigger['id']]->disabled = $module_enabled;
$this->loaded_classes['trigger'][$trigger['id']]->html_template = !empty($trigger['html_template']) ? $trigger['html_template'] : 'trigger';
});
array_walk($this->loaded_modules['logic'], function (&$logic) {
});
array_walk($this->loaded_modules['action'], function (&$action) {
$module_enabled = !in_array($action['id'], ['push-zmq', 'slack-message', 'mattermost-message', 'add-tag', 'writeactions',]);
$action['disabled'] = $module_enabled;
$this->loaded_classes['action'][$action['id']]->disabled = $module_enabled;
});
/* FIXME: end */
}
private function __getEnabledModulesFromModuleService($user)
@ -623,9 +684,18 @@ class Workflow extends AppModel
return $moduleClasses;
}
private function __listPHPModuleFiles()
/**
* __listPHPModuleFiles List all PHP modules files
*
* @param boolean|array $targetDir If provided, will only collect files from that directory
* @return array
*/
private static function __listPHPModuleFiles($targetDir=false): array
{
$dirs = ['trigger', 'logic', 'action'];
if (!empty($targetDir)) {
$dirs = $targetDir;
}
$files = [];
foreach ($dirs as $dir) {
$folder = new Folder(Workflow::MODULE_ROOT_PATH . $dir);
@ -704,17 +774,12 @@ class Workflow extends AppModel
public function getModulesByType($module_type=false): array
{
$this->loadAllWorkflowModules();
$blocks_trigger = $this->loaded_modules['trigger'];
$blocks_logic = $this->loaded_modules['logic'];
$blocks_action = $this->loaded_modules['action'];
array_walk($blocks_trigger, function(&$block) {
$block['html_template'] = !empty($block['html_template']) ? $block['html_template'] : 'trigger';
$block['disabled'] = !in_array($block['id'], ['publish', 'new-attribute', ]);
});
array_walk($blocks_action, function(&$block) {
$block['disabled'] = !in_array($block['id'], ['push-zmq', 'slack-message', 'mattermost-message', 'add-tag', 'writeactions', ]);
});
ksort($blocks_trigger);
ksort($blocks_logic);
ksort($blocks_action);

4
app/Model/WorkflowModules/Module_misp_module.php
View File

@ -19,7 +19,7 @@ class Module_misp_module extends WorkflowBaseModule
public function __construct($misp_module_config)
{
parent::__construct();
$this->id = Inflector::tableize($misp_module_config['name']);
$this->id = Inflector::underscore($misp_module_config['name']);
$this->name = $misp_module_config['name'];
$this->description = $misp_module_config['meta']['description'];
if (!empty($misp_module_config['meta']['icon'])) {
@ -38,7 +38,7 @@ class Module_misp_module extends WorkflowBaseModule
$this->Module = ClassRegistry::init('Module');
}
public function exec(array $node, WorkflowRoamingData $roamingData): bool
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData);
$postData = ['module' => $this->name];

2
app/Model/WorkflowModules/WorkflowBaseModule.php
View File

@ -52,7 +52,7 @@ class WorkflowBaseModule
return $properties;
}
public function exec(array $node, WorkflowRoamingData $roamingData): bool
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
$this->push_zmq([
'module' => $this->name,

7
app/Model/WorkflowModules/action/Module_add_tag.php
View File

@ -23,4 +23,11 @@ class Module_add_tag extends WorkflowBaseModule
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
{
parent::exec($node, $roamingData, $errors);
$errors[] = __('Could not add tag!');
return false;
}
}

4
app/Model/WorkflowModules/action/Module_push_zmq.php
View File

@ -31,9 +31,9 @@ class Module_push_zmq extends WorkflowBaseModule
];
}
public function exec(array $node, WorkflowRoamingData $roamingData): bool
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData);
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
// $this->push_zmq([
// 'Module_push_zmq has passed option' => $params['Content']['value']

4
app/Model/WorkflowModules/logic/Module_if.php
View File

@ -36,9 +36,9 @@ class Module_if extends WorkflowBaseModule
];
}
public function exec(array $node, WorkflowRoamingData $roamingData): bool
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData);
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$ifScope = $params['Scope']['value'];
$ifFilter = json_decode($params['Matching Conditions']['value'], true);

6
app/View/Workflows/editor.ctp
View File

@ -21,11 +21,11 @@ foreach ($modules as $moduleType => $module) {
</div>
<div class="" style="margin-top: 0.5em;">
<div class="btn-group" style="margin-left: 3px;">
<a class="btn btn-primary" href="#"><i class="fa-fw <?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('New') ?></a>
<a class="btn btn-primary" href="<?= $baseurl . '/workflows/add' ?>"><i class="fa-fw <?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('New') ?></a>
<a class="btn btn-primary dropdown-toggle" data-toggle="dropdown" href="#"><span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a id="exportWorkflow" href="#"><i class="fa-fw <?= $this->FontAwesome->getClass('file-export') ?>"></i> <?= __('Export workflow') ?></a></li>
<li><a id="importWorkflow" href="#"><i class="fa-fw <?= $this->FontAwesome->getClass('file-import') ?>"></i> <?= __('Import workflow') ?></a></li>
<li><a id="importWorkflow" href="<?= $baseurl . '/workflows/import/' ?>"><i class="fa-fw <?= $this->FontAwesome->getClass('file-import') ?>"></i> <?= __('Import workflow') ?></a></li>
<li><a id="exportWorkflow" href="<?= $baseurl . '/workflows/export/' . h($selectedWorkflow['Workflow']['id']) ?>"><i class="fa-fw <?= $this->FontAwesome->getClass('file-export') ?>"></i> <?= __('Export workflow') ?></a></li>
</ul>
</div>
<button id="saveWorkflow" class="btn btn-primary" href="#">

10
app/View/Workflows/index.ctp
View File

@ -32,9 +32,10 @@
'element' => 'custom',
'function' => function ($row) use ($baseurl) {
return implode('<br />', array_map(function($trigger) use ($baseurl) {
return sprintf('<a href="%s/workflows/triggerView/%s"><i class="fa-fw %s"></i> %s</a>',
return sprintf('<a href="%s/workflows/triggerView/%s" %s><i class="fa-fw %s"></i> %s</a>',
$baseurl,
h($trigger['id']),
!empty($trigger['disabled']) ? sprintf('class="%s" style="%s" title="%s"', 'muted', 'text-decoration: line-through;', __('Trigger disabled')) : '',
$this->FontAwesome->getClass($trigger['icon']),
h($trigger['id'])
);
@ -128,17 +129,18 @@
[
'url' => $baseurl . '/workflows/view',
'url_params_data_paths' => ['Workflow.id'],
'icon' => 'eye'
'icon' => 'eye',
],
[
'url' => $baseurl . '/workflows/editor',
'url_params_data_paths' => ['Workflow.id'],
'icon' => 'code'
'icon' => 'code',
'dbclickAction' => true,
],
[
'url' => $baseurl . '/workflows/edit',
'url_params_data_paths' => ['Workflow.id'],
'icon' => 'edit'
'icon' => 'edit',
],
[
'url' => $baseurl . '/workflows/export',

3
app/View/Workflows/trigger_index.ctp
View File

@ -92,7 +92,8 @@
[
'url' => $baseurl . '/workflows/triggerView',
'url_params_data_paths' => ['id'],
'icon' => 'eye'
'icon' => 'eye',
'dbclickAction' => true,
],
]
]

2
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -158,7 +158,7 @@ function initDrawflow() {
var centroidY = sumY / nodes.length
centroidX -= offset_block_x / 2
centroidY += offset_block_y / 2
var calc_zoom = Math.min(editor_bcr.width / maxX, editor_bcr.height / maxY) // Zoom out if needed
var calc_zoom = Math.min(Math.min(editor_bcr.width / maxX, editor_bcr.height / maxY), 1) // Zoom out if needed
editor.translate_to(
offset_x - centroidX + offset_block_x / 2 - 100, // Harcoded offset as it's more pleasant if it's slightly positioned top-left
offset_y - centroidY + offset_block_y / 2 - 200 // Harcoded offset as it's more pleasant if it's slightly positioned top-left