MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [workflow] Save state in redis and continued integration in the UI - WiP

pull/8530/head
Sami Mokaddem 2022-05-16 16:49:30 +02:00
parent c633b02ebd
commit 2aba7188ff
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
5 changed files with 198 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Controller/WorkflowsController.php
View File

@ -97,8 +97,8 @@ class WorkflowsController extends AppController
}
$this->set('id', $id);
$this->set('menuData', array('menuList' => 'workflows', 'menuItem' => 'view'));
$execution_path = $this->Workflow->getExecutionPath($this->Auth->user(), $id);
$this->set('execution_path', $execution_path);
// $execution_path = $this->Workflow->getExecutionPath($this->Auth->user(), $id);
// $this->set('execution_path', $execution_path);
}
public function enable($id)
@ -142,6 +142,7 @@ class WorkflowsController extends AppController
{
$modules = $this->Workflow->getModules();
$triggers = $modules['blocks_trigger'];
$triggers = $this->Workflow->attachWorkflowsToTriggers($this->Auth->user(), $triggers, true);
$this->set('data', $triggers);
$this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'index_trigger']);
}

178
app/Model/Workflow.php
View File

@ -44,7 +44,13 @@ class Workflow extends AppModel
// 'User'
];
const CAPTURE_FIELDS = ['name', 'description', 'timestamp', 'priority_level', '', 'data'];
const CAPTURE_FIELDS = ['name', 'description', 'timestamp', 'priority_level', 'data'];
const WORKFLOW_BLOCKING_PATH_NAME = 'output_1';
const WORKFLOW_NON_BLOCKING_PATH_NAME = 'output_2';
const REDIS_KEY_WORKFLOW_PER_TRIGGER = 'workflow:workflow_list:%s';
const REDIS_KEY_TRIGGER_PER_WORKFLOW = 'workflow:trigger_list:%s';
private $moduleByID = [];
@ -68,10 +74,98 @@ class Workflow extends AppModel
$result['Workflow']['data'] = '{}';
}
$results[$k]['Workflow']['data'] = JsonTool::decode($result['Workflow']['data']);
if (!empty($result['Workflow']['id'])) {
$results[$k]['Workflow']['listening_triggers'] = $this->getTriggersPerWorkflow((int) $result['Workflow']['id']);
}
}
return $results;
}
public function afterSave($created, $options = array())
{
$this->updateListeningTriggers($this->data['Workflow']);
}
/**
* updateListeningTriggers Regenerate the list of triggers that will run this workflow
* - collect trigger name for workflow
* - remove wf id from trigger list
* - remove trigger name from workflow
* - add wf id to trigger list
* - add trigger name to workflow
*
* @param array $workflow
*/
private function updateListeningTriggers($workflow)
{
try {
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
$workflow = $this->data;
$workflow['Workflow']['data'] = JsonTool::decode($workflow['Workflow']['data']);
$pipeline = $redis->pipeline();
$trigger_list = $this->getTriggersPerWorkflow((int)$workflow['Workflow']['id']);
foreach ($trigger_list as $trigger_name) {
$pipeline->sRem(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_name), $workflow['Workflow']['id']);
$pipeline->sRem(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow['Workflow']['id']), $trigger_name);
}
$listening_triggers = $this->extractTriggerFromWorkflow($workflow);
foreach ($listening_triggers as $trigger_name) {
$pipeline->sAdd(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_name), $workflow['Workflow']['id']);
$pipeline->sAdd(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow['Workflow']['id']), $trigger_name);
}
$pipeline->exec();
}
/**
* getWorkflowsPerTrigger Get list of workflow IDs listening to the specified trigger
*
* @param string $workflow
*/
private function getWorkflowsPerTrigger(string $trigger_name)
{
try {
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
return $redis->sMembers(sprintf(Workflow::REDIS_KEY_WORKFLOW_PER_TRIGGER, $trigger_name));
}
/**
* getTriggersPerWorkflow Get list of trigger name running to the specified workflow
*
* @param array $workflow
*/
private function getTriggersPerWorkflow(int $workflow_id)
{
try {
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
return $redis->sMembers(sprintf(Workflow::REDIS_KEY_TRIGGER_PER_WORKFLOW, $workflow_id));
}
/**
* getTriggerFromWorkflow Return the list of trigger names that are specified in the workflow
*
* @param array $workflow
* @return array
*/
public function extractTriggerFromWorkflow(array $workflow)
{
$triggers = [];
foreach ($workflow['Workflow']['data'] as $node) {
if ($node['data']['module_type'] == 'trigger') {
$triggers[] = $node['data']['id'];
}
}
return $triggers;
}
/**
* buildACLConditions Generate ACL conditions for viewing the workflow
*
@ -111,6 +205,78 @@ class Workflow extends AppModel
}
}
/**
* attachWorkflowsToTriggers Collect the workflows listening to this trigger
*
* @param array $user
* @param array $triggers
* @param bool $group_per_blocking Wheter or not the workflows should be grouped together if they have a blocking path set
* @return array
*/
public function attachWorkflowsToTriggers(array $user, array $triggers, bool $group_per_blocking=true): array
{
$workflow_IDs = [];
$workflows_per_trigger = [];
foreach ($triggers as $trigger) {
$workflow_IDs_for_trigger = $this->getWorkflowsPerTrigger($trigger['id']);
$workflows_per_trigger[$trigger['id']] = $workflow_IDs_for_trigger;
foreach ($workflow_IDs_for_trigger as $id) {
$workflow_IDs[$id] = true;
}
}
$workflow_IDs = array_keys($workflow_IDs);
$workflows = $this->fetchWorkflows($user, [
'conditions' => [
'Workflow.id' => $workflow_IDs
],
'fields' => ['*'],
'contain' => ['Organisation' => ['fields' => ['*']]],
]);
$workflows = Hash::combine($workflows, '{n}.Workflow.id', '{n}');
foreach ($triggers as $i => $trigger) {
$workflow_IDs = $workflows_per_trigger[$trigger['id']];
$triggers[$i]['Workflows'] = [];
foreach ($workflow_IDs as $workflow_ID) {
$triggers[$i]['Workflows'][] = $workflows[$workflow_ID];
}
usort($triggers[$i]['Workflows'], function($a, $b) {
return $a['Workflow']['priority_level'] - $b['Workflow']['priority_level'];
});
if (!empty($group_per_blocking)) {
$triggers[$i]['Workflows'] = $this->groupWorkflowsPerBlockingType($triggers[$i]['Workflows'], $trigger['id']);
}
}
return $triggers;
}
/**
* groupWorkflowsPerBlockingType Group workflows together if they have a blocking path set
*
* @param array $workflows
* @param string $trigger_name The trigger for which we should decide if it's blocking or not
* @return array
*/
public function groupWorkflowsPerBlockingType(array $workflows, string $trigger_name): array
{
$groupedWorkflows = [
'blocking' => [],
'non-blocking' => [],
];
foreach ($workflows as $workflow) {
foreach ($workflow['Workflow']['data'] as $block) {
if ($block['data']['id'] == $trigger_name) {
if (!empty($block['outputs'][Workflow::WORKFLOW_BLOCKING_PATH_NAME])) {
$groupedWorkflows['blocking'][] = $workflow;
}
if (!empty($block['outputs'][Workflow::WORKFLOW_NON_BLOCKING_PATH_NAME])) {
$groupedWorkflows['non-blocking'][] = $workflow;
}
}
}
}
return $groupedWorkflows;
}
public function getExecutionPath($user, $id): array
{
$this->loadModuleByID();
@ -187,7 +353,7 @@ class Workflow extends AppModel
'description' => 'Lorem ipsum dolor, sit amet consectetur adipisicing elit.',
'module_type' => 'trigger',
'inputs' => 0,
'disabled' => true,
// 'disabled' => true,
'outputs' => 2,
],
[
@ -421,7 +587,13 @@ class Workflow extends AppModel
$params['conditions']['AND'][] = $options['conditions'];
}
if (isset($options['group'])) {
$params['group'] = empty($options['group']) ? $options['group'] : false;
$params['group'] = !empty($options['group']) ? $options['group'] : false;
}
if (isset($options['contain'])) {
$params['contain'] = !empty($options['contain']) ? $options['contain'] : [];
}
if (isset($options['order'])) {
$params['order'] = !empty($options['order']) ? $options['order'] : [];
}
$workflows = $this->find('all', $params);
return $workflows;

37
app/View/Elements/Workflows/executionOrder.ctp
View File

@ -1,44 +1,37 @@
<?php
$trigger['execution_order'] = [
['name' => 'test1', 'id' => 1, 'enabled' => true, 'description' => 'test', 'Organisation' => ['id'=>1, 'name'=>'ORGNAME']],
['name' => 'test2', 'id' => 1, 'enabled' => false, 'description' => 'test', 'Organisation' => ['id'=>1, 'name'=>'ORGNAME']],
['name' => 'test3', 'id' => 1, 'enabled' => true, 'description' => 'test', 'Organisation' => ['id'=>1, 'name'=>'ORGNAME']]
];
?>
<div>
<ul class="unstyled">
<?php foreach ($trigger['execution_order'] as $i => $workflow) : ?>
<?php foreach ($trigger['Workflows']['blocking'] as $i => $workflow) : ?>
<li>
<?php if ($i == 0) : ?>
<i class="fa-fw <?= $this->FontAwesome->getClass('hourglass-start') ?>" style="font-size: larger;" title="<?= __('Blocking execution path') ?>"></i>
<?php else : ?>
<i class="fa-fw fa-rotate-90 <?= $this->FontAwesome->getClass(empty($workflow['enabled']) ? 'arrow-right' : 'level-up-alt') ?>" style="margin-left: <?= $i ?>em"></i>
<?php else: ?>
<i class="fa-fw fa-rotate-90 <?= $this->FontAwesome->getClass(empty($workflow['Workflow']['enabled']) ? 'arrow-right' : 'level-up-alt') ?>" style="margin-left: <?= $i ?>em"></i>
<?php endif; ?>
<a
href="<?= $baseurl . '/workflows/view/' . h($workflow['id']) ?>"
title="<?= empty($workflow['enabled']) ? __('This workflow is disabled') : h($workflow['description']) ?>"
class="bold <?= empty($workflow['enabled']) ? 'muted' : '' ?>"
style="<?= empty($workflow['enabled']) ? 'text-decoration: line-through;' : '' ?>"
href="<?= $baseurl . '/workflows/view/' . h($workflow['Workflow']['id']) ?>"
title="<?= empty($workflow['Workflow']['enabled']) ? __('This workflow is disabled') : h($workflow['Workflow']['description']) ?>"
class="bold <?= empty($workflow['Workflow']['enabled']) ? 'muted' : '' ?>"
style="<?= empty($workflow['Workflow']['enabled']) ? 'text-decoration: line-through;' : '' ?>"
>
<?= h($workflow['name']) ?>
<?= h($workflow['Workflow']['name']) ?>
</a>
<span style="font-size: smaller;">
:: <?= $this->element('genericElements/SingleViews/Fields/orgField', ['data' => $workflow, 'field' => ['path' => '']]) ?>
:: <?= $this->element('genericElements/SingleViews/Fields/orgField', ['data' => $workflow, 'field' => ['path' => 'Organisation']]) ?>
</span>
</li>
<?php endforeach; ?>
</ul>
<ul class="unstyled">
<?php foreach ($trigger['execution_order'] as $i => $workflow) : ?>
<?php foreach ($trigger['Workflows']['non-blocking'] as $i => $workflow) : ?>
<li>
<i class="fa-fw <?= $this->FontAwesome->getClass('arrow-right') ?>" title="<?= __('Parallel execution path') ?>"></i>
<a
href="<?= $baseurl . '/workflows/view/' . h($workflow['id']) ?>"
title="<?= empty($workflow['enabled']) ? __('This workflow is disabled') : h($workflow['description']) ?>"
class="<?= empty($workflow['enabled']) ? 'muted' : '' ?>"
style="<?= empty($workflow['enabled']) ? 'text-decoration: line-through;' : '' ?>"
href="<?= $baseurl . '/workflows/view/' . h($workflow['Workflow']['id']) ?>"
title="<?= empty($workflow['Workflow']['enabled']) ? __('This workflow is disabled') : h($workflow['Workflow']['description']) ?>"
class="<?= empty($workflow['Workflow']['enabled']) ? 'muted' : '' ?>"
style="<?= empty($workflow['Workflow']['enabled']) ? 'text-decoration: line-through;' : '' ?>"
>
<?= h($workflow['name']) ?>
<?= h($workflow['Workflow']['name']) ?>
</a>
</li>
<?php endforeach; ?>

2
app/View/Workflows/trigger_index.ctp
View File

@ -90,7 +90,7 @@
],
[
'url' => $baseurl . '/workflows/trigger_view',
'url_params_data_paths' => ['Workflow.id'],
'url_params_data_paths' => ['id'],
'icon' => 'eye'
],
]

7
app/View/Workflows/view.ctp
View File

@ -50,11 +50,12 @@ echo $this->element(
],
[
'key' => __('Listening Triggers'),
'path' => 'Workflow.name',
'path' => 'Workflow.listening_triggers',
'type' => 'custom',
'function' => function ($row) {
// return $this->element('Workflows/', ['trigger' => $row]);
return 'trigger-name';
return implode('<br />', array_map(function($trigger) {
return sprintf('<a href="/workflows/trigger_view/%s">%s</a>', h($trigger), h($trigger));
}, $row['Workflow']['listening_triggers']));
}
],
[