MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [workflow] Initial work on filtering modules - WiP

new_widgets
Sami Mokaddem 2022-11-11 09:45:38 +01:00
parent 6b1b080eec
commit fd93ac9c5e
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
8 changed files with 289 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Controller/WorkflowsController.php
View File

@ -101,6 +101,9 @@ class WorkflowsController extends AppController
}
}
$this->CRUD->view($id, [
'afterFind' => function($workflow) {
return $this->Workflow->attachLabelToConnections($workflow);
}
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
@ -151,6 +154,7 @@ class WorkflowsController extends AppController
} else {
$workflow = $this->Workflow->fetchWorkflow($workflow_id);
}
$workflow = $this->Workflow->attachLabelToConnections($workflow, $trigger_id);
$modules = $this->Workflow->attachNotificationToModules($modules, $workflow);
$this->loadModel('WorkflowBlueprint');
$workflowBlueprints = $this->WorkflowBlueprint->find('all');

62
app/Lib/Tools/WorkflowGraphTool.php
View File

@ -162,6 +162,12 @@ class GraphWalker
} else if ($node['data']['id'] == 'concurrent-task') {
$this->_evaluateConcurrentTask($node, $roamingData, $outputs['output_1']);
return ['output_1' => []];
} else if ($node['data']['id'] == 'generic-filter-data') {
$this->_evaluateFilterAddLogic($node, $roamingData, $outputs['output_1']);
return ['output_1' => []];
} else if ($node['data']['id'] == 'generic-filter-reset') {
$this->_evaluateFilterRemoveLogic($node, $roamingData, $outputs['output_1']);
return ['output_1' => []];
} else {
$useFirstOutput = $this->_evaluateCustomLogicCondition($node, $roamingData);
return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']];
@ -175,6 +181,18 @@ class GraphWalker
return $result;
}
private function _evaluateFilterAddLogic($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateFilterRemoveLogic($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateCustomLogicCondition($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
@ -357,6 +375,50 @@ class WorkflowGraphTool
return $nodes;
}
/**
* extractFilterNodesFromWorkflow Return the list of generic-filter-data's id (or full module) that are included in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractFilterNodesFromWorkflow(array $graphData, bool $fullNode = false): array
{
$nodes = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-data') {
if (!empty($fullNode)) {
$nodes[] = $node;
} else {
$nodes[] = $node['data']['id'];
}
}
}
return $nodes;
}
/**
* extractResetFilterFromWorkflow Return the list of generic-filter-reset's id (or full module) that are included in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractResetFilterFromWorkflow(array $graphData, bool $fullNode = false): array
{
$nodes = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-reset') {
if (!empty($fullNode)) {
$nodes[] = $node;
} else {
$nodes[] = $node['data']['id'];
}
}
}
return $nodes;
}
/**
* isAcyclic Return if the graph contains a cycle
*

61
app/Model/Workflow.php
View File

@ -1290,6 +1290,67 @@ class Workflow extends AppModel
return $data;
}
public function getLabelsForConnections($workflow, $trigger_id): array
{
$graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data'];
$startNodeID = $this->workflowGraphTool->getNodeIdForTrigger($graphData, $trigger_id);
if ($startNodeID == -1) {
return [];
}
$connections = [];
$filterNodes = $this->workflowGraphTool->extractFilterNodesFromWorkflow($graphData, true);
$filterNodeIDToLabel = Hash::combine($filterNodes, '{n}.id', '{n}.data.indexed_params.filtering-label');
$resetFilterNodes = $this->workflowGraphTool->extractResetFilterFromWorkflow($graphData, true);
$resetFilterNodeIDToLabel = Hash::combine($resetFilterNodes, '{n}.id', '{n}.data.indexed_params.filtering-label');
$roamingData = $this->workflowGraphTool->getRoamingData();
$graphWalker = $this->workflowGraphTool->getWalkerIterator($graphData, $this, $startNodeID, GraphWalker::PATH_TYPE_INCLUDE_LOGIC, $roamingData);
foreach ($graphWalker as $graphNode) {
$node = $graphNode['node'];
$nodeID = $node['id'];
$parsedPathList = GraphWalker::parsePathList($graphNode['path_list']);
foreach ($parsedPathList as $pathEntry) {
if (!empty($filterNodeIDToLabel[$pathEntry['source_id']])) {
$connections[$nodeID][] = $filterNodeIDToLabel[$pathEntry['source_id']];
}
if (!empty($resetFilterNodeIDToLabel[$pathEntry['source_id']])) {
if ($resetFilterNodeIDToLabel[$pathEntry['source_id']] == 'all') {
$connections[$nodeID] = [];
} else {
$connections[$nodeID] = array_values(array_diff($connections[$nodeID], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
}
}
}
}
return $connections;
}
public function attachLabelToConnections($workflow, $trigger_id=null): array
{
$graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data'];
if (is_null($trigger_id)) {
$startNode = $this->workflowGraphTool->extractTriggerFromWorkflow($graphData, true);
$trigger_id = $startNode['data']['id'];
}
$labelsByNodes = $this->getLabelsForConnections($workflow, $trigger_id);
foreach ($graphData as $i => $node) {
if (!empty($labelsByNodes[$node['id']])) {
foreach ($node['inputs'] as $inputName => $inputs) {
foreach ($inputs['connections'] as $j => $connection) {
$workflow['Workflow']['data'][$i]['inputs'][$inputName]['connections'][$j]['labels'] = array_map(function($label) {
return [
'id' => Inflector::variable($label),
'name' => $label,
'variant' => 'info',
];
}, $labelsByNodes[$node['id']]);
}
}
}
}
return $workflow;
}
/**
* moduleSattelesExecution Executes a module using the provided configuration and returns back the result
*

17
app/Model/WorkflowModules/WorkflowBaseModule.php
View File

@ -298,3 +298,20 @@ class WorkflowBaseLogicModule extends WorkflowBaseModule
class WorkflowBaseActionModule extends WorkflowBaseModule
{
}
class WorkflowFilteringLogicModule extends WorkflowBaseLogicModule
{
public $blocking = false;
public $inputs = 1;
public $outputs = 2;
protected function _genFilteringLabels(): array
{
$names = ['A', 'B', 'C', 'D', 'E', 'F'];
$labels = [];
foreach ($names as $name) {
$labels[$name] = __('Label %s', $name);
}
return $labels;
}
}

81
app/Model/WorkflowModules/logic/Module_generic_filter_data.php Normal file
View File

@ -0,0 +1,81 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_generic_filter_data extends WorkflowFilteringLogicModule
{
public $id = 'generic-filter-data';
public $name = 'Filter :: Generic';
public $description = 'Generic data filtering block. The module filters incoming data and forward the matching data to its output.';
public $icon = 'filter';
public $inputs = 1;
public $outputs = 1;
// public $html_template = 'filter-add';
public $params = [];
private $operators = [
'in' => 'In',
'not_in' => 'Not in',
'equals' => 'Equals',
'not_equals' => 'Not equals',
];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'filtering-label',
'label' => __('Filtering Label'),
'type' => 'select',
'options' => $this->_genFilteringLabels(),
],
[
'id' => 'selector',
'label' => __('Data selector'),
'type' => 'input',
'placeholder' => 'Event._AttributeFlattened.{n}',
],
[
'id' => 'value',
'label' => __('Value'),
'type' => 'input',
'placeholder' => 'tlp:red',
],
[
'id' => 'operator',
'label' => __('Operator'),
'type' => 'select',
'default' => 'in',
'options' => $this->operators,
],
[
'id' => 'hash_path',
'label' => __('Hash path'),
'type' => 'input',
'placeholder' => 'Tag.name',
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$selector = $params['selector']['value'];
$path = $params['hash_path']['value'];
$operator = $params['operator']['value'];
$value = $params['value']['value'];
$rData = $roamingData->getData();
$applyFilterFunction = function ($element) use ($value, $operator, $path) {
$selectedData = Hash::extract($element, $path);
return $this->evaluateCondition($selectedData, $operator, $value);
};
$filteredData = Hash::apply($rData, $selector, $applyFilterFunction);
debug($filteredData);
$newRData = $filteredData;
$newRData['_unfilteredData'] = $rData;
$roamingData->setData($newRData);
return true;
}
}

36
app/Model/WorkflowModules/logic/Module_generic_filter_reset.php Normal file
View File

@ -0,0 +1,36 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_generic_filter_reset extends WorkflowFilteringLogicModule
{
public $id = 'generic-filter-reset';
public $name = 'Filter :: Remove filter';
public $description = 'Reset filtering';
public $icon = 'redo-alt';
public $inputs = 1;
public $outputs = 1;
// public $html_template = 'filter-remove';
public $params = [];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'filtering-label',
'label' => __('Filtering Label to remove'),
'type' => 'select',
'options' => ['all' => __('All filters')] + $this->_genFilteringLabels(),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$rData = $roamingData->getData();
$newRData = $rData['_unfilteredData'];
$roamingData->setData($newRData);
return true;
}
}

11
app/webroot/css/workflows-editor.css
View File

@ -665,4 +665,15 @@
.drawflow .drawflow-node.block-type-concurrent > .outputs > .output_1::after {
content: "\f074";
}
.drawflow svg.connection .connection-label-container {
display: flex;
flex-direction: column;
row-gap: 3px;
width: fit-content;
transform: translate(-50%, -50%);
background-color: #ffffffaa;
padding: 3px;
border-radius: 5px;
}

27
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -129,6 +129,9 @@ var dotBlock_error = doT.template(' \
</div> \
</div>')
var dotBlock_connectionLabel = doT.template(' \
<span class="label label-{{=it.variant}}" id="{{=it.id}}">{{=it.name}}</span>')
var classBySeverity = {
'info': 'info',
'warning': 'warning',
@ -771,7 +774,11 @@ function loadWorkflow(workflow) {
Object.values(workflow.data).forEach(function (node) {
for (var input_name in node.inputs) {
node.inputs[input_name].connections.forEach(function (connection) {
editor.addConnection(connection.node, node.id, connection.input, input_name)
connection.labels = connection.labels === undefined ? [] : connection.labels;
var labels = connection.labels.map(function(labelConf) {
return dotBlock_connectionLabel(labelConf)
})
editor.addConnection(connection.node, node.id, connection.input, input_name, labels)
})
}
})
@ -868,7 +875,7 @@ function addNodesFromBlueprint(workflowBlueprint, cursorPosition) {
left: (node.pos_x - minX) * editor.zoom + cursorPosition.left,
}
if (all_modules_by_id[node.data.id] === undefined) {
var errorMessage = 'Invalid ' + node.data.module_data.module_type + ' module id `' + node.data.module_data.id + '` (' + node.id + ')'
var errorMessage = 'Invalid ' + node.data.module_type + ' module id `' + node.data.id + '` (' + node.id + ')'
var html = window['dotBlock_error']({
error: errorMessage,
data: JSON.stringify(node.data.indexed_params, null, 2)
@ -883,20 +890,20 @@ function addNodesFromBlueprint(workflowBlueprint, cursorPosition) {
node.data,
html
)
return
} else {
additionalData = {
indexed_params: node.data.indexed_params,
saved_filters: node.data.saved_filters,
}
addNode(all_modules_by_id[node.data.id], position, additionalData)
}
additionalData = {
indexed_params: node.data.indexed_params,
saved_filters: node.data.saved_filters,
}
addNode(all_modules_by_id[node.data.id], position, additionalData)
oldNewIDMapping[node.id] = editor.nodeId - 1
newNodes.push(getNodeHtmlByID(editor.nodeId - 1)) // nodeId is incremented as soon as a new node is created
})
workflowBlueprint.data.forEach(function (node) {
Object.keys(node.outputs).forEach(function (outputName) {
var newNode = Object.assign({}, all_modules_by_id[node.data.id])
if (newNode.outputs > 0) { // make sure the module configuration didn't change in regards of the outputs
var outputCount = all_modules_by_id[node.data.id] !== undefined ? all_modules_by_id[node.data.id].outputs : Object.keys(node.outputs).length
if (outputCount > 0) { // make sure the module configuration didn't change in regards of the outputs
node.outputs[outputName].connections.forEach(function (connection) {
if (oldNewIDMapping[connection.node] !== undefined) {
editor.addConnection(