misp-training/a.12-misp-workflows/content.tex

486 lines
18 KiB
TeX
Raw Normal View History

2022-06-01 17:59:24 +02:00
% DO NOT COMPILE THIS FILE DIRECTLY!
% This is included by the other .tex files.
\begin{frame}[t,plain]
\titlepage
\end{frame}
\begin{frame}
\frametitle{Content of the presentation}
\begin{itemize}
\item MISP Workflows fundamentals
\item Getting started
\item Design of the system \& how it can be extended
2022-06-01 17:59:24 +02:00
\end{itemize}
\begin{center}
\frame{\includegraphics[width=0.9\linewidth]{pictures/overview.png}}
\end{center}
2022-06-01 17:59:24 +02:00
\end{frame}
\begin{frame}
\frametitle{What problems are we trying to tackle}
\begin{itemize}
\item Initial idea came during GeekWeek7.5\footnote{\href{https://cyber.gc.ca/en/events/geekweek-75}{Workshop organized by the Canadian Cyber Center}} \includegraphics[width=0.3\linewidth]{pictures/geekweek75.jpg}
\item Needs:
\begin{itemize}
\item Prevent default MISP behaviors
\item Hook specific actions to run callbacks
\end{itemize}
2022-06-01 17:59:24 +02:00
\item Use-cases:
\begin{itemize}
2022-06-02 08:24:19 +02:00
\item Prevent publication of events not meeting some criterias
\item Prevent querying thrid-party services (e.g. virustotal) with sensitive information
\item Send notifications in a chat rooms
\item And much much more..
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{itemize}
\end{frame}
\section{Workflow - Fundamentals}
2022-06-01 17:59:24 +02:00
\begin{frame}
\frametitle{Simplistic overview of a Workflow in action}
2022-06-01 17:59:24 +02:00
\begin{enumerate}
\item An \textbf{action} happens in MISP
\item If there is an \textbf{enabled} Workflow for that \textbf{action}, run it
\item If all went fine, MISP \textbf{continue} to perform the action
\begin{itemize}
\item The operation can potentially be cancelled by \texttt{blocking} modules
\end{itemize}
2022-06-01 17:59:24 +02:00
\end{enumerate}
\end{frame}
\begin{frame}
\frametitle{Terminology}
\begin{itemize}
\item \textbf{workflow}: Sequence of all operations (nodes) to be executed. Basically the whole graph.
\item \textbf{execution path}: A path composed of nodes
\item \textbf{trigger}: Starting point of a workflow. Triggers are called when specific actions happen in MISP
2022-06-02 08:24:19 +02:00
\begin{itemize}
\item A trigger can only have one workflow and vice-versa
2022-06-02 08:24:19 +02:00
\end{itemize}
\end{itemize}
2022-06-01 17:59:24 +02:00
\begin{center}
\frame{\includegraphics[width=1.0\linewidth]{pictures/simple-workflow.png}}
2022-06-01 17:59:24 +02:00
\end{center}
\end{frame}
\begin{frame}
\frametitle{Workflow execution process}
Typical execution process:
2022-06-01 17:59:24 +02:00
\begin{enumerate}
\item An action happens in MISP
\item The workflow associated to the trigger is ran
\item Execution result?
\begin{itemize}
\item \texttt{\color{green!50!black}success}: Continue the action
\item \texttt{\color{red}failure} | \texttt{\color{blue}blocked}: Cancel the action
\end{itemize}
\end{enumerate}
\vspace{0.5em}
Example for Event publish:
\begin{enumerate}
\item An Event is about to be published
\item MISP executes the workflow listening to the \texttt{event-publish} trigger
\begin{itemize}
\item {\bf\color{green!50!black}success}: Continue the publishing action
\item {\bf\color{red}failure} | \texttt{\color{blue}blocked}: Stop publishing and log the reason
\end{itemize}
2022-06-01 17:59:24 +02:00
\end{enumerate}
\end{frame}
\begin{frame}
\frametitle{Blocking and non-blocking Workflows}
Currently 2 types of workflows:
2022-06-01 17:59:24 +02:00
\vspace{0.5em}
\begin{itemize}
\item {\bf Blocking}: Completion of the action can be prevented
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item If a \textbf{blocking module} blocks the action
\item If a \textbf{blocking module} raises an exception
2022-06-01 17:59:24 +02:00
\end{itemize}
\vspace{0.5em}
\item {\bf Non-blocking}: Workflow execution outcome has no impact
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item \textbf{Blocking modules} can still stop the execution
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Execution context}
\begin{itemize}
\item Workflows can be triggered by \textbf{any users}
\item Workflows can be triggered by actions done via the \textbf{UI} or \textbf{API}
\item However, the user for which the workflow executes has:
\begin{itemize}
\item The \texttt{site-admin} permission
\item Is from the \texttt{MISP.host\_org\_id}
\end{itemize}
\item Ensures data is processed regardless of ownership and access: \textbf{no ACL}
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Classes of Workflow modules}
2022-06-01 17:59:24 +02:00
\begin{center}
\includegraphics[width=0.6\linewidth]{pictures/module-type.png}
2022-06-01 17:59:24 +02:00
\end{center}
3 classes of modules
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item \textbf{action}: Allow to executes functions, callbacks or scripts
\begin{itemize}
\item Can stop execution
\item e.g. Webhook, block the execution, perform enrichments, ...
\end{itemize}
\item \textbf{logic}: Allow to redirect the execution flow.
\begin{itemize}
\item IF condition, fork the blocking execution into a non-blocking one, ...
\end{itemize}
\item \textbf{blueprint}: Allow to reuse composition of modules
\begin{itemize}
\item Can save subworkflows and its module's configuration
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Sources of Workflow modules}
3 sources of action modules
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item Built-in \textbf{default} modules
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item Part of the MISP codebase
\item \texttt{\scriptsize \textbf{app/Model/}WorkflowModules/action/[module\_name].php}
2022-06-01 17:59:24 +02:00
\end{itemize}
\item User-defined \textbf{custom} modules
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item Written in PHP
\item Can extend existing default modules
\item Can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...)
\item Faster and easier to implement new complex behaviors
\item \texttt{\scriptsize \textbf{app/Lib/}WorkflowModules/action/[module\_name].php}
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Sources of Workflow modules}
3 sources of action modules
\begin{itemize}
\item Modules from the \textbf{enrichment service}
2022-06-02 08:24:19 +02:00
\begin{itemize}
\item \textbf{Default} and \textbf{custom} modules
\item From the \textit{misp-module} \includegraphics[width=0.25\linewidth]{pictures/misp-module-icon.png}
\item Written in Python
\item Can use any python libraries
\item New \textit{misp-module} module type: \texttt{action}
2022-06-02 08:24:19 +02:00
\end{itemize}
2022-06-01 17:59:24 +02:00
\end{itemize}
\vspace{1em}
2022-06-01 17:59:24 +02:00
\begin{center}
$\rightarrow$ Both the PHP and Python systems are \textbf{plug-and-play}
2022-06-01 17:59:24 +02:00
\end{center}
\end{frame}
\begin{frame}
\frametitle{Triggers currently available}
Currently 8 triggers can be hooked. 3 being \textbf{blocking}.
\includegraphics[width=1.0\linewidth]{pictures/triggers.png}
\end{frame}
\section{Workflow - Getting started}
\begin{frame}
\frametitle{Getting started with workflows (1)}
Review MISP settings:
\begin{enumerate}
\item Make sure \texttt{MISP.background\_jobs} is turned on
\item Make sure workers are up-and-running and healthy
\item Turn the setting \texttt{Plugin.Workflow\_enable} on
\begin{center}
\includegraphics[width=0.70\linewidth]{pictures/settings-1.png}
\end{center}
\item {[optional:misp-module]} Turn the setting \texttt{Plugin.Action\_services\_enable} on
\begin{center}
\includegraphics[width=0.70\linewidth]{pictures/settings-2.png}
\end{center}
\end{enumerate}
2022-06-01 17:59:24 +02:00
\end{frame}
\begin{frame}[fragile]
\frametitle{Getting started with workflows (2)}
If you wish to use action modules from \texttt{misp-module}, make sure to have:
\begin{itemize}
\item The latest update of \texttt{misp-module}
\begin{itemize}
\item There should be an \texttt{action\_mod} module type in \url{misp-modules/misp\_modules/modules}
\end{itemize}
\item Restarted your \texttt{misp-module} application
\end{itemize}
\vspace{1em}
\begin{lstlisting}[language=text,firstnumber=1]
# This command should show all `action` modules
$ curl -s http://127.0.0.1:6666/modules | \
jq '.[] | select(.meta."module-type"[] | contains("action")) |
{name: .name, version: .meta.version}'
\end{lstlisting}
\end{frame}
\begin{frame}
\frametitle{Getting started with workflows (3)}
2022-06-01 17:59:24 +02:00
\begin{enumerate}
\item Go to the list of modules
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item \texttt{Administration > Workflows > List Modules}
\item or \url{/workflows/moduleIndex}
2022-06-01 17:59:24 +02:00
\end{itemize}
\item Make sure \textbf{default} modules are loaded
\item {[optional:misp-module]} Make sure \textbf{misp-module} modules are loaded
2022-06-01 17:59:24 +02:00
\end{enumerate}
\end{frame}
\begin{frame}
\frametitle{Creating a workflow with the editor}
2022-06-01 17:59:24 +02:00
\begin{enumerate}
\item Go to the list of triggers \texttt{Administration > Workflows}
\item Enable and edit a \texttt{trigger} from the list
\item Drag an \texttt{action} module from the side panel to the canvas
\item From the \texttt{trigger} output, drag an arrow into the \texttt{action}'s input (left side)
\item Execute the action that would run the trigger and observe the effect!
2022-06-01 17:59:24 +02:00
\end{enumerate}
\begin{center}
\frame{\includegraphics[width=0.7\linewidth]{pictures/triggers.png}}
2022-06-01 17:59:24 +02:00
\end{center}
\begin{center}
\frame{\includegraphics[width=0.50\linewidth]{pictures/editor-1.png}}
2022-06-01 17:59:24 +02:00
\end{center}
\end{frame}
\begin{frame}
\frametitle{Working with the editor}
Operations not allowed:
\begin{itemize}
\item Execution loop are not authorized
2022-06-01 17:59:24 +02:00
\begin{itemize}
\item Current caveat: If an action re-run the workflow in any way
2022-06-01 17:59:24 +02:00
\end{itemize}
\end{itemize}
2022-06-01 17:59:24 +02:00
\begin{center}
\frame{\includegraphics[width=0.7\linewidth]{pictures/editor-not-allowed-1.png}}
2022-06-01 17:59:24 +02:00
\end{center}
\end{frame}
\begin{frame}
\frametitle{Working with the editor}
Operations not allowed:
\begin{itemize}
\item Multiple connections from the same output
\begin{itemize}
\item Execution order not guaranted and confusing for users
\end{itemize}
\end{itemize}
\begin{center}
\frame{\includegraphics[width=0.7\linewidth]{pictures/editor-not-allowed-2.png}}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Working with the editor}
Operations showing a warning:
\begin{itemize}
\item \textbf{Blocking} modules after a \textbf{concurrent tasks} module
\item \textbf{Blocking} modules in a \textbf{non-blocking} workflow
\end{itemize}
\begin{center}
\frame{\includegraphics[width=1.0\linewidth]{pictures/editor-warning-1.png}}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Workflow blueprints}
\begin{enumerate}
\item Blueprints allow to \textbf{re-use parts} of a workflow in another one
\item Blueprints can be saved, exported and \textbf{shared}
\end{enumerate}
\begin{center}
\includegraphics[width=0.5\linewidth]{pictures/blueprint-debugging.png}
\end{center}
Blueprints origins:
\begin{enumerate}
\item From the "official" \texttt{misp-workflow-blueprints} repository
\item Created or imported by users
\end{enumerate}
\end{frame}
2022-06-01 17:59:24 +02:00
\begin{frame}
\frametitle{Workflow blueprints: Create}
Select one or more modules to be saved as blueprint then click on the \texttt{save blueprint} button
2022-06-01 17:59:24 +02:00
\begin{center}
\includegraphics[width=0.85\linewidth]{pictures/blueprint-1.png}
2022-06-01 17:59:24 +02:00
\end{center}
\end{frame}
\begin{frame}[fragile]
\frametitle{Hash path filtering}
\begin{itemize}
\item Some modules have the possibility to filter or check conditions using CakePHP's path expression.
\end{itemize}
\begin{lstlisting}[language=javascript,firstnumber=1]
$path_expression = '{n}[name=fred].id';
$users = [
{'id': 123, 'name': 'fred', 'surname': 'bloggs'},
{'id': 245, 'name': 'fred', 'surname': 'smith'},
{'id': 356, 'name': 'joe', 'surname': 'smith'},
];
$ids = Hash::extract($users, $path_expression);
// => $ids will be [123, 245]
\end{lstlisting}
\begin{center}
\includegraphics[width=0.4\linewidth]{pictures/module-if-generic.png}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Module filtering}
\begin{itemize}
\item Some action modules accept \textbf{filtering} conditions
\item E.g. the \texttt{enrich-event} module will only perform the enrichment on Attributes having a \texttt{tlp:white} Tag
\end{itemize}
\begin{center}
\includegraphics[width=0.7\linewidth]{pictures/module-filtering.png}
\end{center}
\end{frame}
2022-06-01 17:59:24 +02:00
\begin{frame}
\frametitle{Data format in Workflows}
\begin{center}
\includegraphics[width=0.7\linewidth]{pictures/workflow-trigger.png}
\end{center}
\begin{itemize}
\item All triggers will inject data in a workflow
\item In some cases, there is no format (e.g. User after-save)
\item In others, the format is \textbf{compliant with the MISP Core format}
\item In addition to the RFC, the passed data has \textbf{additional properties}
\begin{itemize}
\item Attributes are \textbf{always encapsulated} in the Event or Object
\item Additional key \textbf{\texttt{\_AttributeFlattened}}
\item Additional key \textbf{\texttt{\_allTags}}
\item Additional key \textbf{\texttt{inherited}} for Tags
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Logic module: Concurrent Task}
\begin{itemize}
\item Special type of \textbf{logic} module allowing multiple connections
\item Allows \textbf{breaking the execution} flow into a concurrent tasks to be executed later on by a background worker
\item As a side effect, blocking modules \textbf{cannot cancel} ongoing operations
\end{itemize}
\begin{center}
\frame{\includegraphics[width=0.45\linewidth]{pictures/module-concurrent.png}}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Debugging Workflows: Log Entries}
\begin{itemize}
\item Workflow execution is logged in the application logs:
\begin{itemize}
\item \texttt{/admin/logs/index}
\end{itemize}
\item Or stored on disk in the following file:
\begin{itemize}
\item \texttt{/app/tmp/logs/workflow-execution.log}
\end{itemize}
\item Use the \texttt{webhook-listener.py} tool
\begin{itemize}
\item \texttt{/app/tools/misp-workflows/webhook-listener.py}
\end{itemize}
\end{itemize}
\begin{center}
\includegraphics[width=1.0\linewidth]{pictures/workflow-debug.png}
\end{center}
2022-06-01 17:59:24 +02:00
\end{frame}
\begin{frame}
\frametitle{Debugging Workflows: Debug mode}
\begin{itemize}
\item The \includegraphics[width=70px]{pictures/debug-mode.png} can be turned on for each workflows
\item Each nodes will send data to the provided URL
\begin{itemize}
\item Configure the setting: \texttt{Plugin.Workflow\_debug\_url}
\end{itemize}
\item Result can be visualized in
\begin{itemize}
\item \textbf{offline}: \texttt{tools/misp-workflows/webhook-listener.py}
\item \textbf{online}: \url{requestbin.com} or similar websites
\end{itemize}
\end{itemize}
\begin{center}
\includegraphics[width=0.6\linewidth]{pictures/request-bin.png}
\end{center}
\end{frame}
\section{Learning by examples}
2022-06-01 17:59:24 +02:00
\begin{frame}
\frametitle{Workflow example 1}
2022-06-01 17:59:24 +02:00
\begin{center}
\frame{\includegraphics[width=1.0\linewidth]{pictures/example-1a.png}}
2022-06-01 17:59:24 +02:00
\end{center}
\begin{enumerate}
\item The \texttt{Event-Publish} trigger uses the MISP core format
\item The \texttt{IF::Tag} module checks if at least one of the Attribute has the \texttt{tlp:white} tag
\item If it does, the \texttt{Push-to-ZMQ} module will be executed
\end{enumerate}
2022-06-01 17:59:24 +02:00
\end{frame}
\begin{frame}
\frametitle{Workflow example 2}
2022-06-01 17:59:24 +02:00
\begin{center}
\includegraphics[width=1.0\linewidth]{pictures/example-2a.png}
2022-06-01 17:59:24 +02:00
\end{center}
\begin{itemize}
\item If an event has the \texttt{tlp:red} tag or any of the attribute has it, the publish process will be cancelled
\end{itemize}
2022-06-01 17:59:24 +02:00
\end{frame}
\section{Extending the system}
\begin{frame}
\frametitle{Creating a new module in PHP}
\begin{center}
\includegraphics[width=0.65\linewidth]{pictures/custom-1.png}
\end{center}
\begin{itemize}
\item \texttt{\small \textbf{app/Lib/}WorkflowModules/action/[module\_name].php}
\item Module configuration are defined as public variables
\item The \texttt{exec} function has to be implemented.
\begin{itemize}
\item If it returns \textbf{true}, execution will proceed
\item If it returns \textbf{false}
\begin{itemize}
\item And the module is blocking, the execution will stop and the operation will be blocked
\end{itemize}
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Creating a new module in Python}
\begin{center}
\includegraphics[width=0.65\linewidth]{pictures/custom-2.png}
\end{center}
\begin{itemize}
\item Module configuration are defined in the \texttt{moduleinfo} and \texttt{moduleconfig} variables
\item The \texttt{handler} function has to be implemented.
\item Blocking logic is the same as other modules
\end{itemize}
\end{frame}