diff --git a/a.12-misp-workflows/content.tex b/a.12-misp-workflows/content.tex index 6d257a7..5d26760 100755 --- a/a.12-misp-workflows/content.tex +++ b/a.12-misp-workflows/content.tex @@ -10,24 +10,25 @@ \begin{itemize} \item What problems are we trying to tackle \item MISP Workflows overview - \item Design of the system \& how can it be extended + \item Design of the system \& how it can be extended \end{itemize} \end{frame} \begin{frame} \frametitle{What problems are we trying to tackle} \begin{itemize} - %\item Initial idea came from GeekWeek7.5\footnote{Workshop organized by the Canadian Cyber Center}{https://cyber.gc.ca/en/events/geekweek-75} \item Initial idea came from GeekWeek7.5\footnote{\href{https://cyber.gc.ca/en/events/geekweek-75}{Workshop organized by the Canadian Cyber Center}} \begin{center} \includegraphics[width=0.3\linewidth]{pictures/geekweek75.jpg} \end{center} - \item Experienced users wanted to be able to interact with the behavior of MISP for specific operations - \item Same spirit than web-hooks but more flexible + \item Needs: + \begin{itemize} + \item Interacting with MISP's behavior + \item Hooking specific action via callbacks + \end{itemize} \item Use-cases: \begin{itemize} \item Prevent publication of events not meeting some criterias - \item Enrich events before the actual publication takes place \item Prevent querying thrid-party service (e.g. virustotal) for sensitive information \item Send a notification in chat room when new events get published \item And much much more.. @@ -36,13 +37,12 @@ \end{frame} \begin{frame} - \frametitle{Simplistic overview} + \frametitle{Simplistic overview of a Workflow in action} \begin{enumerate} - \item \textbf{User Interacts} with MISP using the UI or API - \item MISP handles the request, starts \textbf{preparing data} to perform the operation - \item MISP checks if there is an enabled workflow \textbf{listening to the trigger} - \item MISP fetches enabled workflows and \textbf{executes} them - \item If all went fine, MISP \textbf{continue} to perform the operation + \item An \textbf{action} is performed in MISP + \item If there is an \textbf{enabled} Workflow for that \textbf{action} + \item MISP runs the Workflow + \item If all went fine, MISP \textbf{continue} to perform the action \begin{itemize} \item The operation can potentially be cancelled by \texttt{blocking} modules \end{itemize} @@ -53,118 +53,106 @@ \frametitle{Terminology} \begin{enumerate} \item \textbf{workflow}: Sequence of actions to be executed - \item \textbf{execution path}: A path composed of actions to be executed sequentially - \begin{itemize} - \item A workflow can contain more than one execution path - \end{itemize} - \item \textbf{trigger}: Starting point of an \texttt{execution path}. Triggers are called when specific actions are done by MISP + \item \textbf{execution path}: A path composed of nodes to be executed sequentially + \item \textbf{trigger}: Starting point of a workflow. Triggers are called when specific actions happen in MISP \begin{itemize} \item A trigger can only have one workflow and vice-versa \end{itemize} \end{enumerate} \begin{center} - \includegraphics[width=1.0\linewidth]{pictures/workflow-view.png} + \includegraphics[width=1.0\linewidth]{pictures/simple-workflow.png} \end{center} \end{frame} \begin{frame} - \frametitle{Workflow execution} + \frametitle{Workflow execution process} + Typical execution process: \begin{enumerate} - \item An operation happen in MISP (e.g. event publication) - \item A trigger is called - \item Collect enabled workflow listening to called trigger - \item Execute workflow + \item An action happen in MISP + \item The workflow associated to the trigger is run + \item Execution result: \begin{itemize} \item \texttt{\color{green!50!black}success}: Proceed with the operation - \item \texttt{\color{red}failure} | \texttt{\color{blue}cancel}: Cancel the operation + \item \texttt{\color{red}failure} | \texttt{\color{blue}blocked}: Cancel the operation + \end{itemize} + \end{enumerate} + \vspace{0.5em} + Example for Event publish: + \begin{enumerate} + \item An Event is about to be published + \item MISP executes the workflow listening to the \texttt{event-publish} trigger + \begin{itemize} + \item {\bf\color{green!50!black}success}: Proceed the publishing action + \item {\bf\color{red}failure} | \texttt{\color{blue}blocked}: Stop publishing and log the reason \end{itemize} \end{enumerate} \end{frame} \begin{frame} - \frametitle{Execution Paths} - Currently 2 types of execution path: + \frametitle{Blocking and non-blocking Workflows} + Currently 2 types of workflows: \vspace{0.5em} \begin{itemize} - \item {\bf Blocking}: Execution is stoped in case of error or module cancel + \item {\bf Blocking}: Completion of the initial action can be prevented \begin{itemize} - \item Current workflow's blocking execution path is {\bf stopped} + \item If a \textbf{blocking module} blocks the action + \item If a \textbf{blocking module} raises an exception \end{itemize} \vspace{0.5em} - \item {\bf Non-blocking/Parallel}: Stop execution for current path only + \item {\bf Non-blocking}: Workflow execution outcome has no impact \begin{itemize} - \item Current execution path is {\bf stopped} - \item {\bf Resume} execution of remaining paths + \item \textbf{Blocking modules} become \textbf{non-blocking} + \item Execution proceed unless on exception \end{itemize} \end{itemize} \end{frame} -\begin{frame} - \frametitle{Publishing example} - Example: - \begin{enumerate} - \item An Event is published - \item MISP starts the publishing process - \item MISP executes the workflow listening to the trigger - \begin{itemize} - \item {\bf\color{green!50!black}success execution success}: Proceed publishing - \item {\bf\color{red}success execution failure}: Stop publishing, log the reason and report the failure back to the user - \end{itemize} - \end{enumerate} -\end{frame} - \begin{frame} \frametitle{Execution context} \begin{itemize} - \item Workflow are \textit{triggered by any users} + \item Workflows can be triggered by \textbf{any users} + \item Workflows can be triggered by operations done via the \textbf{UI or the API} \item However, the user for which the workflow executes has the \texttt{site-admin} role and is from the \texttt{MISP.host\_org\_id} - \item This is to make sure, all data are processed regardless of the ACL + \item This is to make sure, all data are processed regardless of ownership and access (no ACL) \end{itemize} \end{frame} \begin{frame} - \frametitle{Workflow modules} + \frametitle{Types of Workflow modules} \begin{center} \includegraphics[width=0.5\linewidth]{pictures/module-type.png} \end{center} - 4 types of module + 2 types of action modules \begin{itemize} \item \textbf{logic}: Allow to redirect the execution flow. \begin{itemize} \item IF condition, fork the blocking execution into a non-blocking one, ... \end{itemize} - \item \textbf{action}: Can modify data, prevent execution or perform additional actions + \item \textbf{action}: Allow to perform additional actions, prevent execution or modify data \begin{itemize} - \item Publish to ZMQ, perform enrichments, block the execution, ... - \end{itemize} - \item \textbf{misp-module}: Basically \texttt{action} modules but using the \texttt{misp-module} service for the logic - \begin{itemize} - \item Written in Python! - \end{itemize} - \item \textbf{custom}: Allow user to create their own \texttt{action} and \texttt{logic} module in PHP - \begin{itemize} - \item Can use any functions defined in the application + \item Webhook, block the execution, perform enrichments, ... \end{itemize} \end{itemize} \end{frame} \begin{frame} \frametitle{Workflow modules} - \texttt{action} modules can be from 3 sources + 3 sources of action modules \begin{itemize} \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php} \begin{itemize} - \item Built-in module in the application + \item Built-in \textbf{default} modules in the application \item Written in PHP \item Can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...) \item Fast and easier to interact with for those having internal knowledge of MISP \end{itemize} \item \texttt{\scriptsize app/Lib/WorkflowModules/action/[module\_name].php} \begin{itemize} - \item Same as previous but allow users to create their own without sharing with the community + \item \textbf{Custom} modules written by users \end{itemize} \item \texttt{From the misp-module service} \begin{itemize} + \item \textbf{Default} and \textbf{custom} modules \item Written in Python \item Can use any python libraries \item New \texttt{misp-module} module type: \texttt{action} @@ -181,7 +169,7 @@ \begin{enumerate} \item Make sure \texttt{MISP.background\_jobs} is turned on \item Turn on setting \texttt{Plugin.Workflow\_enable} - \item Make sure workers are up-and-running + \item Make sure workers are up-and-running and healthy \end{enumerate} \begin{center} \includegraphics[width=0.75\linewidth]{pictures/settings-1.png} @@ -191,7 +179,7 @@ \begin{frame} \frametitle{Getting started with workflows (2)} - If you wish to use action modules from the \texttt{misp-module}: + If you wish to use action modules from \texttt{misp-module}: \begin{itemize} \item Make sure you update your \texttt{misp-module} application to the latest version \item Make sure your have the new \texttt{action\_mod} module type in \url{misp-modules/misp\_modules/modules} @@ -240,6 +228,22 @@ \end{center} \end{frame} +\begin{frame} + \frametitle{Workflow blueprints} + \begin{enumerate} + \item Blueprints allow to re-use parts of a workflow in another one + \item Blueprints can be saved, exported and shared + \end{enumerate} + \begin{center} + \includegraphics[width=0.5\linewidth]{pictures/blueprint-debugging.png} + \end{center} + Blueprints origins: + \begin{enumerate} + \item From the "official" \texttt{misp-workflow-blueprints} repository + \item Created or imported by users + \end{enumerate} +\end{frame} + \begin{frame} \frametitle{Workflow blueprints: Create} Select one or more modules to be saved as blueprint then click on the \texttt{save blueprint} button @@ -259,15 +263,54 @@ \end{center} \end{frame} +\begin{frame}[fragile] + \frametitle{Hash path filtering} + \begin{itemize} + \item Some modules have the possibility to filter or check conditions using CakePHP's path expression. + \end{itemize} +\begin{lstlisting}[language=javascript,firstnumber=1] +$path_expression = '{n}[name=fred].id'; +$users = [ + {'id': 123, 'name': 'fred', 'surname': 'bloggs'}, + {'id': 245, 'name': 'fred', 'surname': 'smith'}, + {'id': 356, 'name': 'joe', 'surname': 'smith'}, +]; +$ids = Hash::extract($users, $path_expression); +// => $ids will be [123, 245] +\end{lstlisting} +\end{frame} + \begin{frame} - \frametitle{TODOs / FIXMEs} - \begin{enumerate} - \item Show which workflows use a module and the other way around - \item Perfom parallel execution by a worker (currently in-line) - \item Implement parallel task module - \item ACL-aware: new \texttt{workflow editor} role - \item Standardize how data is passed between modules - \end{enumerate} + \frametitle{Data format in Workflows} + \begin{itemize} + \item All triggers will inject data in a workflow + \item In some cases, there is no format (e.g. User after-save) + \item In others, the format is \textbf{compliant with the MISP Core format} + \item In addition to the RFC, the passed data has \textbf{additional properties} + \begin{itemize} + \item Attributes are always encapsulated in the Event or Object + \item Additional key \texttt{\_AttributeFlattened} + \item Additional key \texttt{\_allTags} + \item Additional key \texttt{inherited} for Tags + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Logic module: Concurrent Task} + \begin{itemize} + \item Special type of \textbf{logic} module + \item Allows breaking the execution flow into a concurrent tasks to be executed later on by a background worker + \item As a side effect, blocking modules cannot cancel an ongoing operation anymore + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Debugging Workflows} + \begin{itemize} + \item Workflow execution is logged in the application logs: \texttt{/admin/logs/index} + \item Or stored on disk in the following file: \texttt{/app/tmp/logs/workflow-execution.log} + \end{itemize} \end{frame} \section{Learning by examples} @@ -293,17 +336,6 @@ \end{itemize} \end{frame} -\begin{frame} - \frametitle{Workflow example 3} - \begin{center} - \includegraphics[width=0.65\linewidth]{pictures/example-3.png} - \end{center} - - \begin{itemize} - \item After a user has been saved, a message containing the user's email will be sent to a Mattermost channel and the user detailed will be posted to the webhook URL - \end{itemize} -\end{frame} - \begin{frame} \frametitle{Creating a new module in PHP} \begin{center} diff --git a/a.12-misp-workflows/pictures/blueprint-debugging.png b/a.12-misp-workflows/pictures/blueprint-debugging.png new file mode 100644 index 0000000..8d983f9 Binary files /dev/null and b/a.12-misp-workflows/pictures/blueprint-debugging.png differ diff --git a/a.12-misp-workflows/pictures/module-type.png b/a.12-misp-workflows/pictures/module-type.png index 54eb8f2..3a306b3 100644 Binary files a/a.12-misp-workflows/pictures/module-type.png and b/a.12-misp-workflows/pictures/module-type.png differ diff --git a/a.12-misp-workflows/pictures/simple-workflow.png b/a.12-misp-workflows/pictures/simple-workflow.png new file mode 100644 index 0000000..f494348 Binary files /dev/null and b/a.12-misp-workflows/pictures/simple-workflow.png differ