diff --git a/a.12-misp-workflows/content.tex b/a.12-misp-workflows/content.tex index 2726c3f..f888e98 100755 --- a/a.12-misp-workflows/content.tex +++ b/a.12-misp-workflows/content.tex @@ -20,7 +20,7 @@ %\item Initial idea came from GeekWeek7.5\footnote{Workshop organized by the Canadian Cyber Center}{https://cyber.gc.ca/en/events/geekweek-75} \item Initial idea came from GeekWeek7.5\footnote{\href{https://cyber.gc.ca/en/events/geekweek-75}{Workshop organized by the Canadian Cyber Center}} \begin{center} - \includegraphics[width=0.5\linewidth]{pictures/geekweek75.jpg} + \includegraphics[width=0.3\linewidth]{pictures/geekweek75.jpg} \end{center} \item Experienced users wanted to be able to interact with the behavior of MISP for specific operations \item Same spirit than web-hooks but more flexible @@ -28,6 +28,9 @@ \begin{itemize} \item Prevent publication of events not meeting some criterias \item Enrich events before the actual publication takes place + \item Prevent querying thrid-party service (e.g. virustotal) for sensitive information + \item Send a notification in chat room when new events get published + \item And much much more.. \end{itemize} \end{itemize} \end{frame} @@ -37,9 +40,12 @@ \begin{enumerate} \item \textbf{User Interacts} with MISP using the UI or API \item MISP handles the request, starts \textbf{preparing data} to perform the operation - \item MISP checks if there are workflows \textbf{listening to the trigger} + \item MISP checks if there is an enabled workflow \textbf{listening to the trigger} \item MISP fetches enabled workflows and \textbf{executes} them \item If all went fine, MISP \textbf{continue} to perform the operation + \begin{itemize} + \item The operation can potentially be cancelled by \texttt{blocking} modules + \end{itemize} \end{enumerate} \end{frame} @@ -51,9 +57,9 @@ \begin{itemize} \item A workflow can contain more than one execution path \end{itemize} - \item \textbf{trigger}: Starting point of an \texttt{execution path}. Triggers are called when specific action are done by MISP + \item \textbf{trigger}: Starting point of an \texttt{execution path}. Triggers are called when specific actions are done by MISP \begin{itemize} - \item A workflow can contain more than one trigger, but only one per type + \item A trigger can only have one workflow and vice-versa \end{itemize} \end{enumerate} \begin{center} @@ -64,13 +70,15 @@ \begin{frame} \frametitle{Workflow execution} \begin{enumerate} + \item An operation happen in MISP (e.g. event publication) \item A trigger is called - \item Collect workflows listening to called trigger - \item Execute workflows in the saved order + \item Collect enabled workflow listening to called trigger + \item Execute workflow + \begin{itemize} + \item \texttt{\color{green!50!black}success}: Proceed with the operation + \item \texttt{\color{red}failure} | \texttt{\color{blue}cancel}: Cancel the operation + \end{itemize} \end{enumerate} - \begin{center} - \includegraphics[width=0.5\linewidth]{pictures/execution-order-1.png} - \end{center} \end{frame} \begin{frame} @@ -78,45 +86,29 @@ Currently 2 types of execution path: \vspace{0.5em} \begin{itemize} - \item {\bf Blocking}: Execution is stoped in case of error + \item {\bf Blocking}: Execution is stoped in case of error or module cancel \begin{itemize} \item Current workflow's blocking execution path is {\bf stopped} - \item Any other blocking path of next workflows {\bf will not be executed} \end{itemize} \vspace{0.5em} - \item {\bf Non-blocking/Deferred}: Stop execution for current path only + \item {\bf Non-blocking/Parallel}: Stop execution for current path only \begin{itemize} \item Current execution path is {\bf stopped} \item {\bf Resume} execution of remaining paths - \item Paths from other workflow will be {\bf executed} \end{itemize} \end{itemize} \end{frame} -\begin{frame} - \frametitle{Execution Order and Execution Types} - \begin{itemize} - \item \textbf{Blocking} paths from all workflows are executed in the saved order - \item If any blocking executions failed, the action that called the trigger will \textbf{be stopped} - \item Then, \textbf{Parallel/Deferred} paths from all workflows are executed. The order is irrelevant - \end{itemize} - - \begin{center} - \includegraphics[width=0.35\linewidth]{pictures/execution-order-2.png} - \includegraphics[width=0.40\linewidth]{pictures/trigger-outputs.png} - \end{center} -\end{frame} - \begin{frame} \frametitle{Publishing example} Example: \begin{enumerate} \item An Event is published \item MISP starts the publishing process - \item MISP executes a workflow listening to the trigger + \item MISP executes the workflow listening to the trigger \begin{itemize} - \item {\bf execution success}: Proceed publishing - \item {\bf execution failure}: Stop publishing, log the reason and report the failure back to the user + \item {\bf\color{green!50!black}success execution success}: Proceed publishing + \item {\bf\color{red}success execution failure}: Stop publishing, log the reason and report the failure back to the user \end{itemize} \end{enumerate} \end{frame} @@ -124,9 +116,9 @@ \begin{frame} \frametitle{Execution context} \begin{itemize} - \item Workflow can be triggered by any users - \item However, the user for which the workflow executes is the workflow creator - \item This is to make sure users with a higher privilege will have their workflow correctly executed + \item Workflow are \textit{triggered by any users} + \item However, the user for which the workflow executes has the \texttt{site-admin} role and is from the \texttt{MISP.host\_org\_id} + \item This is to make sure, all data are processed regardless of the ACL \end{itemize} \end{frame} @@ -135,60 +127,92 @@ \begin{center} \includegraphics[width=0.5\linewidth]{pictures/module-type.png} \end{center} + 4 types of module \begin{itemize} - \item 3 types of modules + \item \textbf{logic}: Allow to redirect the execution flow. \begin{itemize} - \item \texttt{trigger}: Entry point of the execution path - \begin{itemize} - \item Event publish, email about to be sent, feed data about to be saved, ... - \end{itemize} - \item \texttt{logic}: Allow to redirect the execution flow. - \begin{itemize} - \item IF condition, fork the blocking execution into a non-blocking one, ... - \end{itemize} - \item \texttt{action}: Can modify data, prevent execution or perform additional actions - \begin{itemize} - \item Publish to ZMQ, perform enrichments, block the execution, ... - \end{itemize} + \item IF condition, fork the blocking execution into a non-blocking one, ... + \end{itemize} + \item \textbf{action}: Can modify data, prevent execution or perform additional actions + \begin{itemize} + \item Publish to ZMQ, perform enrichments, block the execution, ... + \end{itemize} + \item \textbf{misp-module}: Basically \texttt{action} modules but using the \texttt{misp-module} service for the logic + \begin{itemize} + \item Written in Python! + \end{itemize} + \item \textbf{custom}: Allow user to create their own \texttt{action} and \texttt{logic} module in PHP + \begin{itemize} + \item Can use any functions defined in the application \end{itemize} \end{itemize} \end{frame} \begin{frame} \frametitle{Workflow modules} + \texttt{action} modules can be from 3 sources \begin{itemize} - \item \texttt{action} modules can be from 2 sources + \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php} \begin{itemize} - \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php} - \begin{itemize} - \item Written in PHP - \item They can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...) - \item Fast and easier to interact with for those having internal knowledge of MISP - \end{itemize} - \item \texttt{From the misp-module service} - \begin{itemize} - \item Written in Python - \item They can use any python libraries - \item Easier to write - \item New module type \texttt{action} - \end{itemize} + \item Built-in module in the application + \item Written in PHP + \item Can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...) + \item Fast and easier to interact with for those having internal knowledge of MISP + \end{itemize} + \item \texttt{\scriptsize app/Lib/WorkflowModules/action/[module\_name].php} + \begin{itemize} + \item Same as previous but allow users to create their own without sharing with the community + \end{itemize} + \item \texttt{From the misp-module service} + \begin{itemize} + \item Written in Python + \item Can use any python libraries + \item New \texttt{misp-module} module type: \texttt{action} \end{itemize} - \item Both systems are \textbf{plug-and-play} \end{itemize} + \begin{center} + $\rightarrow$ Both the PHP and Python systems are \textbf{plug-and-play} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Getting started with workflows} + Review MISP settings: + \begin{enumerate} + \item Make sure \texttt{MISP.background\_jobs} is turned on + \item Turn on setting \texttt{Plugin.Workflow\_enable} + \item Make sure workers are up-and-running + \end{enumerate} + \begin{center} + \includegraphics[width=0.75\linewidth]{pictures/settings-1.png} + \includegraphics[width=0.75\linewidth]{pictures/settings-2.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Getting started with workflows} + \begin{enumerate} + \item Go to the trigger list: \texttt{Administration > Workflows} + \begin{itemize} + \item \url{/workflows/triggers} + \end{itemize} + \item Turn a trigger on + \item Use the editor to edit the workflow associated to this trigger + \end{enumerate} \end{frame} \begin{frame} \frametitle{Creating a workflow with the editor} \begin{enumerate} - \item Drag a \texttt{trigger} module from the side panel to the canvas + \item Choose a \texttt{trigger} from the list \item Drag an \texttt{action} module from the side panel to the canvas \item From the \texttt{trigger} output, drag an arrow into the \texttt{action} input (left side) - \begin{itemize} - \item You can choose between a \texttt{blocking} and \texttt{non-blocking} execution path by using the associated trigger output - \end{itemize} \end{enumerate} \begin{center} - \includegraphics[width=1.0\linewidth]{pictures/editor-1.png} + \includegraphics[width=0.8\linewidth]{pictures/usage-1.png} + \end{center} + \begin{center} + \includegraphics[width=0.50\linewidth]{pictures/editor-1.png} \end{center} \end{frame} @@ -196,10 +220,9 @@ \frametitle{Working with the editor} Operations not allowed: \begin{itemize} - \item Can only add the same trigger once - \item Execution loop are blocked + \item Execution loop are not authorized \begin{itemize} - \item Current caveat: If an action re-trigger the workflow + \item Current caveat: If an action re-run the workflow in any way \end{itemize} \end{itemize} \begin{center} @@ -207,223 +230,66 @@ \end{center} \end{frame} +\begin{frame} + \frametitle{Workflow blueprints: Create} + Select one or more modules to be saved as blueprint then click on the \texttt{save blueprint} button + \begin{center} + \includegraphics[width=0.85\linewidth]{pictures/blueprint-1.png} + \end{center} +\end{frame} + +\begin{frame} + \frametitle{Module filtering} + \begin{itemize} + \item Some action module accept \texttt{module filtering} conditions + \item For example, the \texttt{enrich-event} module will only perform the enrichment on Attribute having a \texttt{tlp:white} tag + \end{itemize} + \begin{center} + \includegraphics[width=0.7\linewidth]{pictures/module-filtering.png} + \end{center} +\end{frame} + \begin{frame} \frametitle{TODOs / FIXMEs} \begin{enumerate} - \item Global setting to enable/disabled Workflows - \item Settings to enable/disable modules - \item Increase workflow run counter \item Show which workflows use a module and the other way around \item Perfom parallel execution by a worker (currently in-line) \item Implement parallel task module \item ACL-aware: new \texttt{workflow editor} role \item Standardize how data is passed between modules \end{enumerate} - \begin{enumerate} - \item Various UI glitches \& inconsistencies - \end{enumerate} -\end{frame} - -\begin{frame} - \frametitle{Implemented modules} - \begin{itemize} - \item \textbf{trigger}\texttt{:publish} - Is called whenever an event is published - \item \textbf{logic}\texttt{:if} - Allows to match on Event and Attribute using restSearch - \item \textbf{action}\texttt{:push\_to\_ZMQ} - Pushes provided data to ZMQ, mainly for debugging purposes - \item \textbf{action}\texttt{:enrich\_event} - Enriches the provided Event (or Attributes if filtered by \texttt{if}) for the specified misp-module - \item \textbf{action}\texttt{:stop\_execution} - Stops execution for current path (and all blocking paths) - \end{itemize} \end{frame} \section{Learning by examples} \begin{frame} \frametitle{Workflow example 1} \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-1.png} + \includegraphics[width=0.95\linewidth]{pictures/example-1.png} \end{center} - \begin{enumerate} - \item Will the next blocking path (from another workflow) be executed? - \end{enumerate} -\end{frame} -\begin{frame} - \frametitle{Workflow example 1: Answers} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-1.png} - \end{center} - - \begin{enumerate} - \item Will the next blocking path (from another workflow) be executed? - \begin{itemize} - \item \textbf{No}. We are in a blocking path. As the execution has been stopped, no other blocking paths will be executed. - \end{itemize} - \end{enumerate} + \begin{itemize} + \item The \texttt{zmq} module will be run if at least one of the attribute has the \texttt{tlp:white} tag. + \end{itemize} \end{frame} \begin{frame} \frametitle{Workflow example 2} \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-2.png} + \includegraphics[width=0.95\linewidth]{pictures/example-2.png} \end{center} - \begin{enumerate} - \item Will the next blocking path (from another workflow) be executed? - \item Will \texttt{Enrich Event} module be executed? - \end{enumerate} -\end{frame} -\begin{frame} - \frametitle{Workflow example 2: Answers} - \begin{center} - \includegraphics[width=0.7\linewidth]{pictures/example-2.png} - \end{center} - - \begin{enumerate} - \item Will the next blocking path (from another workflow) be executed? - \begin{itemize} - \item \textbf{No}. Same reason that before - \end{itemize} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{Yes}. The module is in the non-blocking path. Regardless of the result of the blocking path, it will be executed. - \end{itemize} - \end{enumerate} + \begin{itemize} + \item If an event has the \texttt{PAP:RED} tag or any of the attribute has it, the enrichment process will be cancelled + \end{itemize} \end{frame} \begin{frame} \frametitle{Workflow example 3} \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-3.png} + \includegraphics[width=0.65\linewidth]{pictures/example-3.png} \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \item Will the next blocking path (from another workflow) be executed? - \end{enumerate} + \begin{itemize} + \item After a user has been saved, a message containing the user's email will be sent to a Mattermost channel and the user detailed will be posted to the webhook URL + \end{itemize} \end{frame} - -\begin{frame} - \frametitle{Workflow example 3: Answers} - \begin{center} - \includegraphics[width=0.55\linewidth]{pictures/example-3.png} - \end{center} - - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{Yes} - \item The blocking path is executed before the non-blocking one - \item The result of the non-blocking path has no influence on the blocking one - \end{itemize} - \item Will the next blocking path (from another workflow) be executed? - \begin{itemize} - \item \textbf{Yes} - \item The blocking path is executed before the non-blocking one - \item The result of the non-blocking path has no influence the execution of other workflows - \end{itemize} - \end{enumerate} -\end{frame} - -\begin{frame} - \frametitle{Workflow example 4} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-4.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \end{enumerate} -\end{frame} - - -\begin{frame} - \frametitle{Workflow example 4: Answers} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-4.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{Yes} and \textbf{No}. The execution order for the same output is not guaranteed - \item If \texttt{Stop execution} is executed first, it's a no. - \end{itemize} - \end{enumerate} -\end{frame} - -\begin{frame} - \frametitle{Workflow example 5} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-5.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \end{enumerate} -\end{frame} -\begin{frame} - \frametitle{Workflow example 5: Answers} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-5.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{Yes}. The execution order for the same output is not guaranteed - \item However, as we are in a non-blocking path, the outcome of the execution of another path has no impact - \end{itemize} - \end{enumerate} -\end{frame} - -\begin{frame} - \frametitle{Workflow example 6} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-6.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \end{enumerate} -\end{frame} -\begin{frame} - \frametitle{Workflow example 6: Answers} - \begin{center} - \includegraphics[width=0.9\linewidth]{pictures/example-6.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{No}. Even if we are in a non-blocking path, if the current execution path is blocked, the execution will be stopped - \end{itemize} - \end{enumerate} -\end{frame} - -\begin{frame} - \frametitle{Workflow example 7} - \vspace{-2em} - \begin{center} - \includegraphics[width=1.05\linewidth]{pictures/example-7.png} - \end{center} - \begin{center} - \includegraphics[width=0.45\linewidth]{pictures/event-1.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \item Will \texttt{circl.lu} have a tag attached to it? - \end{enumerate} -\end{frame} -\begin{frame} - \frametitle{Workflow example 7: Answers} - \begin{center} - \includegraphics[width=0.7\linewidth]{pictures/example-7.png} - \end{center} - \begin{center} - \includegraphics[width=0.3\linewidth]{pictures/event-1.png} - \end{center} - \begin{enumerate} - \item Will \texttt{Enrich Event} module be executed? - \begin{itemize} - \item \textbf{Yes}. The event contains an attribute satisfying the matching condition - \end{itemize} - \item Will \texttt{circl.lu} have a tag attached to it? - \begin{itemize} - \item \textbf{No}. The event contains an attribute satisfying the matching condition. The \texttt{else} part will not be executed. - \end{itemize} - \end{enumerate} -\end{frame} - diff --git a/a.12-misp-workflows/pictures/ATT&CK-support.png b/a.12-misp-workflows/pictures/ATT&CK-support.png deleted file mode 100644 index 0591bb2..0000000 Binary files a/a.12-misp-workflows/pictures/ATT&CK-support.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/blueprint-1.png b/a.12-misp-workflows/pictures/blueprint-1.png new file mode 100644 index 0000000..1e3acbf Binary files /dev/null and b/a.12-misp-workflows/pictures/blueprint-1.png differ diff --git a/a.12-misp-workflows/pictures/blueprint-2.png b/a.12-misp-workflows/pictures/blueprint-2.png new file mode 100644 index 0000000..bfd3e3e Binary files /dev/null and b/a.12-misp-workflows/pictures/blueprint-2.png differ diff --git a/a.12-misp-workflows/pictures/editor-1.png b/a.12-misp-workflows/pictures/editor-1.png index 2790cfb..c8c3edf 100644 Binary files a/a.12-misp-workflows/pictures/editor-1.png and b/a.12-misp-workflows/pictures/editor-1.png differ diff --git a/a.12-misp-workflows/pictures/editor-not-allowed-1.png b/a.12-misp-workflows/pictures/editor-not-allowed-1.png index 1937e49..d4dc939 100644 Binary files a/a.12-misp-workflows/pictures/editor-not-allowed-1.png and b/a.12-misp-workflows/pictures/editor-not-allowed-1.png differ diff --git a/a.12-misp-workflows/pictures/event-1.png b/a.12-misp-workflows/pictures/event-1.png deleted file mode 100644 index 9df937a..0000000 Binary files a/a.12-misp-workflows/pictures/event-1.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/example-1.png b/a.12-misp-workflows/pictures/example-1.png index 65c6da9..428454b 100644 Binary files a/a.12-misp-workflows/pictures/example-1.png and b/a.12-misp-workflows/pictures/example-1.png differ diff --git a/a.12-misp-workflows/pictures/example-2.png b/a.12-misp-workflows/pictures/example-2.png index 49da900..ecd466e 100644 Binary files a/a.12-misp-workflows/pictures/example-2.png and b/a.12-misp-workflows/pictures/example-2.png differ diff --git a/a.12-misp-workflows/pictures/example-3.png b/a.12-misp-workflows/pictures/example-3.png index 0277695..8b466ef 100644 Binary files a/a.12-misp-workflows/pictures/example-3.png and b/a.12-misp-workflows/pictures/example-3.png differ diff --git a/a.12-misp-workflows/pictures/example-4.png b/a.12-misp-workflows/pictures/example-4.png deleted file mode 100644 index a219647..0000000 Binary files a/a.12-misp-workflows/pictures/example-4.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/example-5.png b/a.12-misp-workflows/pictures/example-5.png deleted file mode 100644 index 5f8023a..0000000 Binary files a/a.12-misp-workflows/pictures/example-5.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/example-6.png b/a.12-misp-workflows/pictures/example-6.png deleted file mode 100644 index 9870d81..0000000 Binary files a/a.12-misp-workflows/pictures/example-6.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/example-7.png b/a.12-misp-workflows/pictures/example-7.png deleted file mode 100644 index 5b5ad1a..0000000 Binary files a/a.12-misp-workflows/pictures/example-7.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/execution-order-1.png b/a.12-misp-workflows/pictures/execution-order-1.png deleted file mode 100644 index 0af73ab..0000000 Binary files a/a.12-misp-workflows/pictures/execution-order-1.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/execution-order-2.png b/a.12-misp-workflows/pictures/execution-order-2.png deleted file mode 100644 index c215aab..0000000 Binary files a/a.12-misp-workflows/pictures/execution-order-2.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/module-filtering.png b/a.12-misp-workflows/pictures/module-filtering.png new file mode 100644 index 0000000..876d5ad Binary files /dev/null and b/a.12-misp-workflows/pictures/module-filtering.png differ diff --git a/a.12-misp-workflows/pictures/settings-1.png b/a.12-misp-workflows/pictures/settings-1.png new file mode 100644 index 0000000..290851b Binary files /dev/null and b/a.12-misp-workflows/pictures/settings-1.png differ diff --git a/a.12-misp-workflows/pictures/settings-2.png b/a.12-misp-workflows/pictures/settings-2.png new file mode 100644 index 0000000..712a31a Binary files /dev/null and b/a.12-misp-workflows/pictures/settings-2.png differ diff --git a/a.12-misp-workflows/pictures/simple-workflow.png b/a.12-misp-workflows/pictures/simple-workflow.png deleted file mode 100644 index 399abf0..0000000 Binary files a/a.12-misp-workflows/pictures/simple-workflow.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/trigger-outputs.png b/a.12-misp-workflows/pictures/trigger-outputs.png deleted file mode 100644 index 18cb19c..0000000 Binary files a/a.12-misp-workflows/pictures/trigger-outputs.png and /dev/null differ diff --git a/a.12-misp-workflows/pictures/usage-1.png b/a.12-misp-workflows/pictures/usage-1.png new file mode 100644 index 0000000..b80b305 Binary files /dev/null and b/a.12-misp-workflows/pictures/usage-1.png differ