chg: [a.12-workflows] Updated slides to reflect the current design

a.12-misp-workflows/content.tex

@@ -20,7 +20,7 @@
         %\item Initial idea came from GeekWeek7.5\footnote{Workshop organized by the Canadian Cyber Center}{https://cyber.gc.ca/en/events/geekweek-75}
         \item Initial idea came from GeekWeek7.5\footnote{\href{https://cyber.gc.ca/en/events/geekweek-75}{Workshop organized by the Canadian Cyber Center}}
         \begin{center}
-            \includegraphics[width=0.5\linewidth]{pictures/geekweek75.jpg}
+            \includegraphics[width=0.3\linewidth]{pictures/geekweek75.jpg}
         \end{center}
         \item Experienced users wanted to be able to interact with the behavior of MISP for specific operations
         \item Same spirit than web-hooks but more flexible
@@ -28,6 +28,9 @@
         \begin{itemize}
             \item Prevent publication of events not meeting some criterias
             \item Enrich events before the actual publication takes place
+            \item Prevent querying thrid-party service (e.g. virustotal) for sensitive information
+            \item Send a notification in chat room when new events get published
+            \item And much much more..
         \end{itemize}
     \end{itemize}
 \end{frame}
@@ -37,9 +40,12 @@
     \begin{enumerate}
         \item \textbf{User Interacts} with MISP using the UI or API
         \item MISP handles the request, starts \textbf{preparing data} to perform the operation
-        \item MISP checks if there are workflows \textbf{listening to the trigger}
+        \item MISP checks if there is an enabled workflow \textbf{listening to the trigger}
         \item MISP fetches enabled workflows and \textbf{executes} them
         \item If all went fine, MISP \textbf{continue} to perform the operation
+        \begin{itemize}
+            \item The operation can potentially be cancelled by \texttt{blocking} modules
+        \end{itemize}
     \end{enumerate}
 \end{frame}
 
@@ -51,9 +57,9 @@
         \begin{itemize}
             \item A workflow can contain more than one execution path
         \end{itemize}
-        \item \textbf{trigger}: Starting point of an \texttt{execution path}. Triggers are called when specific action are done by MISP
+        \item \textbf{trigger}: Starting point of an \texttt{execution path}. Triggers are called when specific actions are done by MISP
         \begin{itemize}
-            \item A workflow can contain more than one trigger, but only one per type
+            \item A trigger can only have one workflow and vice-versa
         \end{itemize}
     \end{enumerate}
     \begin{center}
@@ -64,13 +70,15 @@
 \begin{frame}
     \frametitle{Workflow execution}
     \begin{enumerate}
+        \item An operation happen in MISP (e.g. event publication)
         \item A trigger is called
-        \item Collect workflows listening to called trigger
+        \item Collect enabled workflow listening to called trigger
-        \item Execute workflows in the saved order
+        \item Execute workflow
+        \begin{itemize}
+            \item \texttt{\color{green!50!black}success}: Proceed with the operation
+            \item \texttt{\color{red}failure} | \texttt{\color{blue}cancel}: Cancel the operation
+        \end{itemize}
     \end{enumerate}
-    \begin{center}
-        \includegraphics[width=0.5\linewidth]{pictures/execution-order-1.png}
-    \end{center}
 \end{frame}
 
 \begin{frame}
@@ -78,45 +86,29 @@
     Currently 2 types of execution path:
     \vspace{0.5em}
     \begin{itemize}
-        \item {\bf Blocking}: Execution is stoped in case of error
+        \item {\bf Blocking}: Execution is stoped in case of error or module cancel
         \begin{itemize}
             \item Current workflow's blocking execution path is {\bf stopped}
-            \item Any other blocking path of next workflows {\bf will not be executed}
         \end{itemize}
         \vspace{0.5em}
-        \item {\bf Non-blocking/Deferred}: Stop execution for current path only
+        \item {\bf Non-blocking/Parallel}: Stop execution for current path only
         \begin{itemize}
             \item Current execution path is {\bf stopped}
             \item {\bf Resume} execution of remaining paths
-            \item Paths from other workflow will be {\bf executed}
         \end{itemize}
     \end{itemize}
 \end{frame}
 
-\begin{frame}
-    \frametitle{Execution Order and Execution Types}
-    \begin{itemize}
-        \item \textbf{Blocking} paths from all workflows are executed in the saved order
-        \item If any blocking executions failed, the action that called the trigger will \textbf{be stopped}
-        \item Then, \textbf{Parallel/Deferred} paths from all workflows are executed. The order is irrelevant
-    \end{itemize}
-
-    \begin{center}
-        \includegraphics[width=0.35\linewidth]{pictures/execution-order-2.png}
-        \includegraphics[width=0.40\linewidth]{pictures/trigger-outputs.png}
-    \end{center}
-\end{frame}
-
 \begin{frame}
     \frametitle{Publishing example}
     Example:
     \begin{enumerate}
         \item An Event is published
         \item MISP starts the publishing process
-        \item MISP executes a workflow listening to the trigger
+        \item MISP executes the workflow listening to the trigger
         \begin{itemize}
-            \item {\bf execution success}: Proceed publishing
+            \item {\bf\color{green!50!black}success execution success}: Proceed publishing
-            \item {\bf execution failure}: Stop publishing, log the reason and report the failure back to the user
+            \item {\bf\color{red}success execution failure}: Stop publishing, log the reason and report the failure back to the user
         \end{itemize}
     \end{enumerate}
 \end{frame}
@@ -124,9 +116,9 @@
 \begin{frame}
     \frametitle{Execution context}
     \begin{itemize}
-        \item Workflow can be triggered by any users
+        \item Workflow are \textit{triggered by any users}
-        \item However, the user for which the workflow executes is the workflow creator
+        \item However, the user for which the workflow executes has the \texttt{site-admin} role and is from the \texttt{MISP.host\_org\_id}
-        \item This is to make sure users with a higher privilege will have their workflow correctly executed
+        \item This is to make sure, all data are processed regardless of the ACL
     \end{itemize}
 \end{frame}
 
@@ -135,60 +127,92 @@
     \begin{center}
         \includegraphics[width=0.5\linewidth]{pictures/module-type.png}
     \end{center}
+    4 types of module
     \begin{itemize}
-        \item 3 types of modules
+        \item \textbf{logic}: Allow to redirect the execution flow.
         \begin{itemize}
-            \item \texttt{trigger}: Entry point of the execution path
+            \item IF condition, fork the blocking execution into a non-blocking one, ...
-            \begin{itemize}
+        \end{itemize}
-                \item Event publish, email about to be sent, feed data about to be saved, ...
+        \item \textbf{action}: Can modify data, prevent execution or perform additional actions
-            \end{itemize}
+        \begin{itemize}
-            \item \texttt{logic}: Allow to redirect the execution flow.
+            \item Publish to ZMQ, perform enrichments, block the execution, ...
-            \begin{itemize}
+        \end{itemize}
-                \item IF condition, fork the blocking execution into a non-blocking one, ...
+        \item \textbf{misp-module}: Basically \texttt{action} modules but using the \texttt{misp-module} service for the logic
-            \end{itemize}
+        \begin{itemize}
-            \item \texttt{action}: Can modify data, prevent execution or perform additional actions
+            \item Written in Python!
-            \begin{itemize}
+        \end{itemize}
-                \item Publish to ZMQ, perform enrichments, block the execution, ...
+        \item \textbf{custom}: Allow user to create their own \texttt{action} and \texttt{logic} module in PHP
-            \end{itemize}
+        \begin{itemize}
+            \item Can use any functions defined in the application
         \end{itemize}
     \end{itemize}
 \end{frame}
 
 \begin{frame}
     \frametitle{Workflow modules}
+    \texttt{action} modules can be from 3 sources
     \begin{itemize}
-        \item \texttt{action} modules can be from 2 sources
+        \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php}
         \begin{itemize}
-            \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php}
+            \item Built-in module in the application
-            \begin{itemize}
+            \item Written in PHP
-                \item Written in PHP
+            \item Can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...)
-                \item They can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...)
+            \item Fast and easier to interact with for those having internal knowledge of MISP
-                \item Fast and easier to interact with for those having internal knowledge of MISP
+        \end{itemize}
-            \end{itemize}
+        \item \texttt{\scriptsize app/Lib/WorkflowModules/action/[module\_name].php}
-            \item \texttt{From the misp-module service} 
+        \begin{itemize}
-            \begin{itemize}
+            \item Same as previous but allow users to create their own without sharing with the community
-                \item Written in Python
+        \end{itemize}
-                \item They can use any python libraries
+        \item \texttt{From the misp-module service} 
-                \item Easier to write
+        \begin{itemize}
-                \item New module type \texttt{action}
+            \item Written in Python
-            \end{itemize}
+            \item Can use any python libraries
+            \item New \texttt{misp-module} module type: \texttt{action}
         \end{itemize}
-        \item Both systems are \textbf{plug-and-play}
     \end{itemize}
+    \begin{center}
+        $\rightarrow$ Both the PHP and Python systems are \textbf{plug-and-play}
+    \end{center}
+\end{frame}
+
+\begin{frame}
+    \frametitle{Getting started with workflows}
+    Review MISP settings:
+    \begin{enumerate}
+        \item Make sure \texttt{MISP.background\_jobs} is turned on
+        \item Turn on setting \texttt{Plugin.Workflow\_enable}
+        \item Make sure workers are up-and-running
+    \end{enumerate}
+    \begin{center}
+        \includegraphics[width=0.75\linewidth]{pictures/settings-1.png}
+        \includegraphics[width=0.75\linewidth]{pictures/settings-2.png}
+    \end{center}
+\end{frame}
+
+\begin{frame}
+    \frametitle{Getting started with workflows}
+    \begin{enumerate}
+        \item Go to the trigger list: \texttt{Administration > Workflows}
+        \begin{itemize}
+            \item \url{/workflows/triggers}
+        \end{itemize}
+        \item Turn a trigger on
+        \item Use the editor to edit the workflow associated to this trigger
+    \end{enumerate}
 \end{frame}
 
 \begin{frame}
     \frametitle{Creating a workflow with the editor}
     \begin{enumerate}
-        \item Drag a \texttt{trigger} module from the side panel to the canvas
+        \item Choose a \texttt{trigger} from the list
         \item Drag an \texttt{action} module from the side panel to the canvas
         \item From the \texttt{trigger} output, drag an arrow into the \texttt{action} input (left side)
-        \begin{itemize}
-            \item You can choose between a \texttt{blocking} and \texttt{non-blocking} execution path by using the associated trigger output
-        \end{itemize}
     \end{enumerate}
     \begin{center}
-        \includegraphics[width=1.0\linewidth]{pictures/editor-1.png}
+        \includegraphics[width=0.8\linewidth]{pictures/usage-1.png}
+    \end{center}
+    \begin{center}
+        \includegraphics[width=0.50\linewidth]{pictures/editor-1.png}
     \end{center}
 \end{frame}
 
@@ -196,10 +220,9 @@
     \frametitle{Working with the editor}
     Operations not allowed:
     \begin{itemize}
-        \item Can only add the same trigger once
+        \item Execution loop are not authorized
-        \item Execution loop are blocked
         \begin{itemize}
-            \item Current caveat: If an action re-trigger the workflow
+            \item Current caveat: If an action re-run the workflow in any way
         \end{itemize}
     \end{itemize}
     \begin{center}
@@ -207,223 +230,66 @@
     \end{center}
 \end{frame}
 
+\begin{frame}
+    \frametitle{Workflow blueprints: Create}
+    Select one or more modules to be saved as blueprint then click on the \texttt{save blueprint} button
+    \begin{center}
+        \includegraphics[width=0.85\linewidth]{pictures/blueprint-1.png}
+    \end{center}
+\end{frame}
+
+\begin{frame}
+    \frametitle{Module filtering}
+    \begin{itemize}
+        \item Some action module accept \texttt{module filtering} conditions
+        \item For example, the \texttt{enrich-event} module will only perform the enrichment on Attribute having a \texttt{tlp:white} tag
+    \end{itemize}
+    \begin{center}
+        \includegraphics[width=0.7\linewidth]{pictures/module-filtering.png}
+    \end{center}
+\end{frame}
+
 \begin{frame}
     \frametitle{TODOs / FIXMEs}
     \begin{enumerate}
-        \item Global setting to enable/disabled Workflows
-        \item Settings to enable/disable modules
-        \item Increase workflow run counter
         \item Show which workflows use a module and the other way around
         \item Perfom parallel execution by a worker (currently in-line)
         \item Implement parallel task module
         \item ACL-aware: new \texttt{workflow editor} role
         \item Standardize how data is passed between modules
     \end{enumerate}
-    \begin{enumerate}
-        \item Various UI glitches \& inconsistencies
-    \end{enumerate}
-\end{frame}
-
-\begin{frame}
-    \frametitle{Implemented modules}
-    \begin{itemize}
-        \item \textbf{trigger}\texttt{:publish} - Is called whenever an event is published
-        \item \textbf{logic}\texttt{:if} - Allows to match on Event and Attribute using restSearch
-        \item \textbf{action}\texttt{:push\_to\_ZMQ} - Pushes provided data to ZMQ, mainly for debugging purposes
-        \item \textbf{action}\texttt{:enrich\_event} - Enriches the provided Event (or Attributes if filtered by \texttt{if}) for the specified misp-module
-        \item \textbf{action}\texttt{:stop\_execution} - Stops execution for current path (and all blocking paths)
-    \end{itemize}
 \end{frame}
 
 \section{Learning by examples}
 \begin{frame}
     \frametitle{Workflow example 1}
     \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-1.png}
+        \includegraphics[width=0.95\linewidth]{pictures/example-1.png}
     \end{center}
 
-    \begin{enumerate}
+    \begin{itemize}
-        \item Will the next blocking path (from another workflow) be executed?
+        \item The \texttt{zmq} module will be run if at least one of the attribute has the \texttt{tlp:white} tag.
-    \end{enumerate}
+    \end{itemize}
-\end{frame}
-\begin{frame}
-    \frametitle{Workflow example 1: Answers}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-1.png}
-    \end{center}
-
-    \begin{enumerate}
-        \item Will the next blocking path (from another workflow) be executed?
-        \begin{itemize}
-            \item \textbf{No}. We are in a blocking path. As the execution has been stopped, no other blocking paths will be executed.
-        \end{itemize}
-    \end{enumerate}
 \end{frame}
 
 \begin{frame}
     \frametitle{Workflow example 2}
     \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-2.png}
+        \includegraphics[width=0.95\linewidth]{pictures/example-2.png}
     \end{center}
 
-    \begin{enumerate}
+    \begin{itemize}
-        \item Will the next blocking path (from another workflow) be executed?
+        \item If an event has the \texttt{PAP:RED} tag or any of the attribute has it, the enrichment process will be cancelled
-        \item Will \texttt{Enrich Event} module be executed?
+    \end{itemize}
-    \end{enumerate}
-\end{frame}
-\begin{frame}
-    \frametitle{Workflow example 2: Answers}
-    \begin{center}
-        \includegraphics[width=0.7\linewidth]{pictures/example-2.png}
-    \end{center}
-
-    \begin{enumerate}
-        \item Will the next blocking path (from another workflow) be executed?
-        \begin{itemize}
-            \item \textbf{No}. Same reason that before
-        \end{itemize}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{Yes}. The module is in the non-blocking path. Regardless of the result of the blocking path, it will be executed.
-        \end{itemize}
-    \end{enumerate}
 \end{frame}
 
 \begin{frame}
     \frametitle{Workflow example 3}
     \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-3.png}
+        \includegraphics[width=0.65\linewidth]{pictures/example-3.png}
     \end{center}
 
-    \begin{enumerate}
+    \begin{itemize}
-        \item Will \texttt{Enrich Event} module be executed?
+        \item After a user has been saved, a message containing the user's email will be sent to a Mattermost channel and the user detailed will be posted to the webhook URL
-        \item Will the next blocking path (from another workflow) be executed?
+    \end{itemize}
-    \end{enumerate}
 \end{frame}
-
-\begin{frame}
-    \frametitle{Workflow example 3: Answers}
-    \begin{center}
-        \includegraphics[width=0.55\linewidth]{pictures/example-3.png}
-    \end{center}
-
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{Yes}
-            \item The blocking path is executed before the non-blocking one
-            \item The result of the non-blocking path has no influence on the blocking one
-        \end{itemize}
-        \item Will the next blocking path (from another workflow) be executed?
-        \begin{itemize}
-            \item \textbf{Yes}
-            \item The blocking path is executed before the non-blocking one
-            \item The result of the non-blocking path has no influence the execution of other workflows
-        \end{itemize}
-    \end{enumerate}
-\end{frame}
-
-\begin{frame}
-    \frametitle{Workflow example 4}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-4.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-    \end{enumerate}
-\end{frame}
-
-
-\begin{frame}
-    \frametitle{Workflow example 4: Answers}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-4.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{Yes} and \textbf{No}. The execution order for the same output is not guaranteed
-            \item If \texttt{Stop execution} is executed first, it's a no.
-        \end{itemize}
-    \end{enumerate}
-\end{frame}
-
-\begin{frame}
-    \frametitle{Workflow example 5}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-5.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-    \end{enumerate}
-\end{frame}
-\begin{frame}
-    \frametitle{Workflow example 5: Answers}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-5.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{Yes}. The execution order for the same output is not guaranteed
-            \item However, as we are in a non-blocking path, the outcome of the execution of another path has no impact
-        \end{itemize}
-    \end{enumerate}
-\end{frame}
-
-\begin{frame}
-    \frametitle{Workflow example 6}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-6.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-    \end{enumerate}
-\end{frame}
-\begin{frame}
-    \frametitle{Workflow example 6: Answers}
-    \begin{center}
-        \includegraphics[width=0.9\linewidth]{pictures/example-6.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{No}. Even if we are in a non-blocking path, if the current execution path is blocked, the execution will be stopped
-        \end{itemize}
-    \end{enumerate}
-\end{frame}
-
-\begin{frame}
-    \frametitle{Workflow example 7}
-    \vspace{-2em}
-    \begin{center}
-        \includegraphics[width=1.05\linewidth]{pictures/example-7.png}
-    \end{center}
-    \begin{center}
-        \includegraphics[width=0.45\linewidth]{pictures/event-1.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \item Will \texttt{circl.lu} have a tag attached to it?
-    \end{enumerate}
-\end{frame}
-\begin{frame}
-    \frametitle{Workflow example 7: Answers}
-    \begin{center}
-        \includegraphics[width=0.7\linewidth]{pictures/example-7.png}
-    \end{center}
-    \begin{center}
-        \includegraphics[width=0.3\linewidth]{pictures/event-1.png}
-    \end{center}
-    \begin{enumerate}
-        \item Will \texttt{Enrich Event} module be executed?
-        \begin{itemize}
-            \item \textbf{Yes}. The event contains an attribute satisfying the matching condition
-        \end{itemize}
-        \item Will \texttt{circl.lu} have a tag attached to it?
-        \begin{itemize}
-            \item \textbf{No}. The event contains an attribute satisfying the matching condition. The \texttt{else} part will not be executed.
-        \end{itemize}
-    \end{enumerate}
-\end{frame}
-