misp-training/a.12-misp-workflows/content.tex

% DO NOT COMPILE THIS FILE DIRECTLY!
% This is included by the other .tex files.

\begin{frame}[t,plain]
\titlepage
\end{frame}

\begin{frame}
    \frametitle{Content of the presentation}
    \begin{itemize}
        \item MISP Workflows fundamentals
        \item Getting started
        \item Design of the system \& how it can be extended
    \end{itemize}

    \begin{center}
        \includegraphics[width=0.9\linewidth]{pictures/overview.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{What problems are we trying to tackle}
    \begin{itemize}
        \item Initial idea came from GeekWeek7.5\footnote{\href{https://cyber.gc.ca/en/events/geekweek-75}{Workshop organized by the Canadian Cyber Center}}
        \begin{center}
            \includegraphics[width=0.3\linewidth]{pictures/geekweek75.jpg}
        \end{center}
        \item Needs:
        \begin{itemize}
            \item Prevent default MISP behaviors
            \item Hook specific actions via callbacks
        \end{itemize}
        \item Use-cases:
        \begin{itemize}
            \item Prevent publication of events not meeting some criterias
            \item Prevent querying thrid-party services (e.g. virustotal) with sensitive information
            \item Send a notification in a chat room
            \item And much much more..
        \end{itemize}
    \end{itemize}
\end{frame}

\section{Workflow - Fundamentals}
\begin{frame}
    \frametitle{Simplistic overview of a Workflow in action}
    \begin{enumerate}
        \item An \textbf{action} is performed in MISP
        \item If there is an \textbf{enabled} Workflow for that \textbf{action}, run it
        \item If all went fine, MISP \textbf{continue} to perform the action
        \begin{itemize}
            \item The operation can potentially be cancelled by \texttt{blocking} modules
        \end{itemize}
    \end{enumerate}
\end{frame}

\begin{frame}
    \frametitle{Terminology}
    \begin{itemize}
        \item \textbf{workflow}: Sequence of all operations (nodes) to be executed. Basically the whole graph
        \item \textbf{execution path}: A path composed of nodes
        \item \textbf{trigger}: Starting point of a workflow. Triggers are called when specific actions happen in MISP
        \begin{itemize}
            \item A trigger can only have one workflow and vice-versa
        \end{itemize}
    \end{itemize}
    \begin{center}
        \includegraphics[width=1.0\linewidth]{pictures/simple-workflow.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Workflow execution process}
    Typical execution process:
    \begin{enumerate}
        \item An action happens in MISP
        \item The workflow associated to the trigger is ran
        \item Execution result?
        \begin{itemize}
            \item \texttt{\color{green!50!black}success}: Proceed the action
            \item \texttt{\color{red}failure} | \texttt{\color{blue}blocked}: Cancel the action
        \end{itemize}
    \end{enumerate}
    \vspace{0.5em}
    Example for Event publish:
    \begin{enumerate}
        \item An Event is about to be published
        \item MISP executes the workflow listening to the \texttt{event-publish} trigger
        \begin{itemize}
            \item {\bf\color{green!50!black}success}: Proceed the publishing action
            \item {\bf\color{red}failure} | \texttt{\color{blue}blocked}: Stop publishing and log the reason
        \end{itemize}
    \end{enumerate}
\end{frame}

\begin{frame}
    \frametitle{Blocking and non-blocking Workflows}
    Currently 2 types of workflows:
    \vspace{0.5em}
    \begin{itemize}
        \item {\bf Blocking}: Completion of the initial action can be prevented
        \begin{itemize}
            \item If a \textbf{blocking module} blocks the action
            \item If a \textbf{blocking module} raises an exception
        \end{itemize}
        \vspace{0.5em}
        \item {\bf Non-blocking}: Workflow execution outcome has no impact
        \begin{itemize}
            \item \textbf{Blocking modules} can still stop the execution
        \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}
    \frametitle{Execution context}
    \begin{itemize}
        \item Workflows can be triggered by \textbf{any users}
        \item Workflows can be triggered by actions done via the \textbf{UI} or \textbf{API}
        \item However, the user for which the workflow executes has:
        \begin{itemize}
            \item The \texttt{site-admin} permission
            \item Is from the \texttt{MISP.host\_org\_id}
        \end{itemize}
        \item Ensures data is processed regardless of ownership and access: \textbf{no ACL}
    \end{itemize}
\end{frame}

\begin{frame}
    \frametitle{Classes of Workflow modules}
    \begin{center}
        \includegraphics[width=0.6\linewidth]{pictures/module-type.png}
    \end{center}
    3 classes of modules
    \begin{itemize}
        \item \textbf{action}: Allow to executes actions, callbacks or scripts
        \begin{itemize}
            \item Can stop execution
            \item e.g. Webhook, block the execution, perform enrichments, ...
        \end{itemize}
        \item \textbf{logic}: Allow to redirect the execution flow.
        \begin{itemize}
            \item IF condition, fork the blocking execution into a non-blocking one, ...
        \end{itemize}
        \item \textbf{blueprint}: Allow to reuse composition of modules
        \begin{itemize}
            \item Can save subworkflows and their module's configuration
        \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}
    \frametitle{Sources of Workflow modules}
    3 sources of action modules
    \begin{itemize}
        \item Built-in \textbf{default} modules
        \begin{itemize}
            \item Written in PHP
            \item Can use MISP's built-in functionalities (restsearch, enrichment, push to zmq, ...)
            \item Fast and easier to interact with for those having internal knowledge of MISP
            \item \texttt{\scriptsize app/Model/WorkflowModules/action/[module\_name].php}
        \end{itemize}
        \item User-defined \textbf{custom} modules
        \begin{itemize}
            \item Can extend existing default modules
            \item \texttt{\scriptsize app/Lib/WorkflowModules/action/[module\_name].php}
        \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}
    \frametitle{Sources of Workflow modules}
    3 sources of action modules
    \begin{itemize}
        \item Modules from the \textbf{enrichment service}
        \begin{itemize}
            \item \textbf{Default} and \textbf{custom} modules
            \item \texttt{From the misp-module service} \includegraphics[width=0.25\linewidth]{pictures/misp-module-icon.png}
            \item Written in Python
            \item Can use any python libraries
            \item New \texttt{misp-module} module type: \texttt{action}
        \end{itemize}
    \end{itemize}
    \vspace{1em}
    \begin{center}
        $\rightarrow$ Both the PHP and Python systems are \textbf{plug-and-play}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Triggers currently available}
    Currently 8 triggers can be hooked. 3 being \textbf{blocking}.
    \includegraphics[width=1.0\linewidth]{pictures/triggers.png}
\end{frame}

\section{Workflow - Getting started}
\begin{frame}
    \frametitle{Getting started with workflows (1)}
    Review MISP settings:
    \begin{enumerate}
        \item Make sure \texttt{MISP.background\_jobs} is turned on
        \item Make sure workers are up-and-running and healthy
        \item Turn the setting \texttt{Plugin.Workflow\_enable} on
        \begin{center}
            \includegraphics[width=0.70\linewidth]{pictures/settings-1.png}
        \end{center}
        \item {[optional:misp-module]} Turn the setting \texttt{Plugin.Action\_services\_enable} on
        \begin{center}
            \includegraphics[width=0.70\linewidth]{pictures/settings-2.png}
        \end{center}
    \end{enumerate}
\end{frame}

\begin{frame}[fragile]
    \frametitle{Getting started with workflows (2)}
    If you wish to use action modules from \texttt{misp-module}, make sure to have:
    \begin{itemize}
        \item The latest update of \texttt{misp-module}
        \begin{itemize}
            \item There should be an \texttt{action\_mod} module type in \url{misp-modules/misp\_modules/modules}
        \end{itemize}
        \item Restarted your \texttt{misp-module} application
    \end{itemize}
    \begin{lstlisting}[language=text,firstnumber=1]
# This command should show all `action` modules
$ curl -s http://127.0.0.1:6677/modules | \
jq '.[] | select(.meta."module-type"[] | contains("action")) |
{name: .name, version: .meta.version}'
    \end{lstlisting}
\end{frame}

\begin{frame}
    \frametitle{Getting started with workflows (3)}
    \begin{enumerate}
        \item Go to the list of modules
        \begin{itemize}
            \item \texttt{Administration > Workflows > List Modules}
            \item or \url{/workflows/moduleIndex}
        \end{itemize}
        \item Make sure \textbf{default} modules are loaded
        \item {[optional:misp-module]} Make sure \textbf{misp-module} modules are loaded
    \end{enumerate}
\end{frame}

\begin{frame}
    \frametitle{Creating a workflow with the editor}
    \begin{enumerate}
        \item Go to the list of triggers \texttt{Administration > Workflows}
        \item Enable and edit a \texttt{trigger} from the list
        \item Drag an \texttt{action} module from the side panel to the canvas
        \item From the \texttt{trigger} output, drag an arrow into the \texttt{action}'s input (left side)
        \item Execute the action that would run the trigger and observe the effect!
    \end{enumerate}
    \begin{center}
        \includegraphics[width=0.7\linewidth]{pictures/triggers.png}
    \end{center}
    \begin{center}
        \includegraphics[width=0.50\linewidth]{pictures/editor-1.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Working with the editor}
    Operations not allowed:
    \begin{itemize}
        \item Execution loop are not authorized
        \begin{itemize}
            \item Current caveat: If an action re-run the workflow in any way
        \end{itemize}
    \end{itemize}
    \begin{center}
        \includegraphics[width=0.7\linewidth]{pictures/editor-not-allowed-1.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Working with the editor}
    Operations not allowed:
    \begin{itemize}
        \item Multiple connections from the same output
        \begin{itemize}
            \item Execution order not guaranted and confusing for users
        \end{itemize}
    \end{itemize}
    \begin{center}
        \includegraphics[width=0.7\linewidth]{pictures/editor-not-allowed-2.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Working with the editor}
    Operations showing a warning:
    \begin{itemize}
        \item \textbf{Blocking} modules after a \textbf{concurrent tasks} module
    \end{itemize}
    \begin{center}
        \includegraphics[width=1.0\linewidth]{pictures/editor-warning-1.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Workflow blueprints}
    \begin{enumerate}
        \item Blueprints allow to re-use parts of a workflow in another one
        \item Blueprints can be saved, exported and shared
    \end{enumerate}
    \begin{center}
        \includegraphics[width=0.5\linewidth]{pictures/blueprint-debugging.png}
    \end{center}
    Blueprints origins:
    \begin{enumerate}
        \item From the "official" \texttt{misp-workflow-blueprints} repository
        \item Created or imported by users
    \end{enumerate}
\end{frame}

\begin{frame}
    \frametitle{Workflow blueprints: Create}
    Select one or more modules to be saved as blueprint then click on the \texttt{save blueprint} button
    \begin{center}
        \includegraphics[width=0.85\linewidth]{pictures/blueprint-1.png}
    \end{center}
\end{frame}

\begin{frame}[fragile]
    \frametitle{Hash path filtering}
    \begin{itemize}
        \item Some modules have the possibility to filter or check conditions using CakePHP's path expression.
    \end{itemize}
\begin{lstlisting}[language=javascript,firstnumber=1]
$path_expression = '{n}[name=fred].id';
$users = [
    {'id': 123, 'name': 'fred', 'surname': 'bloggs'},
    {'id': 245, 'name': 'fred', 'surname': 'smith'},
    {'id': 356, 'name': 'joe', 'surname': 'smith'},
];
$ids = Hash::extract($users, $path_expression);
// => $ids will be [123, 245]
\end{lstlisting}
\begin{center}
    \includegraphics[width=0.4\linewidth]{pictures/module-if-generic.png}
\end{center}
\end{frame}

\begin{frame}
    \frametitle{Module filtering}
    \begin{itemize}
        \item Some action modules accept \textbf{filtering} conditions
        \item E.g. the \texttt{enrich-event} module will only perform the enrichment on Attributes having a \texttt{tlp:white} Tag
    \end{itemize}
    \begin{center}
        \includegraphics[width=0.7\linewidth]{pictures/module-filtering.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Data format in Workflows}
    \begin{center}
        \includegraphics[width=0.7\linewidth]{pictures/workflow-trigger.png}
    \end{center}
    \begin{itemize}
        \item All triggers will inject data in a workflow
        \item In some cases, there is no format (e.g. User after-save)
        \item In others, the format is \textbf{compliant with the MISP Core format}
        \item In addition to the RFC, the passed data has \textbf{additional properties}
        \begin{itemize}
            \item Attributes are always encapsulated in the Event or Object
            \item Additional key \texttt{\_AttributeFlattened}
            \item Additional key \texttt{\_allTags}
            \item Additional key \texttt{inherited} for Tags
        \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}
    \frametitle{Logic module: Concurrent Task}
    \begin{itemize}
        \item Special type of \textbf{logic} module allowing multiple connections
        \item Allows breaking the execution flow into a \textbf{concurrent tasks} to be executed later on by a background worker
        \item As a side effect, blocking modules \textbf{cannot cancel} an ongoing operation anymore
    \end{itemize}
    \begin{center}
        \includegraphics[width=0.45\linewidth]{pictures/module-concurrent.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Debugging Workflows: Log Entries}
    \begin{itemize}
        \item Workflow execution is logged in the application logs:
        \begin{itemize}
            \item \texttt{/admin/logs/index}
        \end{itemize}
        \item Or stored on disk in the following file:
        \begin{itemize}
            \item \texttt{/app/tmp/logs/workflow-execution.log}
        \end{itemize}
        \item Use the \texttt{webhook-listener.py} tool
        \begin{itemize}
            \item \texttt{/app/tools/misp-workflows/webhook-listener.py}
        \end{itemize}
    \end{itemize}
    \begin{center}
        \includegraphics[width=1.0\linewidth]{pictures/workflow-debug.png}
    \end{center}
\end{frame}

\begin{frame}
    \frametitle{Debugging Workflows: Debug mode}
    \begin{itemize}
        \item The \includegraphics[width=70px]{pictures/debug-mode.png} can be turned on for each workflows
        \item Each nodes will send data to the provided URL
        \begin{itemize}
            \item Configure the setting: \texttt{Plugin.Workflow\_debug\_url}
        \end{itemize}
        \item Result can be visualized In
        \begin{itemize}
            \item \textbf{offline}: \texttt{tools/misp-workflows/webhook-listener.py}
            \item \textbf{online}: \url{requestbin.com}
        \end{itemize}
    \end{itemize}
    \begin{center}
        \includegraphics[width=0.6\linewidth]{pictures/request-bin.png}
    \end{center}
\end{frame}

\section{Learning by examples}
\begin{frame}
    \frametitle{Workflow example 1}
    \begin{center}
        \includegraphics[width=1.0\linewidth]{pictures/example-1a.png}
    \end{center}

    \begin{enumerate}
        \item The \texttt{Event-Publish} trigger uses the MISP core format
        \item The \texttt{IF::Tag} module checks if at least one of the Attribute has the \texttt{tlp:white} tag
        \item If it does, the \texttt{Push-to-ZMQ} module will be executed
    \end{enumerate}
\end{frame}

\begin{frame}
    \frametitle{Workflow example 2}
    \begin{center}
        \includegraphics[width=1.0\linewidth]{pictures/example-2a.png}
    \end{center}

    \begin{itemize}
        \item If an event has the \texttt{tlp:red} tag or any of the attribute has it, the publish process will be cancelled
    \end{itemize}
\end{frame}


\section{Extending the system}
\begin{frame}
    \frametitle{Creating a new module in PHP}
    \begin{center}
        \includegraphics[width=0.65\linewidth]{pictures/custom-1.png}
    \end{center}

    \begin{itemize}
        \item Module configuration are defined as public variables
        \item The \texttt{exec} function has to be implemented.
        \begin{itemize}
            \item If it returns \textbf{true}, execution will proceed
            \item If it returns \textbf{false}
            \begin{itemize}
                \item And the module is blocking, the execution will stop and the operation will be blocked
                \item And the module is not blocking, the execution for the current path will be stopped
            \end{itemize}
        \end{itemize}
    \end{itemize}
\end{frame}


\begin{frame}
    \frametitle{Creating a new module in Python}
    \begin{center}
        \includegraphics[width=0.65\linewidth]{pictures/custom-2.png}
    \end{center}

    \begin{itemize}
        \item Module configuration are defined in the \texttt{moduleinfo} and \texttt{moduleconfig} variables
        \item The \texttt{handler} function has to be implemented.
        \item Blocking logic is the same as other modules
    \end{itemize}
\end{frame}