new: [6.0/a] added

6.0-misp-dashboard/content.tex

@@ -0,0 +1,177 @@
+% DO NOT COMPILE THIS FILE DIRECTLY!
+% This is included by the other .tex files.
+
+\begin{frame}[t,plain]
+\titlepage
+\end{frame}
+
+\section{MISP ZeroMQ}
+\begin{frame}
+\frametitle{MISP ZeroMQ}
+    MISP includes a flexible publish-subscribe model to allow real-time integration of the MISP activities:
+    \begin{itemize}
+        \item Event publication
+        \item Attribute creation or removal
+        \item Sighting
+        \item User login
+    \end{itemize}
+    \begin{center}
+        $\rightarrow$ Operates at global level in MISP
+    \end{center}
+\end{frame}
+
+\begin{frame}
+    \frametitle{MISP ZeroMQ}
+MISP ZeroMQ functionality can be used for various model of integration or to extend MISP functionalities:
+    \begin{itemize}
+        \item Real-time search of indicators into a SIEM\footnote{Security Information \& Event Management}
+        \item Dashboard activities
+        \item Logging mechanisms
+        \item Continuous indexing
+        \item Custom software or scripting
+    \end{itemize}
+\end{frame}
+
+\section{MISP-Dashboard: An introduction}
+\begin{frame}
+    \frametitle{MISP-Dashboard - Realtime activities and threat intelligence}
+    \vspace{-10px}
+    \begin{center}
+        \includegraphics[width=1.00\linewidth]{images/dashboard-live.png}
+    \end{center}
+\end{frame}
+
+\begin{frame}
+\frametitle{MISP-Dashboard - Features}
+    \vskip -0.5em
+    \begin{center}
+        \centering
+        \includegraphics[scale=0.08]{images/dashboard-geo.png}
+        $\;$
+        \includegraphics[scale=0.08]{images/dashboard-trendings.png}
+    \end{center}
+    \vskip -0.9em
+    \begin{itemize}
+        \item Subscribe to multiple \textbf{ZMQ} MISP instances
+        \item Provides historical geolocalised information
+        \item Present an experimental \textbf{Gamification of the platform}
+        \item Shows when and how MISP is used
+        \item Provides real time information showing current threats and activity
+    \end{itemize}
+\end{frame}
+
+\section{MISP-Dashboard: Architecture and development}
+\lstset{style=bash}
+\begin{frame}[fragile]
+\frametitle{Setting up the dashboard}
+\begin{enumerate}
+    \item Be sure to have a running redis server: e.g.
+        \begin{itemize}
+            \item \texttt{redis-server -p 6250}
+        \end{itemize}
+    \item Update your configuration in \texttt{config.cfg}
+    \item Activate your virtualenv:
+        \begin{itemize}
+            \item \texttt{. ./DASHENV/bin/activate}
+        \end{itemize}
+    \item Listen to the MISP feed by starting the zmq\_subscriber:
+        \begin{itemize}
+            \item \texttt{./zmq\_subscriber.py}
+        \end{itemize}
+    \item Start the dispatcher to process received messages:
+        \begin{itemize}
+            \item \texttt{./zmq\_dispatcher.py}
+        \end{itemize}
+    \item Start the Flask server:
+        \begin{itemize}
+            \item \texttt{./server.py}
+        \end{itemize}
+    \item Access the interface at \url{http://localhost:8001/}
+\end{enumerate}
+\end{frame}
+
+\begin{frame}
+\textbf{\large MISP-Dashboard architecture}\\
+
+\begin{center}
+    \vskip -1.7em
+    \includegraphics[scale=0.195]{images/messagepassing.png}
+\end{center}
+\end{frame}
+
+
+\lstset{style=code,language=python}
+\lstset{basicstyle=\fontsize{7}{9}\ttfamily}
+\begin{frame}[fragile]
+    \frametitle{Writing your handler}
+    \begin{lstlisting}
+# Register your handler
+dico_action = {
+        "misp_json":                  handler_dispatcher,
+        "misp_json_event":            handler_event,
+        "misp_json_self":             handler_keepalive,
+        "misp_json_attribute":        handler_attribute,
+        "misp_json_object":           handler_object,
+        "misp_json_sighting":         YOUR_CUSTOM_SIGHTINGS_HANDLER,
+        "misp_json_organisation":     handler_log,
+        "misp_json_user":             handler_user,
+        "misp_json_conversation":     handler_conversation,
+        "misp_json_object_reference": handler_log,
+}
+    \end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+    \begin{lstlisting}
+# Implement your handler
+
+# e.g. user handler
+def handler_user(zmq_name, jsondata):
+    # json action performed by the user
+    action = jsondata['action']
+    # user json data
+    json_user = jsondata['User']
+    # organisation json data
+    json_org = jsondata['Organisation']
+    # organisation name
+    org = json_org['name']
+    # only consider user login
+    if action == 'login':
+        timestamp = time.time()
+        # users_helper is a class to interact with the DB
+        users_helper.add_user_login(timestamp, org)
+    \end{lstlisting}
+\end{frame}
+
+\begin{frame}
+    \frametitle{Future development}
+    \begin{itemize}
+        \item[] \includegraphics[width=20px]{images/icons/joystick.png} \; Optimizing contribution scoring and model to encourage sharing and contributions enrichment
+        \item[] \includegraphics[width=20px]{images/icons/globe.png} \; Increasing geolocation coverage
+        \item[] \includegraphics[width=20px]{images/icons/zoom.png} \; Global filtering capabilities
+            \begin{itemize}
+                \item[] \quad - Geolocation: Showing wanted attribute or only on specific region
+                \item[] \quad - Trendings: Showing only specified taxonomies
+            \end{itemize}
+        \item[] \includegraphics[width=20px]{images/icons/MISP.png} \; Tighter integration with MISP
+            \begin{itemize}
+                \item[] \quad - Present in MISP by default
+                \item[] \quad - Authenticated / ACL enabled version
+            \end{itemize}
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+   \frametitle{Conclusion}
+MISP-Dashboard can provides realtime information to support security teams, CSIRTs or SOC showing current threats and activity by providing:
+    \begin{itemize}
+        \item Historical geolocalised information
+        \item Geospatial information from specific regions
+        \item The most active events, categories, tags, attributes, ...
+    \end{itemize}
+
+\vskip 0.5em
+It also propose a prototype of gamification of the platform providing incentive to share and contribute to the community
+\end{frame}
+
+

6.0-misp-dashboard/slide.tex

@@ -0,0 +1,139 @@
+\documentclass{beamer}
+\usetheme[numbering=progressbar]{focus}
+\definecolor{main}{RGB}{47, 161, 219}
+\definecolor{textcolor}{RGB}{128, 128, 128}
+\definecolor{background}{RGB}{240, 247, 255}
+
+
+\usepackage[utf8x]{inputenc}
+\usepackage{listings}
+\usepackage{soul}
+\usepackage{siunitx}
+\usepackage{booktabs}
+
+\usepackage{tikz}
+\usetikzlibrary{shapes,snakes,automata,positioning}
+
+\usepackage{xcolor}
+\usepackage{colortbl}
+\definecolor{mygreen}{rgb}{0,0.6,0}
+\definecolor{mygreen2}{rgb}{0,0.56,0.16}
+\definecolor{myred}{rgb}{0.6,0.066,0.066}
+\definecolor{redCIRCL}{RGB}{213,43,30}
+\definecolor{mygray}{rgb}{0.5,0.5,0.5}
+\definecolor{mymauve}{rgb}{0.58,0,0.82}
+\definecolor{mygray}{gray}{0.9}
+\definecolor{mywhite}{rgb}{1,1,1}
+\definecolor{myblack}{rgb}{0,0,0}
+\definecolor{mybeige}{HTML}{eeeeee}
+%\usepackage{tcolorbox}
+\usepackage[listings]{tcolorbox}
+\tcbuselibrary{listings}
+
+\lstdefinestyle{code}{ %
+  backgroundcolor=\color{mybeige},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\ttfamily,        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single,	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{blue},       % keyword style
+  language=Python,                 % the language of the code
+  morekeywords={*,...},           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{myblack}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstdefinestyle{bash}{ %
+  backgroundcolor=\color{black!85},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\color{mywhite},        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{white}\bfseries,       % keyword style
+  language=bash,                 % the language of the code
+  morekeywords={*,$,git, clone,... },           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{mywhite}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstdefinestyle{default}{ %
+  backgroundcolor=\color{white},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\color{black},        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{white}\bfseries,       % keyword style
+  language=bash,                 % the language of the code
+  morekeywords={*,$,git, clone,... },           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{black}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstset{style=code}
+
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+      {\color{white} \usebeamerfont{title}\insertsectionhead}\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\author{\small{\input{../includes/authors.txt}}}
+
+\title{MISP Dashboard}
+\subtitle{Real-time overview of threat intelligence from MISP instances}
+\institute{info@circl.lu}
+\titlegraphic{\includegraphics[scale=0.85]{misp.pdf}}
+\date{\today}
+
+\begin{document}
+\include{content}
+\end{document}
+

a.1-devintro/content.tex

@@ -0,0 +1,220 @@
+% DO NOT COMPILE THIS FILE DIRECTLY!
+% This is included by the other .tex files.
+
+\begin{frame}
+\titlepage
+\end{frame}
+
+\begin{frame}
+  \frametitle{Some things to know in advance...}
+  \begin{itemize}
+    \item MISP is based on PHP 5.6+
+    \item Using the MVC framework CakePHP 2.x
+    \item What we'll look at now will be a quick glance at the structuring / layout of the code
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{MVC frameworks in general}
+  \begin{itemize}
+    \item separation of business logic and views, interconnected by controllers
+    \item main advantage is clear separation of the various components
+    \item lean controllers, fat models (kinda...)
+    \item domain based code reuse
+    \item No interaction between Model and Views, ever
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Structure of MISP Core app directories}
+  \begin{itemize}
+    \item Config: general configuration files
+    \item Console: command line tools
+    \item Controller: Code dealing with requests/responses, generating data for views based on interactions with the models
+    \item Lib: Generic reusable code / libraries
+    \item Model: Business logic, data gathering and modification
+    \item Plugin: Alternative location for plugin specific codes, ordered into controller, model, view files
+    \item View: UI views, populated by the controller
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Controllers - scope}
+  \begin{itemize}
+    \item Each public function in a controller is exposed as an API action
+    \item request routing (admin routing)
+    \item multi-use functions (POST/GET)
+    \item request/response objects
+    \item contains the action code, telling the application what data fetching/modifying calls to make, preparing the resulting data for the resulting view
+    \item grouped into controller files based on model actions
+    \item Accessed via UI, API, AJAX calls directly by users
+    \item For code reuse: behaviours
+    \item Each controller bound to a model
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Controllers - functionalities of controllers}
+  \begin{itemize}
+    \item pagination functionality
+    \item logging functionality
+    \item Controllers actions can access functionality / variables of Models
+    \item Controllers cannot access code of other controller actions (kind of...)
+    \item Access to the authenticated user's data
+    \item beforeFilter(), afterFilter() methods
+    \item Inherited code in AppController
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Controllers - components}
+  \begin{itemize}
+    \item Components = reusable code for Controllers
+    \begin{itemize}
+      \item Authentication components
+      \item RestResponse component
+      \item ACL component
+      \item Cidr component
+      \item IOCImport component (should be moved)
+    \end{itemize}
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Controllers - additional functionalities}
+  \begin{itemize}
+    \item code handling API requests
+    \item auth/session management
+    \item ACL management
+    \item API management
+    \item Security component
+    \item important: quertString/PyMISP versions, MISP version handler
+    \item future improvements to the export mechanisms
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Models - scope}
+\begin{itemize}
+  \item Controls anything that has to do with:
+    \begin{itemize}
+    \item finding subsets of data
+    \item altering existing data
+    \item inherited model: AppModel
+    \item reusable code for models: Behaviours
+    \item regex, trim
+    \end{itemize}
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Models - hooking system}
+  \begin{itemize}
+    \item Versatile hooking system
+    \begin{itemize}
+      \item manipulate the data at certain stages of execution
+      \item code can be located in 3 places: Model hook, AppModel hook, behaviour
+    \end{itemize}
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Model - hooking pipeline (add/edit)}
+  \begin{itemize}
+  \item Hooks / model pipeline for data creation / edits
+    \begin{itemize}
+      \item beforeValidate() (lowercase all hashes)
+      \item validate() (check hash format)
+      \item afterValidate() (we never use it   \item could be interesting if we ever validated without saving)
+      \item beforeSave() (purge existing correlations for an attribute)
+      \item afterSave() (create new correlations for an attribute / zmq)
+    \end{itemize}
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Models - hooking pipeline (delete/read)}
+  \begin{itemize}
+    \item Hooks for deletions
+    \begin{itemize}
+      \item beforeDelete() (purge correlations for an attribute)
+      \item afterDelete() (zmq)
+    \end{itemize}
+    \item Hooks for retrieving data
+    \begin{itemize}
+      \item beforeFind() (modify the find parameters before execution, we don't use it)
+      \item afterFind() (json decode json fields)
+      \end{itemize}
+    \end{itemize}
+  \end{frame}
+
+\begin{frame}
+  \frametitle{Models - misc}
+  \begin{itemize}
+    \item code to handle version upgrades contained in AppModel
+    \item generic cleanup/data migration tools
+    \item centralised redis/pubsub handlers
+    \item (Show example of adding an attribute with trace)
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Views - scope and structure}
+  \begin{itemize}
+    \item templates for views
+    \item layouts
+    \item reusable template code: elements
+    \begin{itemize}
+      \item attribute list, rows (if reused)
+    \end{itemize}
+    \item reusable code: helpers
+    \begin{itemize}
+      \item commandhelper (for discussion boards), highlighter for searches, tag colour helper
+    \end{itemize}
+    \item views per controller
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Views - Types of views and helpers}
+  \begin{itemize}
+    \item ajax views vs normal views
+    \item data views vs normal views vs serialisation in the controller
+    \item sanitisation h()
+    \item creating forms
+    \begin{itemize}
+      \item sanitisation
+      \item CSRF
+    \end{itemize}
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Distribution}
+  \begin{itemize}
+    \item algorithm for checking if a user has access to an attribute
+    \item creator vs owner organisation
+    \item distribution levels and inheritance (events -> objects -> attributes)
+    \item shorthand inherit level
+    \item sharing groups (org list, instance list)
+    \item correlation distribution
+    \item algorithms for safe data fetching (fetchEvents(), fetchAttributes(),...)
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{Testing your code}
+  \begin{itemize}
+    \item funtional testing
+    \item impact scope
+    \begin{itemize}
+      \item view code changes: only impacts request type based views
+      \item controller code changes: Should only affect given action
+      \item model code changes: can have impact on entire application
+      \item lib changes: can have affect on the entire application
+    \end{itemize}
+    \item Don't forget: queryACL, change querystring
+  \end{itemize}
+\end{frame}
+
+

a.1-devintro/slide.tex

@@ -0,0 +1,27 @@
+\documentclass{beamer}
+\usetheme[numbering=progressbar]{focus}
+\definecolor{main}{RGB}{47, 161, 219}
+\definecolor{textcolor}{RGB}{128, 128, 128}
+\definecolor{background}{RGB}{240, 247, 255}
+
+\usepackage[utf8]{inputenc}
+\usepackage{tikz}
+\usepackage{listings}
+\usepackage{adjustbox}
+\usetikzlibrary{positioning}
+\usetikzlibrary{shapes,arrows}
+%\usepackage[T1]{fontenc}
+%\usepackage[scaled]{beramono}
+
+\author{\small{\input{../includes/authors.txt}}}
+
+\title{MISP core development crash course}
+\subtitle{How I learned to stop worrying and love the PHP}
+\institute{\includegraphics[scale=0.5]{misplogo.pdf}}
+\titlegraphic{\includegraphics[scale=0.85]{misp.pdf}}
+
+\date{\input{../includes/location.txt}}
+\begin{document}
+\include{content}
+\end{document}
+

build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 
-slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing")
+slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing" "a.1-devintro")
 mkdir output
 export TEXINPUTS=::`pwd`/themes/
 echo ${TEXINPUTS}