new: [a6/7] updated

a.6-forensic/content.tex

@@ -0,0 +1,72 @@
+% DO NOT COMPILE THIS FILE DIRECTLY!
+% This is included by the other .tex files.
+
+\begin{frame}[t,plain]
+\titlepage
+\end{frame}
+
+\begin{frame}
+\frametitle{DFIR and MISP digital evidences}
+        \begin{itemize}
+                \item {\bf Share analyses and reports} of digital forensic evidences.
+                \item {\bf Propose changes} to existing analyses or reports.
+                \item Extending existing events with additional evidences for local or use in limited distribution sharing (sharing can be defined at event level or attribute level).
+                \item {\bf Evaluate correlations}\footnote{MISP has a flexible correlation engine which can correlate on 1-to-1 value matches, but also on fuzzy hashing (e.g. ssdeep) or CIDR block matching.} of evidences against external or local attributes.
+                \item {\bf Report sightings} such as false-positive or true-positive (e.g. a partner/analyst has seen a similar indicator).
+        \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Benefits of using MISP}
+\begin{itemize}
+        \item  LE can leverage the long-standing experience in information sharing and {\bf bridge their use-cases} with MISP's information sharing mechanisms.
+        \item {\bf Accessing existing MISP information sharing communities} by receiving actionable information from CSIRT/CERT networks or security researchers.
+        \item {\bf Bridging LE communities with other communities}. Sharing groups can be created (and managed) cross-sectors to support specific use-cases.
+        \item The {\bf MISP standard} is a flexible format which can be extended by users using the MISP platform. A MISP object template can be created in under 30 minutes, allowing users to rapidly share information using their own data-models with existing communities.
+\end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Challenges and implementations}
+    \begin{itemize}
+        \item Standard sharing mechanism for forensic cases
+            \begin{itemize}
+                \item MISP allows for the efficient \textbf{collaborative} analysis of digital evidences
+                \item Correlation on certain attributes
+            \end{itemize}
+        \item Importing disk images and file system data activity (\texttt{Mactime})
+            \begin{itemize}
+                \item Development of an adaptable import tool: From Mactime to MISP \texttt{Mactime object}
+            \end{itemize}
+        \item Create, modify and visualise the timeline of events
+            \begin{itemize}
+                \item Development of a flexible timeline system at the event level
+            \end{itemize}
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+        \frametitle{Forensic import (MISP 2.4.98)}
+    \centering
+    \includegraphics[scale=0.3]{pics/import.png}
+    \includegraphics[scale=0.3]{pics/import-table.png}
+    
+    \begin{itemize}
+        \item Possibility to import \textbf{Mactime} files [done]
+        \item Pick only relevant files [done]
+        \item \texttt{MISPObject} will be created [done]
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+        \frametitle{Data visualization (MISP 2.4.100/101?)}
+    \includegraphics[width=1.0\linewidth]{pics/timeline.png}
+    \begin{itemize}
+        \item View: start-date only, spanning and search [dev-branch]
+        \item Manipulate: Edit, Drag and Expand [dev-branch]
+        \item Others: Timezone support [dev-branch]
+    \end{itemize}
+
+    \vspace{0.3cm}
+    $\rightarrow$ For now [dev-branch], supports up to \textbf{micro-seconds} in the database and up to \textbf{milliseconds} in the web interface.
+\end{frame}

a.6-forensic/slide.tex

@@ -0,0 +1,145 @@
+\documentclass{beamer}
+\usetheme[numbering=progressbar]{focus}
+\definecolor{main}{RGB}{47, 161, 219}
+\definecolor{textcolor}{RGB}{128, 128, 128}
+\definecolor{background}{RGB}{240, 247, 255}
+
+
+\usepackage[utf8x]{inputenc}
+\usepackage{listings}
+\usepackage{soul}
+\usepackage{siunitx}
+\usepackage{booktabs}
+%\lstset{ 
+%  backgroundcolor=\color{white},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}
+%  basicstyle=\footnotesize,        % the size of the fonts that are used for the code
+%  breakatwhitespace=false
+%}
+
+\usepackage{tikz}
+\usetikzlibrary{shapes,snakes,automata,positioning}
+
+\usepackage{xcolor}
+\usepackage{colortbl}
+\definecolor{mygreen}{rgb}{0,0.6,0}
+\definecolor{mygreen2}{rgb}{0,0.56,0.16}
+\definecolor{myred}{rgb}{0.6,0.066,0.066}
+\definecolor{redCIRCL}{RGB}{213,43,30}
+\definecolor{mygray}{rgb}{0.5,0.5,0.5}
+\definecolor{mymauve}{rgb}{0.58,0,0.82}
+\definecolor{mygray}{gray}{0.9}
+\definecolor{mywhite}{rgb}{1,1,1}
+\definecolor{myblack}{rgb}{0,0,0}
+\definecolor{mybeige}{HTML}{eeeeee}
+%\usepackage{tcolorbox}
+\usepackage[listings]{tcolorbox}
+\tcbuselibrary{listings}
+
+\lstdefinestyle{code}{ %
+  backgroundcolor=\color{mybeige},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\ttfamily,        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single,	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{blue},       % keyword style
+  language=Python,                 % the language of the code
+  morekeywords={*,...},           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{myblack}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstdefinestyle{bash}{ %
+  backgroundcolor=\color{black!85},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\color{mywhite},        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{white}\bfseries,       % keyword style
+  language=bash,                 % the language of the code
+  morekeywords={*,$,git, clone,... },           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{mywhite}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstdefinestyle{default}{ %
+  backgroundcolor=\color{white},   % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument
+  basicstyle=\footnotesize\color{black},        % the size of the fonts that are used for the code
+  breakatwhitespace=false,         % sets if automatic breaks should only happen at whitespace
+  breaklines=true,                 % sets automatic line breaking
+  captionpos=b,                    % sets the caption-position to bottom
+  commentstyle=\color{mygreen},    % comment style
+  deletekeywords={...},            % if you want to delete keywords from the given language
+  escapeinside={\%*}{*)},          % if you want to add LaTeX within your code
+  extendedchars=true,              % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8
+  frame=single	                   % adds a frame around the code
+  keepspaces=true,                 % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
+  keywordstyle=\color{white}\bfseries,       % keyword style
+  language=bash,                 % the language of the code
+  morekeywords={*,$,git, clone,... },           % if you want to add more keywords to the set
+  numbers=left,                    % where to put the line-numbers; possible values are (none, left, right)
+  numbersep=5pt,                   % how far the line-numbers are from the code
+  numberstyle=\tiny\color{black}, % the style that is used for the line-numbers
+  rulecolor=\color{black},         % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
+  showspaces=false,                % show spaces everywhere adding particular underscores; it overrides 'showstringspaces'
+  showstringspaces=false,          % underline spaces within strings only
+  showtabs=false,                  % show tabs within strings adding particular underscores
+  stepnumber=1,                    % the step between two line-numbers. If it's 1, each line will be numbered
+  stringstyle=\color{mymauve},     % string literal style
+  tabsize=2,	                   % sets default tabsize to 2 spaces
+  title=\lstname                   % show the filename of files included with \lstinputlisting; also try caption instead of title
+}
+\lstset{style=code}
+
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+      {\color{white} \usebeamerfont{title}\insertsectionhead}\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\author{\small{Team CIRCL}}
+
+\title{Forensic support in MISP}
+\subtitle{Tools and visualization to support digital forensic expert}
+\institute{info@circl.lu}
+\date{\today}
+\titlegraphic{\includegraphics[scale=0.85]{misp.pdf}}
+
+
+\begin{document}
+\include{content}
+\end{document}
+

a.7-rest-API/content.tex

@@ -0,0 +1,137 @@
+% DO NOT COMPILE THIS FILE DIRECTLY!
+% This is included by the other .tex files.
+
+\begin{frame}[t,plain]
+\titlepage
+\end{frame}
+
+\begin{frame}
+\frametitle{MISP API reworked}
+    \begin{itemize}
+        \item The MISP API has grown gradually with a UI first design in many cases
+        \item Endpoints all solved specific issues with their own rulesets
+	\item Growth was organic - whenever the need to add a new functionality / filter popped up we've added it
+	\item Lead to frankenmonsters such as this:
+		\begin{adjustbox}{width=\textwidth,height=.5cm,keepaspectratio}
+		    http://localhost:5000/events/csv/download/false/false/tag1\&\&tag2\&\&\!tag3/Network\%20activity/domain
+		\end{adjustbox}
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Goals we've set for ourselves}
+    \begin{itemize}
+        \item Open up every functionality in MISP available via the UI to the API
+	\item Including ones related to {\bf instance management}
+	\item APIs that expect input objects for data creation should be {\bf self-describing}
+	\item {\bf URL parameters should be discouraged}, but still usable by legacy tools (deprecation)
+	\item APIs should be heavily {\bf tested} (Raphael Vinot's exhaustive test suite in PyMISP)
+	\item Largest focus on Export APIs
+    \end{itemize}
+\end{frame}
+
+
+\begin{frame}
+\frametitle{Export API's reimagined}
+    \begin{itemize}
+        \item Scrapped all existing type specific APIs ({\bf deprecated}, documentation moved to legacy, still available)
+	\item {\bf Single entry point} - all export APIs baked into restSearch
+	\item Queries consist of a combination of:
+	\begin{itemize}
+		\item {\bf Scope} (Event, Attribute, Sighting, more coming in the future)
+		\item {\bf Filter parameters} - passed via JSON objects, url parameters (key value or ordered list)
+		\item A {\bf return format}
+	\end{itemize}
+	\item Everything that we could do before the rework we should be able to accomplish after the rework
+	\item Under the hood now also used by the UI search and exports
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Export API's reimagined}
+    \begin{itemize}
+	\item One of our largest issues solved: {\bf pagination}
+	\begin{itemize}
+		\item {\bf Scope specific} pagination (number of events, attributes, etc)
+		\item Simply control it via the framework friendly {\bf page / limit} parameters
+		\item Alternatively, use the improved {\bf time based controls} (timestamp, publish\_timestamp windows)
+	\end{itemize}
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Performance tuning}
+    \begin{itemize}
+	\item Single execution with subqueries
+	\item Internal pagination {\bf aligned with memory limits}
+	\begin{itemize}
+		\item Probing of available memory for the current process
+		\item {\bf Chunking of the query results} to fit in object specific memory envelopes
+		\item Constructing export set on disk in chunks has slashed memory usage considerably
+	\end{itemize}
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Designing tools that use the APIs can be complex, but there's help}
+    \begin{itemize}
+	\item The result of our own frustration
+	\item Built in {\bf ReST client} with templating
+	\item Build queries in a simple interface, automatically set URLs, headers, etc
+	\item Uses the self documentation of APIs
+	\item Export your queries as {\bf cURL or Python scripts}
+	\item Built in testing tools (performance measurements, result parsers)
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{Why is the search API receiving so much focus?}
+    \begin{itemize}
+	\item The {\bf maturity} of the communities and threat intel sharing at large has improved
+	\item We are sharing more
+	\item Most importantly: we are sharing {\bf more context} along with technical indicators
+	\item This allows us to {\bf manage our data more accuractely} before feeding them to our protective tools
+	\item Different contexts (APT targeting me? Persisting techniques?) - lifecycle management
+	\item Use several queries / boolean operators to select the slice of data most relevant for the task
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{CLI tools for the CLI God, automation for the automation throne}
+    \begin{itemize}
+	\item Open up commonly used system management tasks to the CLI
+        \begin{itemize}
+            \item sync servers/feeds
+            \item caching feeds
+            \item Password resets
+            \item Server settings
+            \item Bruteforce protection resets
+            \item Enrichment
+        \end{itemize}
+        \item Goal was also to move away from the often malfunctioning scheduler and have cron friendly CLI scripts
+    \end{itemize}
+\end{frame}
+
+\begin{frame}
+\frametitle{So what does all of this look like in practice?}
+        \begin{center}
+        	\item Demo time!
+        \end{center}
+\end{frame}
+
+\begin{frame}
+\frametitle{Plans for the future}
+    \begin{itemize}
+	\item Add export modules to the restSearch API
+        \item Improve the query language to support some missing features (such as AND boolean operators)
+        \item Support for extended events via the restSearch API
+	\begin{itemize}
+	    \item We're missing a framing structure in the export module system (how are a list of conversions encapsulated and delimited?)
+            \item Proof of concept of the system implemented by Christian Studer already works using the STIX / STIX2 export subsystems
+	    \item Would open us up to simple customiseable search APIs
+	\end{itemize}
+	\item Open up search APIs to other scopes (objects, users, organisations, proposals, feeds, galaxies, taxonomies)
+	\item Sami Mokaddem designing an extensive GUI overlay for easy query construction
+    \end{itemize}
+\end{frame}
+

a.7-rest-API/slide.tex

@@ -0,0 +1,26 @@
+\documentclass{beamer}
+\usetheme[numbering=progressbar]{focus}
+\definecolor{main}{RGB}{47, 161, 219}
+\definecolor{textcolor}{RGB}{128, 128, 128}
+\definecolor{background}{RGB}{240, 247, 255}
+
+
+\usepackage[utf8]{inputenc}
+\usepackage{tikz}
+\usepackage{listings}
+\usepackage{adjustbox}
+\usetikzlibrary{positioning}
+\usetikzlibrary{shapes,arrows}
+%\usepackage[T1]{fontenc}
+%\usepackage[scaled]{beramono}
+
+\author{\small{\input{../includes/authors.txt}}}
+
+\title{MISP restSearch API}
+\subtitle{An easy way to query, add and update your threat intelligence in MISP}
+\institute{\includegraphics[scale=0.5]{misplogo.pdf}}
+\date{\input{../includes/location.txt}}
+\begin{document}
+\include{content}
+\end{document}
+

build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 
-slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing" "a.1-devintro" "a.2-pymisp" "a.3-misp-feed" "a.4-best-practices")
+slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing" "a.1-devintro" "a.2-pymisp" "a.3-misp-feed" "a.4-best-practices" "a.5-decaying-indicators" "a.6-forensic" "a.7-rest-API")
 mkdir output
 export TEXINPUTS=::`pwd`/themes/
 echo ${TEXINPUTS}