diff --git a/a.6-forensic/content.tex b/a.6-forensic/content.tex new file mode 100644 index 0000000..8c14a92 --- /dev/null +++ b/a.6-forensic/content.tex @@ -0,0 +1,72 @@ +% DO NOT COMPILE THIS FILE DIRECTLY! +% This is included by the other .tex files. + +\begin{frame}[t,plain] +\titlepage +\end{frame} + +\begin{frame} +\frametitle{DFIR and MISP digital evidences} + \begin{itemize} + \item {\bf Share analyses and reports} of digital forensic evidences. + \item {\bf Propose changes} to existing analyses or reports. + \item Extending existing events with additional evidences for local or use in limited distribution sharing (sharing can be defined at event level or attribute level). + \item {\bf Evaluate correlations}\footnote{MISP has a flexible correlation engine which can correlate on 1-to-1 value matches, but also on fuzzy hashing (e.g. ssdeep) or CIDR block matching.} of evidences against external or local attributes. + \item {\bf Report sightings} such as false-positive or true-positive (e.g. a partner/analyst has seen a similar indicator). + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Benefits of using MISP} +\begin{itemize} + \item LE can leverage the long-standing experience in information sharing and {\bf bridge their use-cases} with MISP's information sharing mechanisms. + \item {\bf Accessing existing MISP information sharing communities} by receiving actionable information from CSIRT/CERT networks or security researchers. + \item {\bf Bridging LE communities with other communities}. Sharing groups can be created (and managed) cross-sectors to support specific use-cases. + \item The {\bf MISP standard} is a flexible format which can be extended by users using the MISP platform. A MISP object template can be created in under 30 minutes, allowing users to rapidly share information using their own data-models with existing communities. +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Challenges and implementations} + \begin{itemize} + \item Standard sharing mechanism for forensic cases + \begin{itemize} + \item MISP allows for the efficient \textbf{collaborative} analysis of digital evidences + \item Correlation on certain attributes + \end{itemize} + \item Importing disk images and file system data activity (\texttt{Mactime}) + \begin{itemize} + \item Development of an adaptable import tool: From Mactime to MISP \texttt{Mactime object} + \end{itemize} + \item Create, modify and visualise the timeline of events + \begin{itemize} + \item Development of a flexible timeline system at the event level + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Forensic import (MISP 2.4.98)} + \centering + \includegraphics[scale=0.3]{pics/import.png} + \includegraphics[scale=0.3]{pics/import-table.png} + + \begin{itemize} + \item Possibility to import \textbf{Mactime} files [done] + \item Pick only relevant files [done] + \item \texttt{MISPObject} will be created [done] + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Data visualization (MISP 2.4.100/101?)} + \includegraphics[width=1.0\linewidth]{pics/timeline.png} + \begin{itemize} + \item View: start-date only, spanning and search [dev-branch] + \item Manipulate: Edit, Drag and Expand [dev-branch] + \item Others: Timezone support [dev-branch] + \end{itemize} + + \vspace{0.3cm} + $\rightarrow$ For now [dev-branch], supports up to \textbf{micro-seconds} in the database and up to \textbf{milliseconds} in the web interface. +\end{frame} diff --git a/a.6-forensic/logo-circl.pdf b/a.6-forensic/logo-circl.pdf new file mode 100644 index 0000000..62c9239 Binary files /dev/null and b/a.6-forensic/logo-circl.pdf differ diff --git a/a.6-forensic/misp.pdf b/a.6-forensic/misp.pdf new file mode 100644 index 0000000..f7a3f9d Binary files /dev/null and b/a.6-forensic/misp.pdf differ diff --git a/a.6-forensic/pics/import-table.png b/a.6-forensic/pics/import-table.png new file mode 100644 index 0000000..805d9c5 Binary files /dev/null and b/a.6-forensic/pics/import-table.png differ diff --git a/a.6-forensic/pics/import.png b/a.6-forensic/pics/import.png new file mode 100644 index 0000000..2818a20 Binary files /dev/null and b/a.6-forensic/pics/import.png differ diff --git a/a.6-forensic/pics/mactimeObject.png b/a.6-forensic/pics/mactimeObject.png new file mode 100644 index 0000000..31881c3 Binary files /dev/null and b/a.6-forensic/pics/mactimeObject.png differ diff --git a/a.6-forensic/pics/timeline.png b/a.6-forensic/pics/timeline.png new file mode 100644 index 0000000..d903f8c Binary files /dev/null and b/a.6-forensic/pics/timeline.png differ diff --git a/a.6-forensic/slide.tex b/a.6-forensic/slide.tex new file mode 100644 index 0000000..ddd99ae --- /dev/null +++ b/a.6-forensic/slide.tex @@ -0,0 +1,145 @@ +\documentclass{beamer} +\usetheme[numbering=progressbar]{focus} +\definecolor{main}{RGB}{47, 161, 219} +\definecolor{textcolor}{RGB}{128, 128, 128} +\definecolor{background}{RGB}{240, 247, 255} + + +\usepackage[utf8x]{inputenc} +\usepackage{listings} +\usepackage{soul} +\usepackage{siunitx} +\usepackage{booktabs} +%\lstset{ +% backgroundcolor=\color{white}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor} +% basicstyle=\footnotesize, % the size of the fonts that are used for the code +% breakatwhitespace=false +%} + +\usepackage{tikz} +\usetikzlibrary{shapes,snakes,automata,positioning} + +\usepackage{xcolor} +\usepackage{colortbl} +\definecolor{mygreen}{rgb}{0,0.6,0} +\definecolor{mygreen2}{rgb}{0,0.56,0.16} +\definecolor{myred}{rgb}{0.6,0.066,0.066} +\definecolor{redCIRCL}{RGB}{213,43,30} +\definecolor{mygray}{rgb}{0.5,0.5,0.5} +\definecolor{mymauve}{rgb}{0.58,0,0.82} +\definecolor{mygray}{gray}{0.9} +\definecolor{mywhite}{rgb}{1,1,1} +\definecolor{myblack}{rgb}{0,0,0} +\definecolor{mybeige}{HTML}{eeeeee} +%\usepackage{tcolorbox} +\usepackage[listings]{tcolorbox} +\tcbuselibrary{listings} + +\lstdefinestyle{code}{ % + backgroundcolor=\color{mybeige}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument + basicstyle=\footnotesize\ttfamily, % the size of the fonts that are used for the code + breakatwhitespace=false, % sets if automatic breaks should only happen at whitespace + breaklines=true, % sets automatic line breaking + captionpos=b, % sets the caption-position to bottom + commentstyle=\color{mygreen}, % comment style + deletekeywords={...}, % if you want to delete keywords from the given language + escapeinside={\%*}{*)}, % if you want to add LaTeX within your code + extendedchars=true, % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8 + frame=single, % adds a frame around the code + keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible) + keywordstyle=\color{blue}, % keyword style + language=Python, % the language of the code + morekeywords={*,...}, % if you want to add more keywords to the set + numbers=left, % where to put the line-numbers; possible values are (none, left, right) + numbersep=5pt, % how far the line-numbers are from the code + numberstyle=\tiny\color{myblack}, % the style that is used for the line-numbers + rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here)) + showspaces=false, % show spaces everywhere adding particular underscores; it overrides 'showstringspaces' + showstringspaces=false, % underline spaces within strings only + showtabs=false, % show tabs within strings adding particular underscores + stepnumber=1, % the step between two line-numbers. If it's 1, each line will be numbered + stringstyle=\color{mymauve}, % string literal style + tabsize=2, % sets default tabsize to 2 spaces + title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title +} +\lstdefinestyle{bash}{ % + backgroundcolor=\color{black!85}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument + basicstyle=\footnotesize\color{mywhite}, % the size of the fonts that are used for the code + breakatwhitespace=false, % sets if automatic breaks should only happen at whitespace + breaklines=true, % sets automatic line breaking + captionpos=b, % sets the caption-position to bottom + commentstyle=\color{mygreen}, % comment style + deletekeywords={...}, % if you want to delete keywords from the given language + escapeinside={\%*}{*)}, % if you want to add LaTeX within your code + extendedchars=true, % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8 + frame=single % adds a frame around the code + keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible) + keywordstyle=\color{white}\bfseries, % keyword style + language=bash, % the language of the code + morekeywords={*,$,git, clone,... }, % if you want to add more keywords to the set + numbers=left, % where to put the line-numbers; possible values are (none, left, right) + numbersep=5pt, % how far the line-numbers are from the code + numberstyle=\tiny\color{mywhite}, % the style that is used for the line-numbers + rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here)) + showspaces=false, % show spaces everywhere adding particular underscores; it overrides 'showstringspaces' + showstringspaces=false, % underline spaces within strings only + showtabs=false, % show tabs within strings adding particular underscores + stepnumber=1, % the step between two line-numbers. If it's 1, each line will be numbered + stringstyle=\color{mymauve}, % string literal style + tabsize=2, % sets default tabsize to 2 spaces + title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title +} +\lstdefinestyle{default}{ % + backgroundcolor=\color{white}, % choose the background color; you must add \usepackage{color} or \usepackage{xcolor}; should come as last argument + basicstyle=\footnotesize\color{black}, % the size of the fonts that are used for the code + breakatwhitespace=false, % sets if automatic breaks should only happen at whitespace + breaklines=true, % sets automatic line breaking + captionpos=b, % sets the caption-position to bottom + commentstyle=\color{mygreen}, % comment style + deletekeywords={...}, % if you want to delete keywords from the given language + escapeinside={\%*}{*)}, % if you want to add LaTeX within your code + extendedchars=true, % lets you use non-ASCII characters; for 8-bits encodings only, does not work with UTF-8 + frame=single % adds a frame around the code + keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible) + keywordstyle=\color{white}\bfseries, % keyword style + language=bash, % the language of the code + morekeywords={*,$,git, clone,... }, % if you want to add more keywords to the set + numbers=left, % where to put the line-numbers; possible values are (none, left, right) + numbersep=5pt, % how far the line-numbers are from the code + numberstyle=\tiny\color{black}, % the style that is used for the line-numbers + rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here)) + showspaces=false, % show spaces everywhere adding particular underscores; it overrides 'showstringspaces' + showstringspaces=false, % underline spaces within strings only + showtabs=false, % show tabs within strings adding particular underscores + stepnumber=1, % the step between two line-numbers. If it's 1, each line will be numbered + stringstyle=\color{mymauve}, % string literal style + tabsize=2, % sets default tabsize to 2 spaces + title=\lstname % show the filename of files included with \lstinputlisting; also try caption instead of title +} +\lstset{style=code} + + +\AtBeginSection[]{ + \begin{frame} + \vfill + \centering + \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title} + {\color{white} \usebeamerfont{title}\insertsectionhead}\par% + \end{beamercolorbox} + \vfill + \end{frame} +} + +\author{\small{Team CIRCL}} + +\title{Forensic support in MISP} +\subtitle{Tools and visualization to support digital forensic expert} +\institute{info@circl.lu} +\date{\today} +\titlegraphic{\includegraphics[scale=0.85]{misp.pdf}} + + +\begin{document} +\include{content} +\end{document} + diff --git a/a.7-rest-API/content.tex b/a.7-rest-API/content.tex new file mode 100644 index 0000000..7e4cd6c --- /dev/null +++ b/a.7-rest-API/content.tex @@ -0,0 +1,137 @@ +% DO NOT COMPILE THIS FILE DIRECTLY! +% This is included by the other .tex files. + +\begin{frame}[t,plain] +\titlepage +\end{frame} + +\begin{frame} +\frametitle{MISP API reworked} + \begin{itemize} + \item The MISP API has grown gradually with a UI first design in many cases + \item Endpoints all solved specific issues with their own rulesets + \item Growth was organic - whenever the need to add a new functionality / filter popped up we've added it + \item Lead to frankenmonsters such as this: + \begin{adjustbox}{width=\textwidth,height=.5cm,keepaspectratio} + http://localhost:5000/events/csv/download/false/false/tag1\&\&tag2\&\&\!tag3/Network\%20activity/domain + \end{adjustbox} + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Goals we've set for ourselves} + \begin{itemize} + \item Open up every functionality in MISP available via the UI to the API + \item Including ones related to {\bf instance management} + \item APIs that expect input objects for data creation should be {\bf self-describing} + \item {\bf URL parameters should be discouraged}, but still usable by legacy tools (deprecation) + \item APIs should be heavily {\bf tested} (Raphael Vinot's exhaustive test suite in PyMISP) + \item Largest focus on Export APIs + \end{itemize} +\end{frame} + + +\begin{frame} +\frametitle{Export API's reimagined} + \begin{itemize} + \item Scrapped all existing type specific APIs ({\bf deprecated}, documentation moved to legacy, still available) + \item {\bf Single entry point} - all export APIs baked into restSearch + \item Queries consist of a combination of: + \begin{itemize} + \item {\bf Scope} (Event, Attribute, Sighting, more coming in the future) + \item {\bf Filter parameters} - passed via JSON objects, url parameters (key value or ordered list) + \item A {\bf return format} + \end{itemize} + \item Everything that we could do before the rework we should be able to accomplish after the rework + \item Under the hood now also used by the UI search and exports + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Export API's reimagined} + \begin{itemize} + \item One of our largest issues solved: {\bf pagination} + \begin{itemize} + \item {\bf Scope specific} pagination (number of events, attributes, etc) + \item Simply control it via the framework friendly {\bf page / limit} parameters + \item Alternatively, use the improved {\bf time based controls} (timestamp, publish\_timestamp windows) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Performance tuning} + \begin{itemize} + \item Single execution with subqueries + \item Internal pagination {\bf aligned with memory limits} + \begin{itemize} + \item Probing of available memory for the current process + \item {\bf Chunking of the query results} to fit in object specific memory envelopes + \item Constructing export set on disk in chunks has slashed memory usage considerably + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Designing tools that use the APIs can be complex, but there's help} + \begin{itemize} + \item The result of our own frustration + \item Built in {\bf ReST client} with templating + \item Build queries in a simple interface, automatically set URLs, headers, etc + \item Uses the self documentation of APIs + \item Export your queries as {\bf cURL or Python scripts} + \item Built in testing tools (performance measurements, result parsers) + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Why is the search API receiving so much focus?} + \begin{itemize} + \item The {\bf maturity} of the communities and threat intel sharing at large has improved + \item We are sharing more + \item Most importantly: we are sharing {\bf more context} along with technical indicators + \item This allows us to {\bf manage our data more accuractely} before feeding them to our protective tools + \item Different contexts (APT targeting me? Persisting techniques?) - lifecycle management + \item Use several queries / boolean operators to select the slice of data most relevant for the task + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{CLI tools for the CLI God, automation for the automation throne} + \begin{itemize} + \item Open up commonly used system management tasks to the CLI + \begin{itemize} + \item sync servers/feeds + \item caching feeds + \item Password resets + \item Server settings + \item Bruteforce protection resets + \item Enrichment + \end{itemize} + \item Goal was also to move away from the often malfunctioning scheduler and have cron friendly CLI scripts + \end{itemize} +\end{frame} + +\begin{frame} +\frametitle{So what does all of this look like in practice?} + \begin{center} + \item Demo time! + \end{center} +\end{frame} + +\begin{frame} +\frametitle{Plans for the future} + \begin{itemize} + \item Add export modules to the restSearch API + \item Improve the query language to support some missing features (such as AND boolean operators) + \item Support for extended events via the restSearch API + \begin{itemize} + \item We're missing a framing structure in the export module system (how are a list of conversions encapsulated and delimited?) + \item Proof of concept of the system implemented by Christian Studer already works using the STIX / STIX2 export subsystems + \item Would open us up to simple customiseable search APIs + \end{itemize} + \item Open up search APIs to other scopes (objects, users, organisations, proposals, feeds, galaxies, taxonomies) + \item Sami Mokaddem designing an extensive GUI overlay for easy query construction + \end{itemize} +\end{frame} + diff --git a/a.7-rest-API/logo-circl.pdf b/a.7-rest-API/logo-circl.pdf new file mode 100644 index 0000000..62c9239 Binary files /dev/null and b/a.7-rest-API/logo-circl.pdf differ diff --git a/a.7-rest-API/misplogo.pdf b/a.7-rest-API/misplogo.pdf new file mode 100755 index 0000000..60da568 Binary files /dev/null and b/a.7-rest-API/misplogo.pdf differ diff --git a/a.7-rest-API/pics/param-ui.png b/a.7-rest-API/pics/param-ui.png new file mode 100644 index 0000000..7c5ff8a Binary files /dev/null and b/a.7-rest-API/pics/param-ui.png differ diff --git a/a.7-rest-API/slide.tex b/a.7-rest-API/slide.tex new file mode 100644 index 0000000..3ed547a --- /dev/null +++ b/a.7-rest-API/slide.tex @@ -0,0 +1,26 @@ +\documentclass{beamer} +\usetheme[numbering=progressbar]{focus} +\definecolor{main}{RGB}{47, 161, 219} +\definecolor{textcolor}{RGB}{128, 128, 128} +\definecolor{background}{RGB}{240, 247, 255} + + +\usepackage[utf8]{inputenc} +\usepackage{tikz} +\usepackage{listings} +\usepackage{adjustbox} +\usetikzlibrary{positioning} +\usetikzlibrary{shapes,arrows} +%\usepackage[T1]{fontenc} +%\usepackage[scaled]{beramono} + +\author{\small{\input{../includes/authors.txt}}} + +\title{MISP restSearch API} +\subtitle{An easy way to query, add and update your threat intelligence in MISP} +\institute{\includegraphics[scale=0.5]{misplogo.pdf}} +\date{\input{../includes/location.txt}} +\begin{document} +\include{content} +\end{document} + diff --git a/build.sh b/build.sh index bd2acf4..3891400 100644 --- a/build.sh +++ b/build.sh @@ -1,7 +1,7 @@ #!/bin/bash # -slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing" "a.1-devintro" "a.2-pymisp" "a.3-misp-feed" "a.4-best-practices") +slidedecks=("0-misp-introduction-to-information-sharing" "1-misp-usage" "1.2-misp-integration" "1.1-misp-viper-integration" "1.2.1-misp-integration-mail2misp" "2-misp-administration" "3-misp-taxonomy-tagging" "3.1-misp-modules" "3.2-misp-galaxy" "3.3-misp-object-template" "6.0-misp-dashboard" "a.0-contributing" "a.1-devintro" "a.2-pymisp" "a.3-misp-feed" "a.4-best-practices" "a.5-decaying-indicators" "a.6-forensic" "a.7-rest-API") mkdir output export TEXINPUTS=::`pwd`/themes/ echo ${TEXINPUTS}