diff --git a/docs/workshop/0-introduction/d4-1.png b/docs/workshop/0-introduction/d4-1.png new file mode 100644 index 0000000..d46c31e Binary files /dev/null and b/docs/workshop/0-introduction/d4-1.png differ diff --git a/docs/workshop/0-introduction/d4-2.png b/docs/workshop/0-introduction/d4-2.png new file mode 100644 index 0000000..4c191d9 Binary files /dev/null and b/docs/workshop/0-introduction/d4-2.png differ diff --git a/docs/workshop/0-introduction/d4-3.png b/docs/workshop/0-introduction/d4-3.png new file mode 100644 index 0000000..8d435e3 Binary files /dev/null and b/docs/workshop/0-introduction/d4-3.png differ diff --git a/docs/workshop/0-introduction/d4-4.png b/docs/workshop/0-introduction/d4-4.png new file mode 100644 index 0000000..556aea3 Binary files /dev/null and b/docs/workshop/0-introduction/d4-4.png differ diff --git a/docs/workshop/0-introduction/d4-introduction.aux b/docs/workshop/0-introduction/d4-introduction.aux index 6ff2c4b..ba02f9b 100644 --- a/docs/workshop/0-introduction/d4-introduction.aux +++ b/docs/workshop/0-introduction/d4-introduction.aux @@ -58,8 +58,22 @@ \@writefile{nav}{\headcommand {\beamer@framepages {19}{19}}} \@writefile{nav}{\headcommand {\slideentry {0}{0}{20}{20/20}{}{0}}} \@writefile{nav}{\headcommand {\beamer@framepages {20}{20}}} -\@writefile{nav}{\headcommand {\beamer@partpages {1}{20}}} -\@writefile{nav}{\headcommand {\beamer@subsectionpages {1}{20}}} -\@writefile{nav}{\headcommand {\beamer@sectionpages {1}{20}}} -\@writefile{nav}{\headcommand {\beamer@documentpages {20}}} -\@writefile{nav}{\headcommand {\gdef \inserttotalframenumber {19}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{21}{21/21}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {21}{21}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{22}{22/22}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {22}{22}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{23}{23/23}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {23}{23}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{24}{24/24}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {24}{24}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{25}{25/25}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {25}{25}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{26}{26/26}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {26}{26}}} +\@writefile{nav}{\headcommand {\slideentry {0}{0}{27}{27/27}{}{0}}} +\@writefile{nav}{\headcommand {\beamer@framepages {27}{27}}} +\@writefile{nav}{\headcommand {\beamer@partpages {1}{27}}} +\@writefile{nav}{\headcommand {\beamer@subsectionpages {1}{27}}} +\@writefile{nav}{\headcommand {\beamer@sectionpages {1}{27}}} +\@writefile{nav}{\headcommand {\beamer@documentpages {27}}} +\@writefile{nav}{\headcommand {\gdef \inserttotalframenumber {26}}} diff --git a/docs/workshop/0-introduction/d4-introduction.log b/docs/workshop/0-introduction/d4-introduction.log index b9501eb..b4bb9c7 100644 --- a/docs/workshop/0-introduction/d4-introduction.log +++ b/docs/workshop/0-introduction/d4-introduction.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.13) 5 FEB 2019 22:18 +This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.13) 6 FEB 2019 23:37 entering extended mode restricted \write18 enabled. %&-line parsing enabled. @@ -1305,22 +1305,80 @@ LaTeX Font Info: Font shape `T1/FiraSans-OsF/m/it' will be ] [12 +] [13 + +] [14 + +] + +File: d4-4.png Graphic file (type png) + +Package pdftex.def Info: d4-4.png used on input line 191. +(pdftex.def) Requested size: 346.70036pt x 154.83179pt. + +Overfull \hbox (25.18411pt too wide) in paragraph at lines 191--191 +[][] + [] + +[15 + + <./d4-4.png>] + +File: d4-3.png Graphic file (type png) + +Package pdftex.def Info: d4-3.png used on input line 196. +(pdftex.def) Requested size: 343.08702pt x 170.1885pt. + +Overfull \hbox (21.57077pt too wide) in paragraph at lines 196--196 +[][] + [] + +[16 + + <./d4-3.png>] + +File: d4-1.png Graphic file (type png) + +Package pdftex.def Info: d4-1.png used on input line 201. +(pdftex.def) Requested size: 346.33904pt x 142.54642pt. + +Overfull \hbox (24.82278pt too wide) in paragraph at lines 201--201 +[][] + [] + +[17 + + <./d4-1.png>] + +File: d4-2.png Graphic file (type png) + +Package pdftex.def Info: d4-2.png used on input line 207. +(pdftex.def) Requested size: 343.99036pt x 165.67183pt. + +Overfull \hbox (22.4741pt too wide) in paragraph at lines 207--207 +[][] + [] + +[18 + + <./d4-2.png>] [19 + ] LaTeX Font Info: Font shape `T1/FiraSans-OsF/m/n' will be -(Font) scaled to size 12.0pt on input line 168. +(Font) scaled to size 12.0pt on input line 238. (/usr/share/texlive/texmf-dist/tex/latex/listings/lstlang1.sty File: lstlang1.sty 2015/06/04 1.6 listings language file ) (/usr/share/texlive/texmf-dist/tex/latex/listings/lstlang1.sty File: lstlang1.sty 2015/06/04 1.6 listings language file -) (./tcpdump.tex) [13 +) (./tcpdump.tex) [20 -] [14 +] [21 -] (./d4-client.tex) [15 +] (./d4-client.tex) [22 -] [16 +] [23 ] Missing character: There is no s in font nullfont! @@ -1333,17 +1391,17 @@ Missing character: There is no 0 in font nullfont! Missing character: There is no . in font nullfont! Missing character: There is no 4 in font nullfont! -Underfull \hbox (badness 1320) in paragraph at lines 258--258 +Underfull \hbox (badness 1320) in paragraph at lines 328--328 []|\T1/FiraSans-OsF/m/sc/14.4 Observing SYN floods at-tacks in backscat-ter [] -[17 +[24 -] [18 +] [25 -] (./flags.tex) [19 +] (./flags.tex) [26 -] (./pibs.tex) [20 +] (./pibs.tex) [27 ] \tf@nav=\write7 @@ -1355,19 +1413,19 @@ Underfull \hbox (badness 1320) in paragraph at lines 258--258 \tf@snm=\write9 \openout9 = `d4-introduction.snm'. -Package atveryend Info: Empty hook `BeforeClearDocument' on input line 321. -Package atveryend Info: Empty hook `AfterLastShipout' on input line 321. +Package atveryend Info: Empty hook `BeforeClearDocument' on input line 391. +Package atveryend Info: Empty hook `AfterLastShipout' on input line 391. (./d4-introduction.aux) -Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 321. -Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 321. +Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 391. +Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 391. Package rerunfilecheck Info: File `d4-introduction.out' has not changed. (rerunfilecheck) Checksum: D41D8CD98F00B204E9800998ECF8427E;0. ) Here is how much of TeX's memory you used: - 25619 strings out of 492982 - 515071 string characters out of 6134895 - 651506 words of memory out of 5000000 - 28542 multiletter control sequences out of 15000+600000 + 25671 strings out of 492982 + 515632 string characters out of 6134895 + 652036 words of memory out of 5000000 + 28576 multiletter control sequences out of 15000+600000 324948 words of font info for 86 fonts, out of 8000000 for 9000 1141 hyphenation exceptions out of 8191 71i,16n,99p,821b,1405s stack positions out of 5000i,500n,10000p,200000b,80000s @@ -1381,10 +1439,10 @@ ic/fira/FiraSans-Regular.pfb> -Output written on d4-introduction.pdf (20 pages, 602366 bytes). +Output written on d4-introduction.pdf (27 pages, 864063 bytes). PDF statistics: - 207 PDF objects out of 1000 (max. 8388607) - 158 compressed objects within 2 object streams - 41 named destinations out of 1000 (max. 500000) - 58 words of extra memory for PDF output out of 10000 (max. 10000000) + 254 PDF objects out of 1000 (max. 8388607) + 190 compressed objects within 2 object streams + 55 named destinations out of 1000 (max. 500000) + 78 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/docs/workshop/0-introduction/d4-introduction.nav b/docs/workshop/0-introduction/d4-introduction.nav index 4f11164..f3326f5 100644 --- a/docs/workshop/0-introduction/d4-introduction.nav +++ b/docs/workshop/0-introduction/d4-introduction.nav @@ -38,8 +38,22 @@ \headcommand {\beamer@framepages {19}{19}} \headcommand {\slideentry {0}{0}{20}{20/20}{}{0}} \headcommand {\beamer@framepages {20}{20}} -\headcommand {\beamer@partpages {1}{20}} -\headcommand {\beamer@subsectionpages {1}{20}} -\headcommand {\beamer@sectionpages {1}{20}} -\headcommand {\beamer@documentpages {20}} -\headcommand {\gdef \inserttotalframenumber {19}} +\headcommand {\slideentry {0}{0}{21}{21/21}{}{0}} +\headcommand {\beamer@framepages {21}{21}} +\headcommand {\slideentry {0}{0}{22}{22/22}{}{0}} +\headcommand {\beamer@framepages {22}{22}} +\headcommand {\slideentry {0}{0}{23}{23/23}{}{0}} +\headcommand {\beamer@framepages {23}{23}} +\headcommand {\slideentry {0}{0}{24}{24/24}{}{0}} +\headcommand {\beamer@framepages {24}{24}} +\headcommand {\slideentry {0}{0}{25}{25/25}{}{0}} +\headcommand {\beamer@framepages {25}{25}} +\headcommand {\slideentry {0}{0}{26}{26/26}{}{0}} +\headcommand {\beamer@framepages {26}{26}} +\headcommand {\slideentry {0}{0}{27}{27/27}{}{0}} +\headcommand {\beamer@framepages {27}{27}} +\headcommand {\beamer@partpages {1}{27}} +\headcommand {\beamer@subsectionpages {1}{27}} +\headcommand {\beamer@sectionpages {1}{27}} +\headcommand {\beamer@documentpages {27}} +\headcommand {\gdef \inserttotalframenumber {26}} diff --git a/docs/workshop/0-introduction/d4-introduction.pdf b/docs/workshop/0-introduction/d4-introduction.pdf index 8d05852..4609ba5 100644 Binary files a/docs/workshop/0-introduction/d4-introduction.pdf and b/docs/workshop/0-introduction/d4-introduction.pdf differ diff --git a/docs/workshop/0-introduction/d4-introduction.tex b/docs/workshop/0-introduction/d4-introduction.tex index 661db9b..8e6b694 100644 --- a/docs/workshop/0-introduction/d4-introduction.tex +++ b/docs/workshop/0-introduction/d4-introduction.tex @@ -137,10 +137,80 @@ \end{itemize} \end{frame} +\begin{frame} +\frametitle{D4 server handling} + +D4 server reconstructs the encapsulated stream from the D4 sensor and saves it in a Redis stream. + +\begin{itemize} +\item Support TLS connection +\item Unpack D4 header +\item Verify client secret key (HMAC) +\item check blocklist +\item Filter by types (Only accept one connection by type-UUID - except: type 254) +\item Discard incorrect data +\item Save data in a Redis Stream (unique for each session) +\end{itemize} +\end{frame} + +\begin{frame} + \frametitle{D4 server - worker handler} +After the stream is processed depending of the type using dedicated worker. + \begin{itemize} + \item Worker Manager (one by type) + \begin{itemize} + \item Check if a new session is created and valid data are saved in a Redis stream + \item Launch a new Worker for each session + \end{itemize} + \item Worker + \begin{itemize} + \item Get data from a stream + \item Reconstruct data + \item Save data on disk (with file rotation) + \item Save data in Redis. Create a queue for D4 Analyzer(s) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{D4 server - management interface} +The D4 server provides a web interface to manage D4 sensors, sessions and analyzer. + \begin{itemize} +\item Get Sensors status, errors and statistics +\item Get all connected sensors +\item Manage Sensors (stream size limit, secret key, ...) +\item Manage Accepted types +\item UUID/IP blocklist +\item Create Analyzer Queues + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{D4 server - main interface} + \includegraphics[scale=0.18]{d4-4.png} +\end{frame} + +\begin{frame} + \frametitle{D4 server - server management} + \includegraphics[scale=0.18]{d4-3.png} +\end{frame} + +\begin{frame} + \frametitle{D4 server - sensor overview} + \includegraphics[scale=0.18]{d4-1.png} +\end{frame} + + +\begin{frame} + \frametitle{D4 server - sensor management} + \includegraphics[scale=0.18]{d4-2.png} +\end{frame} + + \begin{frame} \frametitle{} -{\center Use-case: migrating a legacy network capture model into a D4 network sensor + {\center Use-case: migrating a legacy network capture model into a D4 network sensor } \end{frame} diff --git a/docs/workshop/0-introduction/server.notes b/docs/workshop/0-introduction/server.notes new file mode 100644 index 0000000..28d1448 --- /dev/null +++ b/docs/workshop/0-introduction/server.notes @@ -0,0 +1,31 @@ +Welcome to the d4-core wiki! + +## Server + +- Support TLS connection +- Unpack header +- Verify client secret key (HMAC) +- check blocklist +- Filter by types + (Only accept one connection by type-UUID - except: type 254) +- Discard incorrect data +- Save data in a Redis Stream (unique for each session) + +## Worker Manager (one by type) + +- Check if a new session is created and valid data are saved in a Redis stream +- Launch a new Worker for each session + +## Worker +- Get data for a stream +- Reconstruct data +- Save data on disk (with file rotation) +- Sava data in Redis. Create a queue for a D4-Analyzer + +## Flask server +- Get Sensors status, errors and statistics +- Get all connected sensors +- Manage Sensors (stream size limit, secret key, ...) +- Manage Accepted types +- UUID/IP blocklist +- Create Analyzer Queues