mirror of https://github.com/MISP/misp-training
418 lines
11 KiB
TeX
418 lines
11 KiB
TeX
% DO NOT COMPILE THIS FILE DIRECTLY!
|
|
% This is included by the other .tex files.
|
|
|
|
\begin{frame}
|
|
\titlepage
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Building a native restSearch export}
|
|
\begin{itemize}
|
|
\item Similar in scope to an {\bf export module} of the MISP modules system
|
|
\item Pros:
|
|
\begin{itemize}
|
|
\item Can be used for composited data coming from a {\bf filtered query}
|
|
\item Fast, {\bf native approach}
|
|
\item Can be built to support {\bf several scopes} (events, attributes, sightings)
|
|
\end{itemize}
|
|
\item Cons...
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Building a native restSearch export}
|
|
\begin{itemize}
|
|
\item Similar in scope to an {\bf export module} of the MISP modules system
|
|
\item Pros:
|
|
\begin{itemize}
|
|
\item Can be used for composited data coming from a {\bf filtered query}
|
|
\item Fast, {\bf native approach}
|
|
\item Can be built to support {\bf several scopes} (events, attributes, sightings)
|
|
\end{itemize}
|
|
\item Cons...
|
|
\end{itemize}
|
|
\begin{center}
|
|
\includegraphics[scale=0.5]{lolphp.jpg}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{So how does restSearch work?}
|
|
\begin{itemize}
|
|
\item Standardised way of collecting {\bf parameters}
|
|
\item Using the parameters, a loop is started to {\bf chunk and gradually build} our export data
|
|
\item The chunk size depends on memory envelopes
|
|
\item Each chunk is {\bf converted piece by piece}...
|
|
\item ... and subsequently are concatenated into a temporary file
|
|
\item Once no more elements are left, the file is sent in the response
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Where does the module system come into play?}
|
|
\begin{itemize}
|
|
\item The export modules handle 5 tasks:
|
|
\begin{itemize}
|
|
\item Pass {\bf meta-information} back to restSearch on the export format itself
|
|
\item Add a {\bf start segment} to the exported data
|
|
\item Do the actual {\bf conversion} from MISP's internal format to the desired export format
|
|
\item Provide a {\bf separator} for data chunks
|
|
\item Have a {\bf closing segment} for the returned data, based on the format\'s conventions
|
|
\end{itemize}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Our little training module: Nibbler, the ever hungry IDS/IPS}
|
|
\begin{center}
|
|
\includegraphics[scale=0.5]{nibbler.jpg}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Nibbler}
|
|
\begin{itemize}
|
|
\item Simplistic tool with its {\bf own proprietary format}
|
|
\item Meant to mimic a typical {\bf in-house tool}
|
|
\item Lightweight scope, for simplicity\'s sake
|
|
\item {\bf pipe separated values}
|
|
\item VALUE | TYPE | DESCRIPTION | REFERENCE | ACTION
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Nibbler format - caveats}
|
|
\begin{itemize}
|
|
\item Rules can be prepended by comments, each comment line starting with \#
|
|
\item Some characters have to be escaped in some custom, crazy ways
|
|
\begin{itemize}
|
|
\item linebreaks: \#\#LINEBREAK\#\#
|
|
\item commas: \#\#COMMA\#\#
|
|
\item pipes: \#\#PIPE\#\#
|
|
\end{itemize}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Nibbler format}
|
|
\begin{itemize}
|
|
\item {\bf Value}: The actual indicator value
|
|
\item {\bf Type}: The format of the indicator
|
|
\item {\bf Description}: A quick description for analysts investigating the alert, why is this relevant
|
|
\item {\bf Reference}: A backreference that the analyst can use to find out more about the alert
|
|
\item {\bf Action}: What should Nibbler do if it trips over the value?
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Supported types}
|
|
\begin{itemize}
|
|
\item IP
|
|
\item Domain
|
|
\item Hostname
|
|
\item MD5
|
|
\item SHA1
|
|
\item SHA256
|
|
\item Filename
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Supported values}
|
|
\begin{itemize}
|
|
\item ALERT - default behaviour, create an alert.
|
|
\item BLOCK - block the action outright. Only set if the tag nibbler:block is present
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Mapping the types to MISP}
|
|
\begin{itemize}
|
|
\item Though we have types to map from MISP, in some cases several types map to a Nibbler type
|
|
\item We've created a rough {\bf mapping} (this is probably the most difficult task) in advance
|
|
\item Some MISP types map to a Nibbler type directly
|
|
\item {\bf Composite} MISP types map to {\bf 2 Nibbler types} each
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{Mapping the types to MISP}
|
|
\begin{itemize}
|
|
\item ip-dst :: IP
|
|
\item ip-src :: IP
|
|
\item domain :: Domain
|
|
\item domain|ip :: Domain, IP
|
|
\item hostname :: Hostname
|
|
\item md5 :: MD5
|
|
\item sha1 :: SHA1
|
|
\item sha256 :: SHA256
|
|
\item filename|md5 :: Filename, MD5
|
|
\item malware-sample :: Filename, MD5
|
|
\item filename|sha1 :: Filename, SHA1
|
|
\item filename|sha256 :: Filename, SHA256
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Export module skeleton}
|
|
\begin{lstlisting}
|
|
<?php
|
|
class NibblerExport
|
|
{
|
|
public $additional_params = array();
|
|
public function handler(
|
|
$data, $options = array()
|
|
) {}
|
|
public function header(
|
|
$options = array()
|
|
) {}
|
|
public function footer() {}
|
|
public function separator() {}
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Additional parameters}
|
|
\begin{lstlisting}
|
|
public $additional_params = array(
|
|
'flatten' => 1
|
|
);
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Adding our mapping}
|
|
\begin{lstlisting}
|
|
private $__mapping = array(
|
|
'ip-dst' => 'IP',
|
|
'ip-src' => 'IP',
|
|
'domain' => 'Domain',
|
|
'domain|ip' => ['Domain', 'IP'],
|
|
'hostname' => 'Hostname',
|
|
'md5' => 'MD5',
|
|
'sha1' => 'SHA1',
|
|
'sha256' => 'SHA256',
|
|
'filename|md5' => array('Filename', 'MD5'),
|
|
'malware-sample' => array('Filename', 'MD5'),
|
|
'filename|sha1' => array('Filename', 'SHA1'),
|
|
'filename|sha256' => array('Filename', 'SHA256')
|
|
);
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Writing the start of the output}
|
|
\begin{lstlisting}
|
|
public function header($options = array())
|
|
{
|
|
return sprintf(
|
|
"# Nibbler rules generated by MISP at %s\n",
|
|
date('Y-m-d H:i:s')
|
|
);
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Footer function - how should the output end?}
|
|
\begin{lstlisting}
|
|
public function footer()
|
|
{
|
|
return "\n";
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{What separates the chunks?}
|
|
\begin{lstlisting}
|
|
public function separator()
|
|
{
|
|
return "\n";
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{The actual legwork, the handler}
|
|
\begin{lstlisting}[
|
|
basicstyle=\tiny
|
|
]
|
|
public function handler($data, $options = array())
|
|
{
|
|
if ($options['scope'] === 'Attribute') {
|
|
$data['Attribute']['AttributeTag'] = $data['AttributeTag'];
|
|
return $this->__convertAttribute($data['Attribute'], $data['Event']);
|
|
}
|
|
if ($options['scope'] === 'Event') {
|
|
$result = array();
|
|
foreach ($data['Attribute'] as $attribute) {
|
|
$temp = $this->__convertAttribute($attribute, $data['Event']);
|
|
if ($temp) $result[] = $temp;
|
|
}
|
|
return implode($this->separator(), $result);
|
|
}
|
|
return '';
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Building an optional internal converter function}
|
|
\begin{lstlisting}
|
|
private function __convertAttribute($attribute, $event)
|
|
{
|
|
if (empty($this->__mapping[$attribute['type']])) {
|
|
// mapping not found - invalid type for nibbler
|
|
return '';
|
|
}
|
|
if (is_array($this->__mapping[$attribute['type']])) {
|
|
// handle mappings for composites - slide
|
|
} else {
|
|
// handle simple mappings - slide
|
|
}
|
|
// return 1 or 2 lines, separated by separator()
|
|
return implode($this->separator(), $result);
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Handling the simple case}
|
|
\begin{lstlisting}
|
|
$result[] = sprintf(
|
|
'%s|%s|%s|%s|%s',
|
|
$this->__escapeSpecialChars($attribute['value']),
|
|
$this->__mapping[$attribute['type']],
|
|
$event['uuid'],
|
|
$this->__escapeSpecialChars($event['info']),
|
|
'ALERT'
|
|
);
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Handling the case for composites}
|
|
\begin{lstlisting}
|
|
$attribute['value'] = explode(
|
|
'|', $attribute['value']
|
|
);
|
|
foreach (array(0,1) as $part) {
|
|
$result[] = sprintf(
|
|
'%s|%s|%s|%s|%s',
|
|
$this->__escapeSpecialChars(
|
|
$attribute['value'][$part]
|
|
),
|
|
$this->__mapping[$attribute['type']][$part],
|
|
$event['uuid'],
|
|
$this->__escapeSpecialChars($event['info']),
|
|
'ALERT'
|
|
);
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Putting it together}
|
|
\begin{lstlisting}[
|
|
basicstyle=\tiny
|
|
]
|
|
private function __convertAttribute($attribute, $event) {
|
|
if (empty($this->__mapping[$attribute['type']])) return '';
|
|
$result = array();
|
|
$attributes = array();
|
|
if (is_array($this->__mapping[$attribute['type']])) {
|
|
$attribute['value'] = explode('|', $attribute['value']);
|
|
foreach (array(0,1) as $part) {
|
|
$result[] = sprintf(
|
|
'%s|%s|%s|%s|%s',
|
|
$this->__escapeSpecialChars($attribute['value'][$part]),
|
|
$this->__mapping[$attribute['type']][$part],
|
|
/events/view/ . $event['uuid'],
|
|
$this->__escapeSpecialChars($event['info']),
|
|
$this->__decideOnAction($attribute['AttributeTag'])
|
|
);
|
|
}
|
|
} else {
|
|
$result[] = sprintf(
|
|
'%s|%s|%s|%s|%s',
|
|
$this->__escapeSpecialChars($attribute['value']),
|
|
$this->__mapping[$attribute['type']],
|
|
/events/view/ . $event['uuid'],
|
|
$this->__escapeSpecialChars($event['info']),
|
|
$this->__decideOnAction($attribute['AttributeTag'])
|
|
);
|
|
}
|
|
return implode($this->separator(), $result);
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Adding the function that decides on the action}
|
|
\begin{lstlisting}
|
|
private function __decideOnAction($attributeTags)
|
|
{
|
|
foreach($attributeTags as $attributeTag) {
|
|
if (
|
|
$attributeTag['Tag']['name'] ===
|
|
'nibbler:block'
|
|
) {
|
|
return 'BLOCK';
|
|
}
|
|
}
|
|
return 'ALERT';
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Finalising the export module... The escaping function}
|
|
\begin{lstlisting}
|
|
private function __escapeSpecialChars($value)
|
|
{
|
|
$value = preg_replace(
|
|
"/\r|\n/", "##LINEBREAK##", $value
|
|
);
|
|
$value = preg_replace(
|
|
"/,/", "##COMMA##", $value
|
|
);
|
|
$value = preg_replace(
|
|
"/\|/", "##PIPE##", $value
|
|
);
|
|
return $value;
|
|
}
|
|
\end{lstlisting}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Modifying the MISP core to know about the export module}
|
|
\begin{itemize}
|
|
\item The {\bf models} that we are targeting by scope (Event, Attribute) {\bf need to be updated}
|
|
\item They are located in {\bf /var/www/MISP/app/Model/}
|
|
\item The global variable {\bf \$validFormats} houses all mappings
|
|
\item Simply add a new line such as the following:
|
|
\item 'nibbler' => array('nibbler', 'NibblerExport', 'nibbler')
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile]
|
|
\frametitle{Let us test the module!}
|
|
\begin{itemize}
|
|
\item Use the {\bf rest client} to test it conveniently
|
|
\item Both the event and attribute level restSearch function should work
|
|
\item Simply set the {\bf returnFormat} to nibbler, which should also show up as a valid export format
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}
|
|
\frametitle{REST client}
|
|
\begin{center}
|
|
\includegraphics[scale=0.5]{nibbler_rest_client.png}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
|