misp-training/a.9-restsearch-dev/content.tex

418 lines
11 KiB
TeX

% DO NOT COMPILE THIS FILE DIRECTLY!
% This is included by the other .tex files.
\begin{frame}
\titlepage
\end{frame}
\begin{frame}
\frametitle{Building a native restSearch export}
\begin{itemize}
\item Similar in scope to an {\bf export module} of the MISP modules system
\item Pros:
\begin{itemize}
\item Can be used for composited data coming from a {\bf filtered query}
\item Fast, {\bf native approach}
\item Can be built to support {\bf several scopes} (events, attributes, sightings)
\end{itemize}
\item Cons...
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Building a native restSearch export}
\begin{itemize}
\item Similar in scope to an {\bf export module} of the MISP modules system
\item Pros:
\begin{itemize}
\item Can be used for composited data coming from a {\bf filtered query}
\item Fast, {\bf native approach}
\item Can be built to support {\bf several scopes} (events, attributes, sightings)
\end{itemize}
\item Cons...
\end{itemize}
\begin{center}
\includegraphics[scale=0.5]{lolphp.jpg}
\end{center}
\end{frame}
\begin{frame}
\frametitle{So how does restSearch work?}
\begin{itemize}
\item Standardised way of collecting {\bf parameters}
\item Using the parameters, a loop is started to {\bf chunk and gradually build} our export data
\item The chunk size depends on memory envelopes
\item Each chunk is {\bf converted piece by piece}...
\item ... and subsequently are concatenated into a temporary file
\item Once no more elements are left, the file is sent in the response
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Where does the module system come into play?}
\begin{itemize}
\item The export modules handle 5 tasks:
\begin{itemize}
\item Pass {\bf meta-information} back to restSearch on the export format itself
\item Add a {\bf start segment} to the exported data
\item Do the actual {\bf conversion} from MISP's internal format to the desired export format
\item Provide a {\bf separator} for data chunks
\item Have a {\bf closing segment} for the returned data, based on the format\'s conventions
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Our little training module: Nibbler, the ever hungry IDS/IPS}
\begin{center}
\includegraphics[scale=0.5]{nibbler.jpg}
\end{center}
\end{frame}
\begin{frame}
\frametitle{Nibbler}
\begin{itemize}
\item Simplistic tool with its {\bf own proprietary format}
\item Meant to mimic a typical {\bf in-house tool}
\item Lightweight scope, for simplicity\'s sake
\item {\bf pipe separated values}
\item VALUE | TYPE | DESCRIPTION | REFERENCE | ACTION
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Nibbler format - caveats}
\begin{itemize}
\item Rules can be prepended by comments, each comment line starting with \#
\item Some characters have to be escaped in some custom, crazy ways
\begin{itemize}
\item linebreaks: \#\#LINEBREAK\#\#
\item commas: \#\#COMMA\#\#
\item pipes: \#\#PIPE\#\#
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Nibbler format}
\begin{itemize}
\item {\bf Value}: The actual indicator value
\item {\bf Type}: The format of the indicator
\item {\bf Description}: A quick description for analysts investigating the alert, why is this relevant
\item {\bf Reference}: A backreference that the analyst can use to find out more about the alert
\item {\bf Action}: What should Nibbler do if it trips over the value?
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Supported types}
\begin{itemize}
\item IP
\item Domain
\item Hostname
\item MD5
\item SHA1
\item SHA256
\item Filename
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Supported values}
\begin{itemize}
\item ALERT - default behaviour, create an alert.
\item BLOCK - block the action outright. Only set if the tag nibbler:block is present
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Mapping the types to MISP}
\begin{itemize}
\item Though we have types to map from MISP, in some cases several types map to a Nibbler type
\item We've created a rough {\bf mapping} (this is probably the most difficult task) in advance
\item Some MISP types map to a Nibbler type directly
\item {\bf Composite} MISP types map to {\bf 2 Nibbler types} each
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Mapping the types to MISP}
\begin{itemize}
\item ip-dst :: IP
\item ip-src :: IP
\item domain :: Domain
\item domain|ip :: Domain, IP
\item hostname :: Hostname
\item md5 :: MD5
\item sha1 :: SHA1
\item sha256 :: SHA256
\item filename|md5 :: Filename, MD5
\item malware-sample :: Filename, MD5
\item filename|sha1 :: Filename, SHA1
\item filename|sha256 :: Filename, SHA256
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Export module skeleton}
\begin{lstlisting}
<?php
class NibblerExport
{
public $additional_params = array();
public function handler(
$data, $options = array()
) {}
public function header(
$options = array()
) {}
public function footer() {}
public function separator() {}
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Additional parameters}
\begin{lstlisting}
public $additional_params = array(
'flatten' => 1
);
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Adding our mapping}
\begin{lstlisting}
private $__mapping = array(
'ip-dst' => 'IP',
'ip-src' => 'IP',
'domain' => 'Domain',
'domain|ip' => ['Domain', 'IP'],
'hostname' => 'Hostname',
'md5' => 'MD5',
'sha1' => 'SHA1',
'sha256' => 'SHA256',
'filename|md5' => array('Filename', 'MD5'),
'malware-sample' => array('Filename', 'MD5'),
'filename|sha1' => array('Filename', 'SHA1'),
'filename|sha256' => array('Filename', 'SHA256')
);
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Writing the start of the output}
\begin{lstlisting}
public function header($options = array())
{
return sprintf(
"# Nibbler rules generated by MISP at %s\n",
date('Y-m-d H:i:s')
);
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Footer function - how should the output end?}
\begin{lstlisting}
public function footer()
{
return "\n";
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{What separates the chunks?}
\begin{lstlisting}
public function separator()
{
return "\n";
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{The actual legwork, the handler}
\begin{lstlisting}[
basicstyle=\tiny
]
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
$data['Attribute']['AttributeTag'] = $data['AttributeTag'];
return $this->__convertAttribute($data['Attribute'], $data['Event']);
}
if ($options['scope'] === 'Event') {
$result = array();
foreach ($data['Attribute'] as $attribute) {
$temp = $this->__convertAttribute($attribute, $data['Event']);
if ($temp) $result[] = $temp;
}
return implode($this->separator(), $result);
}
return '';
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Building an optional internal converter function}
\begin{lstlisting}
private function __convertAttribute($attribute, $event)
{
if (empty($this->__mapping[$attribute['type']])) {
// mapping not found - invalid type for nibbler
return '';
}
if (is_array($this->__mapping[$attribute['type']])) {
// handle mappings for composites - slide
} else {
// handle simple mappings - slide
}
// return 1 or 2 lines, separated by separator()
return implode($this->separator(), $result);
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Handling the simple case}
\begin{lstlisting}
$result[] = sprintf(
'%s|%s|%s|%s|%s',
$this->__escapeSpecialChars($attribute['value']),
$this->__mapping[$attribute['type']],
$event['uuid'],
$this->__escapeSpecialChars($event['info']),
'ALERT'
);
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Handling the case for composites}
\begin{lstlisting}
$attribute['value'] = explode(
'|', $attribute['value']
);
foreach (array(0,1) as $part) {
$result[] = sprintf(
'%s|%s|%s|%s|%s',
$this->__escapeSpecialChars(
$attribute['value'][$part]
),
$this->__mapping[$attribute['type']][$part],
$event['uuid'],
$this->__escapeSpecialChars($event['info']),
'ALERT'
);
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Putting it together}
\begin{lstlisting}[
basicstyle=\tiny
]
private function __convertAttribute($attribute, $event) {
if (empty($this->__mapping[$attribute['type']])) return '';
$result = array();
$attributes = array();
if (is_array($this->__mapping[$attribute['type']])) {
$attribute['value'] = explode('|', $attribute['value']);
foreach (array(0,1) as $part) {
$result[] = sprintf(
'%s|%s|%s|%s|%s',
$this->__escapeSpecialChars($attribute['value'][$part]),
$this->__mapping[$attribute['type']][$part],
/events/view/ . $event['uuid'],
$this->__escapeSpecialChars($event['info']),
$this->__decideOnAction($attribute['AttributeTag'])
);
}
} else {
$result[] = sprintf(
'%s|%s|%s|%s|%s',
$this->__escapeSpecialChars($attribute['value']),
$this->__mapping[$attribute['type']],
/events/view/ . $event['uuid'],
$this->__escapeSpecialChars($event['info']),
$this->__decideOnAction($attribute['AttributeTag'])
);
}
return implode($this->separator(), $result);
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Adding the function that decides on the action}
\begin{lstlisting}
private function __decideOnAction($attributeTags)
{
foreach($attributeTags as $attributeTag) {
if (
$attributeTag['Tag']['name'] ===
'nibbler:block'
) {
return 'BLOCK';
}
}
return 'ALERT';
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Finalising the export module... The escaping function}
\begin{lstlisting}
private function __escapeSpecialChars($value)
{
$value = preg_replace(
"/\r|\n/", "##LINEBREAK##", $value
);
$value = preg_replace(
"/,/", "##COMMA##", $value
);
$value = preg_replace(
"/\|/", "##PIPE##", $value
);
return $value;
}
\end{lstlisting}
\end{frame}
\begin{frame}[fragile]
\frametitle{Modifying the MISP core to know about the export module}
\begin{itemize}
\item The {\bf models} that we are targeting by scope (Event, Attribute) {\bf need to be updated}
\item They are located in {\bf /var/www/MISP/app/Model/}
\item The global variable {\bf \$validFormats} houses all mappings
\item Simply add a new line such as the following:
\item 'nibbler' => array('nibbler', 'NibblerExport', 'nibbler')
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Let us test the module!}
\begin{itemize}
\item Use the {\bf rest client} to test it conveniently
\item Both the event and attribute level restSearch function should work
\item Simply set the {\bf returnFormat} to nibbler, which should also show up as a valid export format
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{REST client}
\begin{center}
\includegraphics[scale=0.5]{nibbler_rest_client.png}
\end{center}
\end{frame}