misp-training/a.9-restsearch-dev/content.tex

% DO NOT COMPILE THIS FILE DIRECTLY!
% This is included by the other .tex files.

\begin{frame}
\titlepage
\end{frame}

\begin{frame}
  \frametitle{Building a native restSearch export}
  \begin{itemize}
    \item Similar in scope to an {\bf export module} of the MISP modules system
    \item Pros:
    \begin{itemize}
      \item Can be used for composited data coming from a {\bf filtered query}
      \item Fast, {\bf native approach}
      \item Can be built to support {\bf several scopes} (events, attributes, sightings)
    \end{itemize}
    \item Cons...
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Building a native restSearch export}
  \begin{itemize}
        \item Similar in scope to an {\bf export module} of the MISP modules system
    \item Pros:
    \begin{itemize}
      \item Can be used for composited data coming from a {\bf filtered query}
      \item Fast, {\bf native approach}
      \item Can be built to support {\bf several scopes} (events, attributes, sightings)
    \end{itemize}
    \item Cons...
  \end{itemize}
  \begin{center}
    \includegraphics[scale=0.5]{lolphp.jpg}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{So how does restSearch work?}
  \begin{itemize}
    \item Standardised way of collecting {\bf parameters}
    \item Using the parameters, a loop is started to {\bf chunk and gradually build} our export data
    \item The chunk size depends on memory envelopes
    \item Each chunk is {\bf converted piece by piece}...
    \item ... and subsequently are concatenated into a temporary file
    \item Once no more elements are left, the file is sent in the response
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Where does the module system come into play?}
  \begin{itemize}
    \item The export modules handle 5 tasks:
    \begin{itemize}
      \item Pass {\bf meta-information} back to restSearch on the export format itself
      \item Add a {\bf start segment} to the exported data
      \item Do the actual {\bf conversion} from MISP's internal format to the desired export format
      \item Provide a {\bf separator} for data chunks
      \item Have a {\bf closing segment} for the returned data, based on the format\'s conventions
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Our little training module: Nibbler, the ever hungry IDS/IPS}
  \begin{center}
    \includegraphics[scale=0.5]{nibbler.jpg}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Nibbler}
  \begin{itemize}
    \item Simplistic tool with its {\bf own proprietary format}
    \item Meant to mimic a typical {\bf in-house tool}
    \item Lightweight scope, for simplicity\'s sake
    \item {\bf pipe separated values}
    \item VALUE | TYPE | DESCRIPTION | REFERENCE | ACTION
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Nibbler format - caveats}
  \begin{itemize}
    \item Rules can be prepended by comments, each comment line starting with \#
    \item Some characters have to be escaped in some custom, crazy ways
    \begin{itemize}
       \item linebreaks: \#\#LINEBREAK\#\#
       \item commas: \#\#COMMA\#\#
       \item pipes: \#\#PIPE\#\#
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Nibbler format}
  \begin{itemize}
    \item {\bf Value}: The actual indicator value
    \item {\bf Type}: The format of the indicator
    \item {\bf Description}: A quick description for analysts investigating the alert, why is this relevant
    \item {\bf Reference}: A backreference that the analyst can use to find out more about the alert
    \item {\bf Action}: What should Nibbler do if it trips over the value?
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Supported types}
  \begin{itemize}
    \item IP
    \item Domain
    \item Hostname
    \item MD5
    \item SHA1
    \item SHA256
    \item Filename
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Supported values}
  \begin{itemize}
    \item ALERT - default behaviour, create an alert.
    \item BLOCK - block the action outright. Only set if the tag nibbler:block is present
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Mapping the types to MISP}
  \begin{itemize}
    \item Though we have types to map from MISP, in some cases several types map to a Nibbler type
    \item We've created a rough {\bf mapping} (this is probably the most difficult task) in advance
    \item Some MISP types map to a Nibbler type directly
    \item {\bf Composite} MISP types map to {\bf 2 Nibbler types} each
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Mapping the types to MISP}
  \begin{itemize}
    \item ip-dst :: IP
    \item ip-src :: IP
    \item domain :: Domain
    \item domain|ip :: Domain, IP
    \item hostname :: Hostname
    \item md5 :: MD5
    \item sha1 :: SHA1
    \item sha256 :: SHA256
    \item filename|md5 :: Filename, MD5
    \item malware-sample :: Filename, MD5
    \item filename|sha1 :: Filename, SHA1
    \item filename|sha256 :: Filename, SHA256
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Export module skeleton}
  \begin{lstlisting}
<?php
class NibblerExport
{
    public $additional_params = array();
    public function handler(
        $data, $options = array()
    ) {}
    public function header(
        $options = array()
    ) {}
    public function footer() {}
    public function separator() {}
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Additional parameters}
  \begin{lstlisting}
public $additional_params = array(
    'flatten' => 1
);
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Adding our mapping}
  \begin{lstlisting}
private $__mapping = array(
  'ip-dst' => 'IP',
  'ip-src' => 'IP',
  'domain' => 'Domain',
  'domain|ip' => ['Domain', 'IP'],
  'hostname' => 'Hostname',
  'md5' => 'MD5',
  'sha1' => 'SHA1',
  'sha256' => 'SHA256',
  'filename|md5' => array('Filename', 'MD5'),
  'malware-sample' => array('Filename', 'MD5'),
  'filename|sha1' => array('Filename', 'SHA1'),
  'filename|sha256' => array('Filename', 'SHA256')
);
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Writing the start of the output}
  \begin{lstlisting}
public function header($options = array())
{
    return sprintf(
        "# Nibbler rules generated by MISP at %s\n",
        date('Y-m-d H:i:s')
    );
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Footer function - how should the output end?}
  \begin{lstlisting}
public function footer()
{
    return "\n";
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{What separates the chunks?}
  \begin{lstlisting}
public function separator()
{
    return "\n";
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{The actual legwork, the handler}
  \begin{lstlisting}[
    basicstyle=\tiny
  ]
public function handler($data, $options = array())
{
  if ($options['scope'] === 'Attribute') {
    $data['Attribute']['AttributeTag'] = $data['AttributeTag'];
    return $this->__convertAttribute($data['Attribute'], $data['Event']);
  }
  if ($options['scope'] === 'Event') {
    $result = array();
    foreach ($data['Attribute'] as $attribute) {
      $temp = $this->__convertAttribute($attribute, $data['Event']);
      if ($temp) $result[] = $temp;
    }
    return implode($this->separator(), $result);
  }
  return '';
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Building an optional internal converter function}
  \begin{lstlisting}
private function __convertAttribute($attribute, $event)
{
  if (empty($this->__mapping[$attribute['type']])) {
    // mapping not found - invalid type for nibbler
    return '';
  }
  if (is_array($this->__mapping[$attribute['type']])) {
    // handle mappings for composites - slide
  } else {
    // handle simple mappings - slide 
  }
  // return 1 or 2 lines, separated by separator()
  return implode($this->separator(), $result);
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Handling the simple case}
  \begin{lstlisting}
$result[] = sprintf(
  '%s|%s|%s|%s|%s',
  $this->__escapeSpecialChars($attribute['value']),
  $this->__mapping[$attribute['type']],
  $event['uuid'],
  $this->__escapeSpecialChars($event['info']),
  'ALERT'
);
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Handling the case for composites}
  \begin{lstlisting}
$attribute['value'] = explode(
  '|', $attribute['value']
);
foreach (array(0,1) as $part) {
  $result[] = sprintf(
    '%s|%s|%s|%s|%s',
    $this->__escapeSpecialChars(
      $attribute['value'][$part]
    ),
    $this->__mapping[$attribute['type']][$part],
    $event['uuid'],
    $this->__escapeSpecialChars($event['info']),
    'ALERT'
  );
}
  \end{lstlisting}
\end{frame}


\begin{frame}[fragile]
  \frametitle{Putting it together}
  \begin{lstlisting}[
    basicstyle=\tiny
  ]
private function __convertAttribute($attribute, $event) {
  if (empty($this->__mapping[$attribute['type']])) return '';
  $result = array();
  $attributes = array();
  if (is_array($this->__mapping[$attribute['type']])) {
    $attribute['value'] = explode('|', $attribute['value']);
    foreach (array(0,1) as $part) {
      $result[] = sprintf(
        '%s|%s|%s|%s|%s',
        $this->__escapeSpecialChars($attribute['value'][$part]),
        $this->__mapping[$attribute['type']][$part],
        /events/view/ . $event['uuid'],
        $this->__escapeSpecialChars($event['info']),
        $this->__decideOnAction($attribute['AttributeTag'])
      );
    }
  } else {
    $result[] = sprintf(
      '%s|%s|%s|%s|%s',
      $this->__escapeSpecialChars($attribute['value']),
      $this->__mapping[$attribute['type']],
      /events/view/ . $event['uuid'],
      $this->__escapeSpecialChars($event['info']),
      $this->__decideOnAction($attribute['AttributeTag'])
    );
  }
  return implode($this->separator(), $result);
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Adding the function that decides on the action}
  \begin{lstlisting}
private function __decideOnAction($attributeTags)
{
  foreach($attributeTags as $attributeTag) {
    if (
      $attributeTag['Tag']['name'] ===
        'nibbler:block'
    ) {
      return 'BLOCK';
    }
  }
  return 'ALERT';
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Finalising the export module... The escaping function}
  \begin{lstlisting}
private function __escapeSpecialChars($value)
{
  $value = preg_replace(
    "/\r|\n/", "##LINEBREAK##", $value
  );
  $value = preg_replace(
    "/,/", "##COMMA##", $value
  );
  $value = preg_replace(
    "/\|/", "##PIPE##", $value
  );
  return $value;
}
  \end{lstlisting}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Modifying the MISP core to know about the export module}
  \begin{itemize}
    \item The {\bf models} that we are targeting by scope (Event, Attribute) {\bf need to be updated}
    \item They are located in {\bf /var/www/MISP/app/Model/}
    \item The global variable {\bf \$validFormats} houses all mappings
    \item Simply add a new line such as the following:
    \item 'nibbler' => array('nibbler', 'NibblerExport', 'nibbler')
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Let us test the module!}
  \begin{itemize}
    \item Use the {\bf rest client} to test it conveniently
    \item Both the event and attribute level restSearch function should work
    \item Simply set the {\bf returnFormat} to nibbler, which should also show up as a valid export format
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{REST client}
  \begin{center}
    \includegraphics[scale=0.5]{nibbler_rest_client.png}
  \end{center}
\end{frame}