MISP
/
misp-training
mirror of https://github.com/MISP/misp-training
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Updated MISP modules slides

master
chrisr3d 2020-02-19 11:31:28 +01:00
parent ca0d506346
commit 4a3f73e6af
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
1 changed files with 342 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

664
3.1-misp-modules/content.tex
View File

@ -154,6 +154,134 @@
\end{adjustbox} \end{adjustbox}
\end{frame} \end{frame}
\begin{frame}
\frametitle{MISP modules - configuration in the UI}
\includegraphics[scale=0.50]{modules-integration.png}
\end{frame}
\begin{frame}
\frametitle{MISP modules - How it's integrated in the UI?}
\includegraphics[scale=0.40]{screenshots/enrichment1.PNG}\\
\includegraphics[scale=0.38]{screenshots/enrichment2.PNG}\\
\includegraphics[scale=0.35]{screenshots/enrichment3.PNG}
\end{frame}
\begin{frame}
\frametitle{MISP modules - main types of modules}
\begin{itemize}
\item Expansion modules - enrich data that is in MISP
\begin{itemize}
\item Hover type - showing the expanded values directly on the attributes
\item Expansion type - showing and adding the expanded values via a proposal form
\end{itemize}
\item Import modules - import new data into MISP
\item Export modules - export existing data from MISP
\end{itemize}
\end{frame}
% \begin{frame}[fragile]
% \frametitle{Creating your Expansion module (Skeleton)}
% \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
% \begin{lstlisting}[language=python]
% import json
% import dns.resolver
%
% misperrors = {'error' : 'Error'}
% mispattributes = {'input': [], 'output': []}
% moduleinfo = {'version': '', 'author': '',
% 'description': '', 'module-type': []}
%
% def handler(q=False):
% if q is False:
% return False
% request = json.loads(q)
% r = {'results': [{'types': [], 'values':[]}]}
% return r
% def introspection():
% return mispattributes
% def version():
% return moduleinfo
%
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
% \begin{frame}[fragile]
% \frametitle{Creating your Expansion module (metadata 1)}
% \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
% \begin{lstlisting}[language=python]
% misperrors = {'error' : 'Error'}
% mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', 'ip-dst']}
% moduleinfo = {'version': '', 'author': '',
% 'description': '', 'module-type': []}
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
%
% \begin{frame}[fragile]
% \frametitle{Creating your Expansion module (metadata 2)}
% \begin{adjustbox}{width=\textwidth,height=10cm,keepaspectratio}
% \begin{lstlisting}[language=python,showstringspaces=false]
% misperrors = {'error' : 'Error'}
% mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', 'ip-dst']}
% moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy',
% 'description': 'Simple DNS expansion service to
% resolve IP address from MISP attributes', 'module-type': ['expansion','hover']}
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
%
% \begin{frame}[fragile]
% \frametitle{Creating your Expansion module (handler 1)}
% \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
% \begin{lstlisting}[language=python]
% def handler(q=False):
% if q is False:
% return False
% request = json.loads(q)
% # MAGIC
% # MORE MAGIC
% r = {'results': [
% {'types': output_types, 'values':values},
% {'types': output_types2, 'values':values2}
% ]}
% return r
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
%
%
% \begin{frame}[fragile]
% \frametitle{Creating your Expansion module (handler 2)}
% \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
% \begin{lstlisting}[language=python]
% if request.get('hostname'):
% toquery = request['hostname']
% elif request.get('domain'):
% toquery = request['domain']
% else:
% return False
% r = dns.resolver.Resolver()
% r.timeout = 2
% r.lifetime = 2
% r.nameservers = ['8.8.8.8']
% try:
% answer = r.query(toquery, 'A')
% except dns.resolver.NXDOMAIN:
% misperrors['error'] = "NXDOMAIN"
% return misperrors
% except dns.exception.Timeout:
% misperrors['error'] = "Timeout"
% return misperrors
% except:
% misperrors['error'] = "DNS resolving error"
% return misperrors
% r = {'results': [{'types': mispattributes['output'], 'values':[str(answer[0])]}]}
% return r
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
\begin{frame}[fragile] \begin{frame}[fragile]
\frametitle{Querying a module} \frametitle{Querying a module}
\begin{itemize} \begin{itemize}
@ -171,136 +299,8 @@
\end{lstlisting} \end{lstlisting}
\end{frame} \end{frame}
\begin{frame}
\frametitle{MISP modules - How it's integrated in the UI?}
\includegraphics[scale=0.40]{screenshots/enrichment1.PNG}\\
\includegraphics[scale=0.38]{screenshots/enrichment2.PNG}\\
\includegraphics[scale=0.35]{screenshots/enrichment3.PNG}
\end{frame}
\begin{frame}
\frametitle{MISP modules - configuration in the UI}
\includegraphics[scale=0.50]{modules-integration.png}
\end{frame}
\begin{frame}
\frametitle{MISP modules - main types of modules}
\begin{itemize}
\item Expansion modules - enrich data that is in MISP
\begin{itemize}
\item Hover type - showing the expanded values directly on the attributes
\item Expansion type - showing and adding the expanded values via a proposal form
\end{itemize}
\item Import modules - import new data into MISP
\item Export modules - export existing data from MISP
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your Expansion module (Skeleton)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
import json
import dns.resolver
misperrors = {'error' : 'Error'}
mispattributes = {'input': [], 'output': []}
moduleinfo = {'version': '', 'author': '',
'description': '', 'module-type': []}
def handler(q=False):
if q is False:
return False
request = json.loads(q)
r = {'results': [{'types': [], 'values':[]}]}
return r
def introspection():
return mispattributes
def version():
return moduleinfo
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your Expansion module (metadata 1)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
misperrors = {'error' : 'Error'}
mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', 'ip-dst']}
moduleinfo = {'version': '', 'author': '',
'description': '', 'module-type': []}
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your Expansion module (metadata 2)}
\begin{adjustbox}{width=\textwidth,height=10cm,keepaspectratio}
\begin{lstlisting}[language=python,showstringspaces=false]
misperrors = {'error' : 'Error'}
mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', 'ip-dst']}
moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy',
'description': 'Simple DNS expansion service to
resolve IP address from MISP attributes', 'module-type': ['expansion','hover']}
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your Expansion module (handler 1)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
def handler(q=False):
if q is False:
return False
request = json.loads(q)
# MAGIC
# MORE MAGIC
r = {'results': [
{'types': output_types, 'values':values},
{'types': output_types2, 'values':values2}
]}
return r
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your Expansion module (handler 2)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
if request.get('hostname'):
toquery = request['hostname']
elif request.get('domain'):
toquery = request['domain']
else:
return False
r = dns.resolver.Resolver()
r.timeout = 2
r.lifetime = 2
r.nameservers = ['8.8.8.8']
try:
answer = r.query(toquery, 'A')
except dns.resolver.NXDOMAIN:
misperrors['error'] = "NXDOMAIN"
return misperrors
except dns.exception.Timeout:
misperrors['error'] = "Timeout"
return misperrors
except:
misperrors['error'] = "DNS resolving error"
return misperrors
r = {'results': [{'types': mispattributes['output'], 'values':[str(answer[0])]}]}
return r
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile] \begin{frame}[fragile]
\frametitle{Creating your module - finished DNS module} \frametitle{Creating your module - DNS module}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] \begin{lstlisting}[language=python]
import json import json
@ -423,205 +423,206 @@
\end{itemize} \end{itemize}
\end{frame} \end{frame}
\begin{frame}[fragile] % \begin{frame}[fragile]
\frametitle{Creating your Import module (Skeleton)} % \frametitle{Creating your Import module (Skeleton)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] % \begin{lstlisting}[language=python]
import json % import json
%
% misperrors = {'error' : 'Error'}
% userConfig = {
% 'number1': {
% 'type': 'Integer',
% 'regex': '/^[0-4]$/i',
% 'errorMessage': 'Expected a number in range [0-4]',
% 'message': 'Column number used for value'
% }
% };
% inputSource = ['file', 'paste']
% moduleinfo = {'version': '', 'author': '',
% 'description': '', 'module-type': ['import']}
% moduleconfig=[]
%
% def handler(q=False):
% if q is False:
% return False
% request = json.loads(q)
% request["data"] = base64.b64decode(request["data"])
% r = {'results': [{'categories': [], 'types': [], 'values':[]}]}
% return r
%
% def introspection():
% return {'userConfig': userConfig, 'inputSource': inputSource, 'moduleConfig': moduleConfig}
%
% def version():
% return moduleinfo
% \end{lstlisting}
% \end{adjustbox}
% \end{frame}
misperrors = {'error' : 'Error'} % \begin{frame}[fragile]
userConfig = { % \frametitle{Creating your import module (userConfig and inputSource)}
'number1': { % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
'type': 'Integer', % \begin{lstlisting}[language=python]
'regex': '/^[0-4]$/i', % userConfig = {
'errorMessage': 'Expected a number in range [0-4]', % 'number1': {
'message': 'Column number used for value' % 'type': 'Integer',
} % 'regex': '/^[0-4]$/i',
}; % 'errorMessage': 'Expected a number in range [0-4]',
inputSource = ['file', 'paste'] % 'message': 'Column number used for value'
moduleinfo = {'version': '', 'author': '', % }
'description': '', 'module-type': ['import']} % };
moduleconfig=[] % inputSource = ['file', 'paste']
% \end{lstlisting}
def handler(q=False): % \end{adjustbox}
if q is False: % \end{frame}
return False %
request = json.loads(q) % \begin{frame}[fragile]
request["data"] = base64.b64decode(request["data"]) % \frametitle{Creating your import module (Handler)}
r = {'results': [{'categories': [], 'types': [], 'values':[]}]} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
return r % \begin{lstlisting}[language=python]
% def handler(q=False):
def introspection(): % if q is False:
return {'userConfig': userConfig, 'inputSource': inputSource, 'moduleConfig': moduleConfig} % return False
% request = json.loads(q)
def version(): % request["data"] = base64.b64decode(request["data"])
return moduleinfo % r = {'results': [{'categories': [], 'types': [], 'values':[]}]}
\end{lstlisting} % return r
\end{adjustbox} % \end{lstlisting}
\end{frame} % \end{adjustbox}
% \end{frame}
\begin{frame}[fragile] %
\frametitle{Creating your import module (userConfig and inputSource)} % \begin{frame}[fragile]
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \frametitle{Creating your import module (Introspection)}
\begin{lstlisting}[language=python] % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
userConfig = { % \begin{lstlisting}[language=python]
'number1': { % def introspection():
'type': 'Integer', % modulesetup = {}
'regex': '/^[0-4]$/i', % try:
'errorMessage': 'Expected a number in range [0-4]', % userConfig
'message': 'Column number used for value' % modulesetup['userConfig'] = userConfig
} % except NameError:
}; % pass
inputSource = ['file', 'paste'] % try:
\end{lstlisting} % moduleConfig
\end{adjustbox} % modulesetup['moduleConfig'] = moduleConfig
\end{frame} % except NameError:
% pass
\begin{frame}[fragile] % try:
\frametitle{Creating your import module (Handler)} % inputSource
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % modulesetup['inputSource'] = inputSource
\begin{lstlisting}[language=python] % except NameError:
def handler(q=False): % pass
if q is False: % return modulesetup
return False % \end{lstlisting}
request = json.loads(q) % \end{adjustbox}
request["data"] = base64.b64decode(request["data"]) % \end{frame}
r = {'results': [{'categories': [], 'types': [], 'values':[]}]}
return r
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile]
\frametitle{Creating your import module (Introspection)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
def introspection():
modulesetup = {}
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
moduleConfig
modulesetup['moduleConfig'] = moduleConfig
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
\end{lstlisting}
\end{adjustbox}
\end{frame}
\begin{frame}[fragile] \begin{frame}[fragile]
\frametitle{Export modules} \frametitle{Export modules}
\begin{itemize} \begin{itemize}
\item Not the preferred way to export data from MISP
\item Input is currently only a single event \item Input is currently only a single event
\item Dynamic settings
\item Later on to be expanded to event collections / attribute collections
\item Output is a file in the export format served back to the user \item Output is a file in the export format served back to the user
\item Export modules was recently introduced but a CEF export module already available \item Will be moved / merged with MISP built-in export modules
\item Lots of ideas for upcoming modules and including interaction with misp-darwin \begin{itemize}
\item Allows export of event / attribute collections
\end{itemize}
\end{itemize} \end{itemize}
\end{frame} \end{frame}
\begin{frame}[fragile] % \begin{frame}[fragile]
\frametitle{Creating your Export module (Skeleton)} % \frametitle{Creating your Export module (Skeleton)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] % \begin{lstlisting}[language=python]
import json % import json
inputSource = ['event'] % inputSource = ['event']
outputFileExtension = 'txt' % outputFileExtension = 'txt'
responseType = 'application/txt' % responseType = 'application/txt'
moduleinfo = {'version': '0.1', 'author': 'Andras Iklody', % moduleinfo = {'version': '0.1', 'author': 'Andras Iklody',
'description': 'Skeleton export module', % 'description': 'Skeleton export module',
'module-type': ['export']} % 'module-type': ['export']}
%
def handler(q=False): % def handler(q=False):
if q is False: % if q is False:
return False % return False
request = json.loads(q) % request = json.loads(q)
# insert your magic here! % # insert your magic here!
output = my_magic(request["data"]) % output = my_magic(request["data"])
r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} % r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')}
return r % return r
%
def introspection(): % def introspection():
return {'userConfig': userConfig, 'inputSource': inputSource, 'moduleConfig': moduleConfig, 'outputFileExtension': outputFileExtension} % return {'userConfig': userConfig, 'inputSource': inputSource, 'moduleConfig': moduleConfig, 'outputFileExtension': outputFileExtension}
%
def version(): % def version():
return moduleinfo % return moduleinfo
\end{lstlisting} % \end{lstlisting}
\end{adjustbox} % \end{adjustbox}
\end{frame} % \end{frame}
%
\begin{frame}[fragile] % \begin{frame}[fragile]
\frametitle{Creating your export module (settings)} % \frametitle{Creating your export module (settings)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] % \begin{lstlisting}[language=python]
inputSource = ['event'] % inputSource = ['event']
outputFileExtension = 'txt' % outputFileExtension = 'txt'
responseType = 'application/txt' % responseType = 'application/txt'
\end{lstlisting} % \end{lstlisting}
\end{adjustbox} % \end{adjustbox}
\end{frame} % \end{frame}
%
\begin{frame}[fragile] % \begin{frame}[fragile]
\frametitle{Creating your export module (handler)} % \frametitle{Creating your export module (handler)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] % \begin{lstlisting}[language=python]
def handler(q=False): % def handler(q=False):
if q is False: % if q is False:
return False % return False
request = json.loads(q) % request = json.loads(q)
# insert your magic here! % # insert your magic here!
output = my_magic(request["data"]) % output = my_magic(request["data"])
r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} % r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')}
return r % return r
\end{lstlisting} % \end{lstlisting}
\end{adjustbox} % \end{adjustbox}
\end{frame} % \end{frame}
%
\begin{frame}[fragile] % \begin{frame}[fragile]
\frametitle{Creating your export module (introspection)} % \frametitle{Creating your export module (introspection)}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio} % \begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python] % \begin{lstlisting}[language=python]
def introspection(): % def introspection():
modulesetup = {} % modulesetup = {}
try: % try:
responseType % responseType
modulesetup['responseType'] = responseType % modulesetup['responseType'] = responseType
except NameError: % except NameError:
pass % pass
try: % try:
userConfig % userConfig
modulesetup['userConfig'] = userConfig % modulesetup['userConfig'] = userConfig
except NameError: % except NameError:
pass % pass
try: % try:
moduleConfig % moduleConfig
modulesetup['moduleConfig'] = moduleConfig % modulesetup['moduleConfig'] = moduleConfig
except NameError: % except NameError:
pass % pass
try: % try:
outputFileExtension % outputFileExtension
modulesetup['outputFileExtension'] = outputFileExtension % modulesetup['outputFileExtension'] = outputFileExtension
except NameError: % except NameError:
pass % pass
try: % try:
inputSource % inputSource
modulesetup['inputSource'] = inputSource % modulesetup['inputSource'] = inputSource
except NameError: % except NameError:
pass % pass
return modulesetup % return modulesetup
\end{lstlisting} % \end{lstlisting}
\end{adjustbox} % \end{adjustbox}
\end{frame} % \end{frame}
\begin{frame}[fragile] \begin{frame}[fragile]
\frametitle{New expansion \& import modules format} \frametitle{New expansion \& import modules format}
@ -636,8 +637,20 @@
\end{adjustbox} \end{adjustbox}
\begin{itemize} \begin{itemize}
\item Takes a standard MISP attribute as input \item Takes a standard MISP attribute as input
\item Can return MISP attributes, objects \& tags \item Returns MISP format
\item Supports relationships \begin{itemize}
\item Attributes
\item Objects (with their references)
\item Tags
\end{itemize}
\end{itemize}
\begin{adjustbox}{width=\textwidth,height=5cm,keepaspectratio}
\begin{lstlisting}[language=python]
results = {'Attribute': [...], 'Object': [...],
'Tag': [...]}
\end{lstlisting}
\end{adjustbox}
\begin{itemize}
\item First modules supporting this new export format \item First modules supporting this new export format
\begin{itemize} \begin{itemize}
\item urlhaus expansion module \item urlhaus expansion module
@ -652,11 +665,15 @@
\end{frame} \end{frame}
\begin{frame}[fragile] \begin{frame}[fragile]
\frametitle{Upcoming additions to the module system - General} \frametitle{Future of the modules system}
\begin{itemize} \begin{itemize}
\item Expose the modules to the APIs \item Enrichment on full events
\item Move the modules to background processes with a messaging system \item Move the modules to background processes with a messaging system
\item Difficulty is dealing with uncertain results on import (without the user having final say) \item Have a way to skip the results preview
\begin{itemize}
\item Preview can be very heavy
\item Difficulty is dealing with uncertain results (without the user having final say)
\end{itemize}
\end{itemize} \end{itemize}
\end{frame} \end{frame}
@ -670,4 +687,3 @@
\end{itemize} \end{itemize}
\end{frame} \end{frame}