From 8ac2449d712920cd72606dac166e7ea62851c7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 16 Jul 2018 18:09:44 +0200 Subject: [PATCH] new: Add Jupyter for search --- docs/tutorial/Search.ipynb | 365 +++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 docs/tutorial/Search.ipynb diff --git a/docs/tutorial/Search.ipynb b/docs/tutorial/Search.ipynb new file mode 100644 index 0000000..47b420b --- /dev/null +++ b/docs/tutorial/Search.ipynb @@ -0,0 +1,365 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The URL of the MISP instance to connect to\n", + "misp_url = 'https:///'\n", + "# Can be found in the MISP web interface under \n", + "# http://+MISP_URL+/users/view/me -> Authkey\n", + "misp_key = ''\n", + "# Should PyMISP verify the MISP certificate\n", + "misp_verifycert = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pymisp import PyMISP\n", + "\n", + "misp = PyMISP(misp_url, misp_key, misp_verifycert, debug=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Index Search (fast, only returns events metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search unpublished events\n", + "\n", + "**WARNING**: By default, the search query will only return all the events listed on teh index page" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(published=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get the meta data of events" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(eventid=[17217, 1717, 1721, 17218])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search Tag & mix with other parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(tag='TODO:VT-ENRICHMENT')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(tag='TODO:VT-ENRICHMENT', published=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(tag=['!TODO:VT-ENRICHMENT', 'tlp:white'], published=False) # ! means \"not this tag\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Full text search on event info field" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(eventinfo='circl')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search in the values of each attributes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(attribute='8.8.8.8')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search by org" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(org='CIRCL')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search updated events" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search_index(timestamp='1h')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Search full events (Slower, returns full events)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Getting timestamps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime, date, timedelta\n", + "from dateutil.parser import parse\n", + "\n", + "int(datetime.now().timestamp())\n", + "\n", + "d = parse('2018-03-24')\n", + "int(d.timestamp())\n", + "\n", + "today = int(datetime.today().timestamp())\n", + "yesterday = int((datetime.today() - timedelta(days=1)).timestamp())\n", + "\n", + "print(today, yesterday)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(not_values='8.8.8.8')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(category='Payload delivery')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', metadata=True) # no attributes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(timestamp=['2h', '1h'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', enforceWarninglist=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', deleted=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', publish_timestamp=1521846000) # everything published since that timestamp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', last='1d') # everything published in the last " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', timestamp=[yesterday, today]) # everything updated since that timestamp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(values='8.8.8.8', withAttachments=True) # Return attachments" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Search for attributes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(controller='attributes', values='8.8.8.8')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = misp.search(controller='attributes', values='wrapper.no', event_timestamp='5d') # only consider events updated since this timestamp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}