PyMISP/docs/tutorial/Search-FullOverview.ipynb

604 lines
12 KiB
Plaintext
Raw Normal View History

{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# The URL of the MISP instance to connect to\n",
2019-09-16 21:52:38 +02:00
"misp_url = 'https://127.0.0.1:8443'\n",
"# Can be found in the MISP web interface under ||\n",
"# http://+MISP_URL+/users/view/me -> Authkey\n",
2019-09-16 21:52:38 +02:00
"misp_key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh'\n",
"# Should PyMISP verify the MISP certificate\n",
"misp_verifycert = False"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Getting the API key (automatically generated on the trainig VM)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"\n",
"api_file = Path('apikey')\n",
"if api_file.exists():\n",
" misp_url = 'http://127.0.0.1'\n",
" misp_verifycert = False\n",
" with open(api_file) as f:\n",
" misp_key = f.read().strip()\n",
" print(misp_key)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Initialize PyMISP - NG"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pymisp import ExpandedPyMISP\n",
"\n",
"misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Index Search (fast, only returns events metadata)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search unpublished events\n",
"\n",
2019-04-11 23:14:16 +02:00
"**WARNING**: By default, the search query will only return all the events listed on the index page"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(published=False, metadata=True)\n",
"print(r)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get the meta data of events"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(eventid=[1,2,3], metadata=True, pythonify=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search Tag & mix with other parameters"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(tags=['tlp:white'], metadata=True, pythonify=True)\n",
"for e in r:\n",
" print(e)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"print('No attributes are in the event', r[0].attributes)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(tags='TODO:VT-ENRICHMENT', published=False)"
2019-04-11 23:14:16 +02:00
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(tags=['!TODO:VT-ENRICHMENT', 'tlp:white'], metadata=True, published=False) # ! means \"not this tag\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Full text search on event info field"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(eventinfo='circl', metadata=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search by org"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(org='CIRCL', metadata=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search updated events"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-09-16 21:52:38 +02:00
"r = misp.search(timestamp='1h', metadata=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Search full events (Slower, returns full events)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Getting timestamps"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime, date, timedelta\n",
"from dateutil.parser import parse\n",
"\n",
"int(datetime.now().timestamp())\n",
"\n",
"d = parse('2018-03-24')\n",
"int(d.timestamp())\n",
"\n",
"today = int(datetime.today().timestamp())\n",
"yesterday = int((datetime.today() - timedelta(days=1)).timestamp())\n",
"\n",
"print(today, yesterday)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"complex_query = misp.build_complex_query(or_parameters=['uibo.lembit@mail.ee', '103.195.185.222'])"
]
},
2019-04-11 23:14:16 +02:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(complex_query)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"complex_query = misp.build_complex_query(or_parameters=['59.157.4.2', 'hotfixmsupload.com', '8.8.8.8'])\n",
"events = misp.search(value=complex_query, pythonify=True)\n",
"\n",
"for e in events:\n",
" print(e)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value=complex_query, pythonify=True)\n",
"print(r)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(category='Payload delivery')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='uibo.lembit@mail.ee', metadata=True, pythonify=True) # no attributes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(timestamp=['2h', '1h'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', enforceWarninglist=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', deleted=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', publish_timestamp=1521846000) # everything published since that timestamp"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', last='1d') # everything published in the last <interval>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', timestamp=[yesterday, today]) # everything updated since that timestamp"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(value='8.8.8.8', withAttachments=True) # Return attachments"
]
},
2019-04-11 23:14:16 +02:00
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(r)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Search for attributes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"r = misp.search(controller='attributes', value='8.8.8.8')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(controller='attributes', value='wrapper.no', event_timestamp='5d') # only consider events updated since this timestamp"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"print(r)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Search attributes (specified in controller) where the attribute type is 'ip-src'\n",
"# And the to_ids flag is set\n",
"attributes = misp.search(controller='attributes', type_attribute='ip-src', to_ids=0, pythonify=True)\n",
"\n",
"event_ids = set()\n",
"for attr in attributes:\n",
" event_ids.add(attr.event_id)\n",
2019-04-11 23:14:16 +02:00
"\n",
"# Fetch all related events\n",
"for event_id in event_ids:\n",
" event = misp.get_event(event_id)\n",
" print(event.info)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
2019-04-11 23:14:16 +02:00
"## Last *published* attributes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"attributes = misp.search(controller='attributes', publish_timestamp='1d', pythonify=True)\n",
"\n",
2019-04-11 23:14:16 +02:00
"for attribute in attributes:\n",
" print(attribute.event_id, attribute)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"attributes = misp.search(controller='attributes', publish_timestamp=['2d', '1h'], pythonify=True)\n",
"\n",
2019-04-11 23:14:16 +02:00
"for a in attributes:\n",
" print(a)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Last *updated* attributes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"from datetime import datetime\n",
"\n",
"ts = int(datetime.now().timestamp())\n",
"\n",
"attributes = misp.search(controller='attributes', timestamp=ts - 36000, pythonify=True)\n",
"\n",
"for a in attributes:\n",
" print(a)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Orther output formats\n",
"\n",
"**Warning**: For that to work, the matching event has to be published"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"r = misp.search(controller='attributes', value='8.8.8.8', return_format='csv')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"r = misp.search(controller='events', value='9.8.8.8', return_format='snort')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(controller='events', value='9.8.8.8', return_format='suricata')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(controller='events', value='9.8.8.8', return_format='stix')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = misp.search(controller='events', value='9.8.8.8', return_format='stix2')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print(r)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Search in logs"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
2019-04-11 23:14:16 +02:00
"logs = misp.search_logs(model='Tag', title='tlp:white')\n",
"print(logs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"logs = misp.search_logs(model='Event', pythonify=True)\n",
"#print(logs)\n",
"for l in logs:\n",
" print(l.title)"
]
2019-09-16 21:52:38 +02:00
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2019-09-11 14:20:55 +02:00
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}