chg: Rework notebooks.

docs/tutorial/FullOverview.ipynb

@@ -53,30 +53,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Getting the API key (automatically generated on the trainig VM)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pathlib import Path\n",
-    "\n",
-    "api_file = Path('apikey')\n",
-    "if api_file.exists():\n",
-    "    misp_url = 'http://127.0.0.1'\n",
-    "    misp_verifycert = False\n",
-    "    with open(api_file) as f:\n",
-    "        misp_key = f.read().strip()\n",
-    "    print(misp_key)"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -272,6 +248,15 @@
     "print(attribute_second.to_json())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(event.to_json())"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -312,6 +297,57 @@
     "print(event.published)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## MISPAttribute"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attr_type = 'ip-dst'\n",
+    "value = '1.1.1.1'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPAttribute\n",
+    "\n",
+    "# Attribute data already defined\n",
+    "attribute = MISPAttribute()\n",
+    "attribute.type = attr_type\n",
+    "attribute.value = value\n",
+    "print(attribute)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# An attribute can also be loaded directly from a JSON\n",
+    "json = '''{\n",
+    "    \"type\": \"ip-dst\",\n",
+    "    \"value\": \"127.0.0.1\",\n",
+    "    \"category\": \"Network activity\",\n",
+    "    \"to_ids\": false\n",
+    "    }'''\n",
+    "\n",
+    "attribute = MISPAttribute()\n",
+    "attribute.from_json(json)\n",
+    "print(attribute)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -354,7 +390,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## One-liner to add an object to a MISPEvent\n",
+    "## Short version to add an object to a MISPEvent\n",
     "\n",
     "You can also add the object directly in a misp event this way"
    ]
@@ -374,6 +410,10 @@
     "misp_object.add_attribute('ip', value='149.13.33.14')\n",
     "misp_object.add_attribute('first-seen', value='2018-04-11')\n",
     "misp_object.add_attribute('last-seen', value='2018-06-11')\n",
+    "\n",
+    "misp_object.add_attributes('ip', {'value': '10.8.8.8', 'to_ids': False}, '10.9.8.8')\n",
+    "\n",
+    "\n",
     "misp_object.add_reference(obj_attr.uuid, 'related-to', 'Expanded with passive DNS entry')\n",
     "\n",
     "print(event.to_json())\n"
@@ -383,7 +423,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Helpers for MISPObjects \n",
+    "# Helpers for MISPObjects \n",
     "\n",
     "For some objects, we have helpers in order to make your life easier. The most relevant example is the file object: when you have a file to push on MISP, there are plenty of indicators you can extract at once, and it is pretty simple to automate, so we made it a oneliner.\n",
     "\n",
@@ -424,7 +464,74 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Generic helper\n",
+    "### Excel support \n",
+    "\n",
+    "(okay, CSV, but that's the same thing, right?)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash \n",
+    "\n",
+    "cat ../../tests/csv_testfiles/valid_fieldnames.csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash \n",
+    "\n",
+    "cat ../../tests/csv_testfiles/invalid_fieldnames.csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp.tools import CSVLoader\n",
+    "from pymisp import MISPEvent\n",
+    "from pathlib import Path\n",
+    "\n",
+    "csv1 = CSVLoader(template_name='file', csv_path=Path('../../tests/csv_testfiles/valid_fieldnames.csv'))\n",
+    "event = MISPEvent()\n",
+    "event.info = 'Test event from CSV loader'\n",
+    "for o in csv1.load():\n",
+    "    event.add_object(**o)\n",
+    "\n",
+    "print(event.to_json())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "event = MISPEvent()\n",
+    "event.info = 'Test event from CSV loader'\n",
+    "csv2 = CSVLoader(template_name='file', csv_path=Path('../../tests/csv_testfiles/invalid_fieldnames.csv'),\n",
+    "                 fieldnames=['SHA1', 'fileName', 'size-in-bytes'], has_fieldnames=True)\n",
+    "\n",
+    "for o in csv2.load():\n",
+    "    event.add_object(**o)\n",
+    "    \n",
+    "print(event.to_json())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Generic helper\n",
     "\n",
     "This helper is meant to be used when you alreadu have a script that does the mapping between your own code, and the MISPObject template."
    ]
@@ -449,6 +556,13 @@
     "print(misp_object.to_json())"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## User defined objects"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -647,98 +761,6 @@
     "print(existing_event.attributes[0].to_json())"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Full example"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pymisp import MISPEvent, MISPObject\n",
-    "from pymisp import PyMISP\n",
-    "\n",
-    "event = MISPEvent()\n",
-    "event.info = 'This is my new MISP event'  # Required\n",
-    "event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config\n",
-    "event.threat_level_id = 2  # Optional, defaults to MISP.default_event_threat_level in MISP config\n",
-    "event.analysis = 1  # Optional, defaults to 0 (initial analysis)\n",
-    "\n",
-    "mispObject = MISPObject('file')\n",
-    "mispObject.add_attribute('filename', type='filename',\n",
-    "                         value='filename.exe',\n",
-    "                         Tag=[{'name': 'tlp:amber'}])\n",
-    "\n",
-    "event.add_object(mispObject)\n",
-    "\n",
-    "# The URL of the MISP instance to connect to\n",
-    "misp_url = 'http://127.0.0.1:8080'\n",
-    "# Can be found in the MISP web interface under \n",
-    "# http://+MISP_URL+/users/view/me -> Authkey\n",
-    "misp_key = 'xe5okWNY2OB3O9ljR6t2cJPNsv4u1VZB0C1mKwtB'\n",
-    "# Should PyMISP verify the MISP certificate\n",
-    "misp_verifycert = False\n",
-    "\n",
-    "misp = PyMISP(misp_url, misp_key, misp_verifycert)\n",
-    "res = misp.add_event(event)\n",
-    "existing_event = MISPEvent()\n",
-    "existing_event.load(res)\n",
-    "mispObject = MISPObject('file')\n",
-    "mispObject.add_attribute('filename', type='filename',\n",
-    "                         value='filename2.exe',\n",
-    "                         Tag=[{'name': 'tlp:white'}])\n",
-    "\n",
-    "existing_event.add_object(mispObject)\n",
-    "print(existing_event.to_json())\n",
-    "\n",
-    "res = misp.update(existing_event)\n",
-    "existing_event = MISPEvent()\n",
-    "existing_event.load(res)\n",
-    "print(existing_event.to_json())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pymisp import MISPEvent, MISPObject\n",
-    "from pymisp import PyMISP\n",
-    "\n",
-    "event = MISPEvent()\n",
-    "\n",
-    "event.info = 'This is my new MISP event'  # Required\n",
-    "event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config\n",
-    "event.threat_level_id = 2  # Optional, defaults to MISP.default_event_threat_level in MISP config\n",
-    "event.analysis = 1  # Optional, defaults to 0 (initial analysis)\n",
-    "\n",
-    "mispObject = MISPObject('file')\n",
-    "mispObject.add_attribute('filename', type='filename',\n",
-    "                                         value='filename.exe',\n",
-    "                                         Tag=[{'name':'tlp:amber'}]) \n",
-    "event.add_object(mispObject)\n",
-    "\n",
-    "# The URL of the MISP instance to connect to\n",
-    "misp_url = 'http://127.0.0.1:8080'\n",
-    "# Can be found in the MISP web interface under \n",
-    "# http://+MISP_URL+/users/view/me -> Authkey\n",
-    "misp_key = 'yB8DMS8LkfYYpcVX8bN2v7xwDZDMp4bpW0sNqNGj'\n",
-    "# Should PyMISP verify the MISP certificate\n",
-    "misp_verifycert = False\n",
-    "\n",
-    "misp = PyMISP(misp_url, misp_key, misp_verifycert)\n",
-    "res = misp.add_event(event)\n",
-    "existing_event = MISPEvent()\n",
-    "existing_event.load(res)\n",
-    "print(existing_event.to_json())"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -769,6 +791,668 @@
     "\n",
     "print(event.to_json())\n"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "***"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Getting the API key (automatically generated on the trainig VM)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "api_file = Path('apikey')\n",
+    "if api_file.exists():\n",
+    "    misp_url = 'http://127.0.0.1'\n",
+    "    misp_verifycert = False\n",
+    "    with open(api_file) as f:\n",
+    "        misp_key = f.read().strip()\n",
+    "    print(misp_key)\n",
+    "else:\n",
+    "    print(\"Unable to find the api key\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Initialize variables if you run the notebook locally"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The URL of the MISP instance to connect to\n",
+    "misp_url = 'http://127.0.0.1:8080/'\n",
+    "# Can be found in the MISP web interface under \n",
+    "# http://+MISP_URL+/users/view/me -> Authkey\n",
+    "misp_key = 'HRizIMmaxBOXAQSzKZ874rDWUsQEk4vGAGBoljQO'\n",
+    "# Should PyMISP verify the MISP certificate\n",
+    "misp_verifycert = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import ExpandedPyMISP, PyMISP\n",
+    "\n",
+    "misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)\n",
+    "misp_old = PyMISP(misp_url, misp_key, misp_verifycert)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Full example"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## New API\n",
+    "\n",
+    "Returns MISPEvent."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPEvent, MISPObject\n",
+    "\n",
+    "event = MISPEvent()\n",
+    "event.info = 'This is my new MISP event'  # Required\n",
+    "event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config\n",
+    "event.threat_level_id = 2  # Optional, defaults to MISP.default_event_threat_level in MISP config\n",
+    "event.analysis = 1  # Optional, defaults to 0 (initial analysis)\n",
+    "\n",
+    "mispObject = MISPObject('file')\n",
+    "mispObject.add_attribute('filename', type='filename',\n",
+    "                         value='filename.exe',\n",
+    "                         Tag=[{'name': 'tlp:amber'}])\n",
+    "\n",
+    "event.add_object(mispObject)\n",
+    "\n",
+    "print(misp)\n",
+    "existing_event = misp.add_event(event)\n",
+    "print(existing_event)\n",
+    "mispObject = MISPObject('file')\n",
+    "mispObject.add_attribute('filename', type='filename',\n",
+    "                         value='filename2.exe',\n",
+    "                         Tag=[{'name': 'tlp:white'}])\n",
+    "\n",
+    "existing_event.add_object(mispObject)\n",
+    "print(existing_event.to_json())\n",
+    "\n",
+    "res = misp.update(existing_event)\n",
+    "existing_event = MISPEvent()\n",
+    "existing_event.load(res)\n",
+    "print(existing_event.to_json())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Old API\n",
+    "\n",
+    "Returns plain JSON"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPEvent, MISPObject\n",
+    "\n",
+    "event = MISPEvent()\n",
+    "event.info = 'This is my new MISP event'  # Required\n",
+    "event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config\n",
+    "event.threat_level_id = 2  # Optional, defaults to MISP.default_event_threat_level in MISP config\n",
+    "event.analysis = 1  # Optional, defaults to 0 (initial analysis)\n",
+    "\n",
+    "mispObject = MISPObject('file')\n",
+    "mispObject.add_attribute('filename', type='filename',\n",
+    "                         value='filename.exe',\n",
+    "                         Tag=[{'name': 'tlp:amber'}])\n",
+    "\n",
+    "event.add_object(mispObject)\n",
+    "\n",
+    "print(misp)\n",
+    "res = misp.add_event(event)\n",
+    "print(res)\n",
+    "existing_event = MISPEvent()\n",
+    "existing_event.load(res)\n",
+    "mispObject = MISPObject('file')\n",
+    "mispObject.add_attribute('filename', type='filename',\n",
+    "                         value='filename2.exe',\n",
+    "                         Tag=[{'name': 'tlp:white'}])\n",
+    "\n",
+    "existing_event.add_object(mispObject)\n",
+    "print(existing_event.to_json())\n",
+    "\n",
+    "res = misp.update(existing_event)\n",
+    "existing_event = MISPEvent()\n",
+    "existing_event.load(res)\n",
+    "print(existing_event.to_json())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Interacting with a MISP instance"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Creating An Event"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "event = misp.new_event(distribution=1,\n",
+    "               threat_level_id=1,\n",
+    "               analysis=1,\n",
+    "               info=\"Event from notebook\")\n",
+    "print(\"Event id: %s\" % event.id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "event = misp_old.new_event(distribution=1,\n",
+    "               threat_level_id=1,\n",
+    "               analysis=1,\n",
+    "               info=\"Event from notebook\")\n",
+    "print(\"Event id: %s\" % event['Event']['id'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Using the MISPEvent constructor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPEvent\n",
+    "\n",
+    "event_obj = MISPEvent()\n",
+    "event_obj.distribution = 1\n",
+    "event_obj.threat_level_id = 1\n",
+    "event_obj.analysis = 1\n",
+    "event_obj.info = \"Event from notebook 2\"\n",
+    "event = misp.add_event(event_obj)\n",
+    "event_id = event.id\n",
+    "print(\"Event id: %s\" % event_id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Fetching an event"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "event_id = 2752"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fetch by ID\n",
+    "event = misp.get_event(event_id)\n",
+    "print(event)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fetch by ID\n",
+    "event = misp_old.get_event(event_id)\n",
+    "print(event)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Add an attribute to an event"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attr_type = \"ip-src\"\n",
+    "value = \"8.8.8.8\"\n",
+    "category = \"Network activity\"\n",
+    "to_ids = False"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Oldish API"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "proposal = False\n",
+    "updated_event = misp.add_named_attribute(event_id,\n",
+    "                                 attr_type,\n",
+    "                                 value,\n",
+    "                                 category=category,\n",
+    "                                 to_ids=to_ids,\n",
+    "                                 proposal=proposal)\n",
+    "print(updated_event)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Cleaner way"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "value = \"9.8.8.8\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPAttribute\n",
+    "\n",
+    "# Attribute data already defined\n",
+    "attribute = MISPAttribute()\n",
+    "attribute.type = attr_type\n",
+    "attribute.value = value\n",
+    "attribute.category = category\n",
+    "attribute.to_ids = to_ids\n",
+    "\n",
+    "attribute_to_change = misp.add_attribute(event_id, attribute)\n",
+    "print(attribute_to_change.id, attribute_to_change)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Propose new Attribute"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPAttribute\n",
+    "\n",
+    "attr_type = \"ip-src\"\n",
+    "value = \"10.8.8.8\"\n",
+    "category = \"Network activity\"\n",
+    "to_ids = False\n",
+    "\n",
+    "# Attribute data already defined\n",
+    "attribute = MISPAttribute()\n",
+    "attribute.type = attr_type\n",
+    "attribute.value = value\n",
+    "attribute.category = category\n",
+    "attribute.to_ids = to_ids\n",
+    "\n",
+    "proposal = misp.add_attribute_proposal(event_id, attribute)\n",
+    "print(proposal.id, proposal)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Other things on proposals"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "proposal = misp.get_attribute_proposal(21)\n",
+    "print(proposal.to_json())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "proposal = misp.accept_attribute_proposal(25)\n",
+    "print(proposal)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "proposal = misp.discard_attribute_proposal(27)\n",
+    "print(proposal)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Propose change to attribute"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPShadowAttribute\n",
+    "\n",
+    "proposal = MISPShadowAttribute()\n",
+    "proposal.type = 'ip-dst'\n",
+    "proposal.category = 'External analysis'\n",
+    "proposal.to_ids = False\n",
+    "\n",
+    "attribute = misp.update_attribute_proposal(attribute_to_change.id, proposal)\n",
+    "print(attribute.to_json())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attribute = misp.update_attribute_proposal(attribute_to_change.id, {'to_ids': False, 'comment': \"This is crap\"})\n",
+    "print(attribute.to_json())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Update existing event"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymisp import MISPAttribute, MISPObject\n",
+    "\n",
+    "attr_type = \"ip-src\"\n",
+    "value = \"20.8.8.8\"\n",
+    "category = \"Network activity\"\n",
+    "to_ids = False\n",
+    "\n",
+    "# Attribute data already defined\n",
+    "attribute = MISPAttribute()\n",
+    "attribute.type = attr_type\n",
+    "attribute.value = value\n",
+    "attribute.category = category\n",
+    "attribute.to_ids = to_ids\n",
+    "\n",
+    "# New Python 3.6 API\n",
+    "event = misp.get(event_id)\n",
+    "\n",
+    "## Add the attribute to the event\n",
+    "event.add_attribute(**attribute)\n",
+    "event.add_attribute(type='domain', value='circl.lu', disable_correlation=True)\n",
+    "\n",
+    "mispObject = MISPObject('file')\n",
+    "mispObject.add_attribute('filename', type='filename',\n",
+    "                         value='filename2.exe',\n",
+    "                         Tag=[{'name': 'tlp:white'}])\n",
+    "\n",
+    "event.add_object(mispObject)\n",
+    "\n",
+    "## Push the updated event to MISP\n",
+    "event_dict = misp.update_event(event)\n",
+    "print(event_dict)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Sightings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.sighting(value=event.attributes[1].value)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.sighting_list(event.attributes[1].id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Direct call, no validation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.direct_call('attributes/add/58', {'type': 'ip-dst', 'value': '8.11.8.8'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.direct_call('events')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Admin Stuff"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.get_sharing_groups()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## User"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.get_users_list()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.add_user('bar@foo.de', 1, 3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Organisations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.get_organisations_list()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Roles"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.get_roles_list()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Feeds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.get_feeds_list()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misp.cache_feeds_all()"
+   ]
   }
  ],
  "metadata": {

docs/tutorial/Search-FullOverview.ipynb

@@ -10,7 +10,7 @@
     "misp_url = 'http://127.0.0.1:8080'\n",
     "# Can be found in the MISP web interface under ||\n",
     "# http://+MISP_URL+/users/view/me -> Authkey\n",
-    "misp_key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu'\n",
+    "misp_key = 'HRizIMmaxBOXAQSzKZ874rDWUsQEk4vGAGBoljQO'\n",
     "# Should PyMISP verify the MISP certificate\n",
     "misp_verifycert = False"
    ]
@@ -70,7 +70,7 @@
    "source": [
     "## Search unpublished events\n",
     "\n",
-    "**WARNING**: By default, the search query will only return all the events listed on teh index page"
+    "**WARNING**: By default, the search query will only return all the events listed on the index page"
    ]
   },
   {
@@ -123,7 +123,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = misp.search_index(tag='TODO:VT-ENRICHMENT', published=False)"
+    "print('No attributes are in the event', r[0].attributes)"
    ]
   },
   {
@@ -132,7 +132,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = misp.search_index(tag=['!TODO:VT-ENRICHMENT', 'tlp:white'], published=False)  # ! means \"not this tag\""
+    "r = misp.search_index(tags='TODO:VT-ENRICHMENT', published=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search_index(tags=['!TODO:VT-ENRICHMENT', 'tlp:white'], published=False)  # ! means \"not this tag\""
    ]
   },
   {
@@ -227,6 +236,28 @@
     "complex_query = misp.build_complex_query(or_parameters=['uibo.lembit@mail.ee', '103.195.185.222'])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(complex_query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "complex_query = misp.build_complex_query(or_parameters=['59.157.4.2', 'hotfixmsupload.com', '8.8.8.8'])\n",
+    "events = misp.search(value=complex_query, pythonify=True)\n",
+    "\n",
+    "for e in events:\n",
+    "    print(e)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -318,6 +349,15 @@
     "r = misp.search(value='8.8.8.8', withAttachments=True)  # Return attachments"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(r)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -331,7 +371,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = misp.search(controller='attributes', value='8.8.8.9')"
+    "r = misp.search(controller='attributes', value='8.8.8.8')"
    ]
   },
   {
@@ -349,14 +389,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Because reason"
+    "print(r)"
    ]
   },
   {
@@ -365,22 +398,146 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "tag_to_remove = 'foo'\n",
+    "# Search attributes (specified in controller) where the attribute type is 'ip-src'\n",
+    "# And the to_ids flag is set\n",
+    "attributes = misp.search(controller='attributes', type_attribute='ip-src', to_ids=0, pythonify=True)\n",
     "\n",
-    "events = misp.search(tags=tag_to_remove, pythonify=True)\n",
+    "event_ids = set()\n",
+    "for attr in attributes:\n",
+    "    event_ids.add(event_id)\n",
     "\n",
-    "for event in events:\n",
-    "    for tag in event.tags:\n",
-    "        if tag.name == tag_to_remove:\n",
-    "            print(f'Got {tag_to_remove} in {event.info}')\n",
-    "            misp.untag(event.uuid, tag_to_remove)\n",
-    "            break\n",
-    "    for attribute in event.attributes:\n",
-    "        for tag in attribute.tags:\n",
-    "            if tag.name == tag_to_remove:\n",
-    "                print(f'Got {tag_to_remove} in {attribute.value}')\n",
-    "                misp.untag(attribute.uuid, tag_to_remove)\n",
-    "                break"
+    "# Fetch all related events\n",
+    "for event_id in event_ids:\n",
+    "    event = misp.get_event(event_id)\n",
+    "    print(event.info)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Last *published* attributes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attributes = misp.search(controller='attributes', publish_timestamp='1d', pythonify=True)\n",
+    "\n",
+    "for attribute in attributes:\n",
+    "    print(attribute.event_id, attribute)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "attributes = misp.search(controller='attributes', publish_timestamp=['2d', '1h'], pythonify=True)\n",
+    "\n",
+    "for a in attributes:\n",
+    "    print(a)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Last *updated* attributes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from datetime import datetime\n",
+    "\n",
+    "ts = int(datetime.now().timestamp())\n",
+    "\n",
+    "attributes = misp.search(controller='attributes', timestamp=ts - 36000, pythonify=True)\n",
+    "\n",
+    "for a in attributes:\n",
+    "    print(a)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Orther output formats\n",
+    "\n",
+    "**Warning**: For that to work, the matching event has to be published"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search(controller='attributes', value='8.8.8.8', return_format='csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search(controller='events', value='9.8.8.8', return_format='snort')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search(controller='events', value='9.8.8.8', return_format='suricata')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search(controller='events', value='9.8.8.8', return_format='stix')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = misp.search(controller='events', value='9.8.8.8', return_format='stix2')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "print(r)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Search in logs"
    ]
   },
   {
@@ -404,23 +561,6 @@
     "for l in logs:\n",
     "    print(l.title)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "log = misp.search_logs(model='Tag', title=tag_to_remove)[0]\n",
-    "roles = misp.get_roles_list()\n",
-    "for r in roles:\n",
-    "    if r['Role']['name'] == 'User':\n",
-    "        new_role = r['Role']['id']\n",
-    "        break\n",
-    "user = misp.get_user(log['Log']['user_id'])\n",
-    "user['User']['role_id'] = new_role\n",
-    "misp.edit_user(user['User']['id'], **user['User'])"
-   ]
   }
  ],
  "metadata": {