From da4e84399314d14a87dd916c87dd257c440c6897 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 16 Nov 2017 17:47:27 -0500 Subject: [PATCH 001/150] issue #104 with TAXIICollectionSource tweak --- docs/guide/datastore.ipynb | 294 ++++++++++++++++++++++++++++++------- stix2/sources/taxii.py | 24 ++- stix2/utils.py | 2 +- 3 files changed, 261 insertions(+), 59 deletions(-) diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 7fc0997..49ea4fd 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -52,6 +52,19 @@ "print = json_print" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# without this configuration, only last print() call is outputted in cells\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "InteractiveShell.ast_node_interactivity = \"all\"" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -85,50 +98,229 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--797ae2b5-3f7a-44c5-8ecd-33ba22fdc2b5\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--11913f42-2d52-4b9d-842f-94bf06819a66\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "intrusion-set",\n",
+       "    "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:31:53.197Z",\n",
+       "    "modified": "2017-05-31T21:31:53.197Z",\n",
+       "    "name": "DragonOK",\n",
+       "    "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]",\n",
+       "    "aliases": [\n",
+       "        "DragonOK"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "mitre-attack",\n",
+       "            "url": "https://attack.mitre.org/wiki/Group/G0017",\n",
+       "            "external_id": "G0017"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Operation Quantum Entanglement",\n",
+       "            "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.",\n",
+       "            "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Symbiotic APT Groups",\n",
+       "            "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.",\n",
+       "            "url": "https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&D%20Track%20Insight%20into%20Symbiotic%20APT.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "New DragonOK",\n",
+       "            "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.",\n",
+       "            "url": "http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/"\n",
+       "        }\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--02b90f02-a96a-43ee-88f1-1e87297941f2",\n",
+       "    "created": "2017-11-13T07:00:24.000Z",\n",
+       "    "modified": "2017-11-13T07:00:24.000Z",\n",
+       "    "name": "Ransomware IP Blocklist",\n",
+       "    "description": "IP Blocklist address from abuse.ch",\n",
+       "    "pattern": "[ ipv4-addr:value = '91.237.247.24' ]",\n",
+       "    "valid_from": "2017-11-13T07:00:24Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity",\n",
+       "        "Ransomware",\n",
+       "        "Botnet",\n",
+       "        "C&C"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "abuse.ch",\n",
+       "            "url": "https://ransomwaretracker.abuse.ch/blocklist/"\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -136,22 +328,22 @@ "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n", "\n", "# create FileSystemStore\n", - "fs = FileSystemSource(\"/tmp/stix2_data\")\n", + "fs = FileSystemSource(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/\")\n", "\n", "# create TAXIICollectionSource\n", - "colxn = Collection('https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/')\n", + "colxn = Collection('https://test.freetaxii.com:8000/osint/collections/a9c22eaf-0f3e-482c-8bb4-45ae09e75d9b/')\n", "ts = TAXIICollectionSource(colxn)\n", "\n", "# add them both to the CompositeDataSource\n", "cs = CompositeDataSource()\n", - "cs.add_data_sources([fs, ts])\n", + "cs.add_data_sources([fs,ts])\n", "\n", "# get an object that is only in the filesystem\n", - "ta = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", - "print(ta)\n", + "intrusion_set = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", + "print(intrusion_set)\n", "\n", "# get an object that is only in the TAXII collection\n", - "ind = cs.get('indicator--37a6a5de-a5b9-425a-903a-4ae9cbf1ff3f')\n", + "ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n", "print(ind)\n" ] }, @@ -197,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": true }, @@ -231,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": true }, @@ -266,9 +458,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "cti-python-stix2", "language": "python", - "name": "python2" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 8eb5069..512fb58 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,6 +1,7 @@ """ Python STIX 2.x TAXIICollectionStore """ +from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse @@ -121,9 +122,13 @@ class TAXIICollectionSource(DataSource): # dont extract TAXII filters from query (to send to TAXII endpoint) # as directly retrieveing a STIX object by ID - stix_objs = self.collection.get_object(stix_id)["objects"] + try: + stix_objs = self.collection.get_object(stix_id)["objects"] + stix_obj = list(apply_common_filters(stix_objs, query)) - stix_obj = list(apply_common_filters(stix_objs, query)) + except HTTPError: + # if resource not found or access is denied from TAXII server, return None + stix_obj = [] if len(stix_obj): stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) @@ -209,13 +214,18 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters)["objects"] + try: + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] - # deduplicate data (before filtering as reduces wasted filtering) - all_data = deduplicate(all_data) + # deduplicate data (before filtering as reduces wasted filtering) + all_data = deduplicate(all_data) - # apply local (CompositeDataSource, TAXIICollectionSource and query filters) - all_data = list(apply_common_filters(all_data, query)) + # apply local (CompositeDataSource, TAXIICollectionSource and query filters) + all_data = list(apply_common_filters(all_data, query)) + + except HTTPError: + # if resources not found or access is denied from TAXII server, return empty list + all_data = [] # parse python STIX objects from the STIX object dicts stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] diff --git a/stix2/utils.py b/stix2/utils.py index f23dbe2..f0d43f1 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,5 +1,5 @@ """Utility functions and classes for the stix2 library.""" - +# HACK from collections import Mapping import copy import datetime as dt From ec42182cb1616d2c7319ebfc1bf9a6a4c0bc4457 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 17 Nov 2017 12:19:06 -0500 Subject: [PATCH 002/150] issue #107 , also MemorySource.load_from_file() parses JSON into python-stix2 objects now --- docs/guide/memory.ipynb | 578 ++++++++++++++++++++++++++++++++-------- stix2/sources/memory.py | 12 +- 2 files changed, 479 insertions(+), 111 deletions(-) diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index 75c0475..1119a64 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -62,8 +62,9 @@ "\n", "\n", "### Memory API\n", + "A note on adding and retreiving STIX content to the Memory suite. As mentioned, under the hood is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects. While (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", "\n", - "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) and [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). These methods both add STIX content to an internal dictionary (maintained by [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore)). STIX content that is to be added can be in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as, is how it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dictionaries (of STIX objects) are stored as Python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). Also, for [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, individually or in a Bundle. \n", + "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) . For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", "\n", "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", "\n", @@ -74,26 +75,101 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d91ef175-8a82-470a-a610-bbd2ee8a1516\",\n", - " \"created\": \"2017-09-29T19:52:16.930Z\",\n", - " \"modified\": \"2017-09-29T19:52:16.930Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"description\": \"Crusades C2 implant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\",\n", - " \"valid_from\": \"2017-09-29T19:52:16.930909Z\"\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--2f61e4e7-0891-4e09-b79a-66f5e594fec0",\n",
+       "    "created": "2017-11-17T17:01:31.590Z",\n",
+       "    "modified": "2017-11-17T17:01:31.590Z",\n",
+       "    "description": "Crusades C2 implant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']",\n",
+       "    "valid_from": "2017-11-17T17:01:31.590939Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -115,26 +191,101 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--79fdaad7-c461-49bb-ad1d-caa5e9c51c90\",\n", - " \"created\": \"2017-09-29T19:52:17.021Z\",\n", - " \"modified\": \"2017-09-29T19:52:17.021Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"description\": \"Crusades stage 2 implant variant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']\",\n", - " \"valid_from\": \"2017-09-29T19:52:17.021728Z\"\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--ddb765ba-ff1e-4285-bf33-1f6d08f583d6",\n",
+       "    "created": "2017-11-17T17:01:31.799Z",\n",
+       "    "modified": "2017-11-17T17:01:31.799Z",\n",
+       "    "description": "Crusades stage 2 implant variant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']",\n",
+       "    "valid_from": "2017-11-17T17:01:31.799228Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -157,82 +308,209 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": 5, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "-----------------------\n", - "{'name': 'Urban2', 'created': '2017-09-12T13:26:18.023Z', 'labels': ['rootkit'], 'modified': '2017-09-12T13:26:18.023Z', 'type': 'malware', 'id': 'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'}\n", - "-----------------------\n", - "{\n", - " \"type\": \"malware\",\n", - " \"id\": \"malware--2b3dd412-18a5-4e81-8742-4977068eb3eb\",\n", - " \"created\": \"2017-09-29T19:52:17.028Z\",\n", - " \"modified\": \"2017-09-29T19:52:17.028Z\",\n", - " \"name\": \"Alexios\",\n", - " \"labels\": [\n", - " \"rootkit\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "malware",\n",
+       "    "id": "malware--e8170e70-522f-4ec3-aa22-afb55bfad0b0",\n",
+       "    "created": "2017-11-17T17:01:31.806Z",\n",
+       "    "modified": "2017-11-17T17:01:31.806Z",\n",
+       "    "name": "Alexios",\n",
+       "    "labels": [\n",
+       "        "rootkit"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "from stix2 import Filter\n", "\n", - "# add dictionary (of STIX object) to MemoryStore\n", - "# (this dict would assumably come from output of another source,\n", - "# i.e. a loaded json file, NOT manually created as done here for sample purposes)\n", - "\n", - "malware = {\n", - " \"type\": \"malware\",\n", - " \"id\" : \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\",\n", - " \"labels\": [\"rootkit\"],\n", - " \"name\": \"Urban2\",\n", - " \"created\": \"2017-09-12T13:26:18.023Z\",\n", - " \"modified\": \"2017-09-12T13:26:18.023Z\"\n", - "}\n", - "\n", - "mem.add(malware)\n", - "\n", - "results = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])\n", - "for r in results:\n", - " # note that python STIX objects are pretty-printed\n", - " # due to some python dunder method magic, but normal\n", - " # python dictionaries are not by default. Thus the\n", - " # python STIX objects and python STIX dictionaries\n", - " # that match the above query can be easily identified visually\n", - " print(\"-----------------------\")\n", - " print(r)" + "mal = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])[0]\n", + "print(mal)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"report\",\n", - " \"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\n", - " \"created\": \"2017-05-08T18:34:08.042Z\",\n", - " \"modified\": \"2017-05-08T18:34:08.042Z\",\n", - " \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\",\n", - " \"published\": \"2017-05-08T10:24:11.011Z\",\n", - " \"object_refs\": [\n", - " \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"\n", - " ],\n", - " \"labels\": [\n", - " \"threat-report\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "report",\n",
+       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
+       "    "created": "2017-05-08T18:34:08.042Z",\n",
+       "    "modified": "2017-05-08T18:34:08.042Z",\n",
+       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
+       "    "published": "2017-05-08T10:24:11.011Z",\n",
+       "    "object_refs": [\n",
+       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
+       "    ],\n",
+       "    "labels": [\n",
+       "        "threat-report"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -257,15 +535,103 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{u'name': u'The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.', u'created': u'2017-05-08T18:34:08.042Z', u'labels': [u'threat-report'], u'modified': u'2017-05-08T18:34:08.042Z', u'object_refs': [u'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'], u'published': u'2017-05-08T10:24:11.011Z', u'type': u'report', u'id': u'report--2add14d6-bbf3-4308-bb8e-226d314a08e4'}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "report",\n",
+       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
+       "    "created": "2017-05-08T18:34:08.042Z",\n",
+       "    "modified": "2017-05-08T18:34:08.042Z",\n",
+       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
+       "    "published": "2017-05-08T10:24:11.011Z",\n",
+       "    "object_refs": [\n",
+       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
+       "    ],\n",
+       "    "labels": [\n",
+       "        "threat-report"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -280,17 +646,15 @@ "report = mem_2.get(\"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\")\n", "\n", "# for visualpurposes\n", - "# Note: Since STIX content was added to MemoryStore as json,\n", - "# it is maintained as python dictionaries ( as opposed to STIX objects)\n", "print(report)" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "cti-python-stix2", "language": "python", - "name": "python2" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 308d0d0..32a6756 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -1,9 +1,6 @@ """ Python STIX 2.0 Memory Source/Sink -TODO: - Run through tests again, lot of changes. - TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) can save a lot of time for successive queries @@ -302,7 +299,14 @@ class MemorySource(DataSource): return all_data def load_from_file(self, file_path, allow_custom=False, version=None): + """ Load JSON formatted STIX content from file and add to Memory.""" file_path = os.path.abspath(file_path) - stix_data = json.load(open(file_path, "r")) + + # converting the STIX content to JSON encoded string before calling + # _add() so that the STIX content is added as python-stix2 objects + # to the in-memory dict. Otherwise, if you pass a dict to _add(), + # it gets stored as a dict. + stix_data = json.dumps(json.load(open(file_path, "r"))) + _add(self, stix_data, allow_custom=allow_custom, version=version) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ From 0347bb52fdcb7a3d49bedc1659df0774a3257248 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 21 Nov 2017 14:39:50 -0500 Subject: [PATCH 003/150] added remove_custom_stix() utility; tweaked new_version() to make it a little safer --- stix2/utils.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index f0d43f1..974fd94 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -16,6 +16,9 @@ from .exceptions import (InvalidValueError, RevokeError, # timestamps in a single object, the timestamps will vary by a few microseconds. NOW = object() +# STIX object properties that cannot be modified +STIX_UNMOD_PROPERTIES = ["created", "created_by_ref", "id", "type"] + class STIXdatetime(dt.datetime): def __new__(cls, *args, **kwargs): @@ -215,7 +218,7 @@ def new_version(data, **kwargs): properties_to_change = kwargs.keys() # Make sure certain properties aren't trying to change - for prop in ["created", "created_by_ref", "id", "type"]: + for prop in STIX_UNMOD_PROPERTIES: if prop in properties_to_change: unchangable_properties.append(prop) if unchangable_properties: @@ -227,8 +230,11 @@ def new_version(data, **kwargs): elif 'modified' in data: old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond') new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') - if new_modified_property < old_modified_property: - raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") + if new_modified_property <= old_modified_property: + raise InvalidValueError(cls, 'modified', + "The new modified datetime cannot be before the current modified datetime. The new modified time can " + "also not be equal to the current modified datetime because if a consumer receives two objects that are " + "different, but have the same id and modified timestamp, it is not defined how the consumer handles the objects.") new_obj_inner.update(kwargs) # Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass return cls(**{k: v for k, v in new_obj_inner.items() if v is not None}) @@ -255,3 +261,47 @@ def get_class_hierarchy_names(obj): for cls in obj.__class__.__mro__: names.append(cls.__name__) return names + + +def remove_custom_stix(stix_obj): + """remove any custom STIX objects or properties + + Args: + stix_obj (dict OR python-stix obj): a single python-stix object + or dict of a STIX object + """ + + if stix_obj["type"].startswith("x_"): + # if entire object is custom, discard + return None + + custom_props = [] + for prop in stix_obj.items(): + if prop[0].startswith("x_"): + # for every custom property, record it and set value to None + # (so we can pass it to new_version() and it will be dropped) + custom_props.append((prop[0], None)) + + if custom_props: + # obtain set of object properties that can be transferred + # to a new object version. This is 1)custom props with their + # values set to None, and 2)any properties left that are not + # unmodifiable STIX properties or the "modified" property + + # set of properties that are not supplied to new_version() + # to be used for updating properties. This includes unmodifiable + # properties (properties that new_version() just re-uses from the + # existing STIX object) and the "modified" property. We dont supply the + # "modified" property so that new_version() creates a new datetime + # value for this property + non_supplied_props = STIX_UNMOD_PROPERTIES + ["modified"] + + props = [(prop, stix_obj[prop]) for prop in stix_obj if prop not in non_supplied_props] + + # add to set the custom properties we want to get rid of (with their value=None) + props.extend(custom_props) + + return new_version(stix_obj, **(dict(props))) + + else: + return stix_obj From 2399ee62ecee09cfac61dd1ede442fb7bc0d6312 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 Nov 2017 12:03:10 -0500 Subject: [PATCH 004/150] adjusting tests --- stix2/sources/filesystem.py | 66 +++++++++++-------- stix2/sources/memory.py | 120 ++++++++++++++++------------------ stix2/sources/taxii.py | 62 ++++++++++-------- stix2/test/test_filesystem.py | 10 +-- stix2/test/test_memory.py | 102 ++++++++++++++--------------- stix2/test/test_versioning.py | 42 +++++++++++- stix2/utils.py | 11 ++-- 7 files changed, 231 insertions(+), 182 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index e92c525..050ca23 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -1,8 +1,6 @@ """ Python STIX 2.0 FileSystem Source/Sink -TODO: - Test everything """ import json @@ -22,7 +20,12 @@ class FileSystemStore(DataStore): Args: stix_dir (str): path to directory of STIX objects - bundlify (bool): Whether to wrap objects in bundles when saving them. + allow_custom (bool): whether to allow custom STIX content to be + pushed/retrieved. Defaults to True for FileSystemSource side(retrieving data) + and False for FileSystemSink side(pushing data). However, when + parameter is supplied, it will be applied to both FileSystemSource + and FileSystemSink. + bundlify (bool): whether to wrap objects in bundles when saving them. Default: False. Attributes: @@ -30,10 +33,16 @@ class FileSystemStore(DataStore): sink (FileSystemSink): FileSystemSink """ - def __init__(self, stix_dir, bundlify=False): + def __init__(self, stix_dir, allow_custom=None, bundlify=False): + if not allow_custom: + allow_custom_source = True + allow_custom_sink = False + else: + allow_custom_sink = allow_custom_source = allow_custom + super(FileSystemStore, self).__init__( - source=FileSystemSource(stix_dir=stix_dir), - sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + source=FileSystemSource(stix_dir=stix_dir, allow_custom=allow_custom_source), + sink=FileSystemSink(stix_dir=stix_dir, allow_custom=allow_custom_sink, bundlify=bundlify) ) @@ -46,13 +55,16 @@ class FileSystemSink(DataSink): Args: stix_dir (str): path to directory of STIX objects. + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSource. Default: False bundlify (bool): Whether to wrap objects in bundles when saving them. Default: False. """ - def __init__(self, stix_dir, bundlify=False): + def __init__(self, stix_dir, allow_custom=False, bundlify=False): super(FileSystemSink, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.allow_custom = allow_custom self.bundlify = bundlify if not os.path.exists(self._stix_dir): @@ -71,20 +83,18 @@ class FileSystemSink(DataSink): os.makedirs(os.path.dirname(path)) if self.bundlify: - stix_obj = Bundle(stix_obj) + stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom) with open(path, "w") as f: f.write(str(stix_obj)) - def add(self, stix_data=None, allow_custom=False, version=None): + def add(self, stix_data=None, version=None): """Add STIX objects to file directory. Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or list of), dict (or list of), or a STIX 2.0 json encoded string. - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -100,24 +110,24 @@ class FileSystemSink(DataSink): self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) + stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) if stix_data["type"] == "bundle": # extract STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) else: # adding json-formatted STIX - self._check_path_and_write(stix_data) + self._check_path_and_write(stix_data,) elif isinstance(stix_data, Bundle): # recursively add individual STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects for stix_obj in stix_data: - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), " @@ -134,11 +144,14 @@ class FileSystemSource(DataSource): Args: stix_dir (str): path to directory of STIX objects + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSink. Default: True """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, allow_custom=True): super(FileSystemSource, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.allow_custom = allow_custom if not os.path.exists(self._stix_dir): raise ValueError("directory path for STIX data does not exist: %s" % self._stix_dir) @@ -147,15 +160,13 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -167,7 +178,7 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + all_data = self.query(query=query, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -176,7 +187,7 @@ class FileSystemSource(DataSource): return stix_obj - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions @@ -186,8 +197,6 @@ class FileSystemSource(DataSource): stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -197,9 +206,9 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)] + return [self.get(stix_id=stix_id, version=version, _composite_filters=_composite_filters)] - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + def query(self, query=None, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -210,8 +219,6 @@ class FileSystemSource(DataSource): query (list): list of filters to search on _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -221,6 +228,7 @@ class FileSystemSource(DataSource): parsed into a python STIX objects and then returned. """ + all_data = [] if query is None: @@ -304,7 +312,7 @@ class FileSystemSource(DataSource): all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 32a6756..f094f90 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -21,7 +21,7 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None, allow_custom=False, version=None): +def _add(store, stix_data=None, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. @@ -29,8 +29,6 @@ def _add(store, stix_data=None, allow_custom=False, version=None): Args: stix_data (list OR dict OR STIX object): STIX objects to be added - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -43,28 +41,19 @@ def _add(store, stix_data=None, allow_custom=False, version=None): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom, version=version) + _add(store, stix_obj, version=version) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data - elif isinstance(stix_data, str): - # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) - if stix_data["type"] == "bundle": - # recurse on each STIX object in bundle - for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom, version=version) - else: - _add(store, stix_data, allow_custom=allow_custom, version=version) - elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj, allow_custom=allow_custom, version=version) + _add(store, stix_obj, version=version) else: - raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") + raise TypeError("stix_data expected to be a python-stix2 object (or list of), JSON formatted STIX (or list of)," + " or a JSON formatted STIX bundle. stix_data was of type: " + str(type(stix_data))) class MemoryStore(DataStore): @@ -78,8 +67,9 @@ class MemoryStore(DataStore): Args: stix_data (list OR dict OR STIX object): STIX content to be added - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom STIX content. + Only applied when export/input functions called, i.e. + load_from_file() and save_to_file(). Defaults to True. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -89,11 +79,11 @@ class MemoryStore(DataStore): sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False, version=None): + def __init__(self, stix_data=None, allow_custom=True, version=None): self._data = {} if stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) super(MemoryStore, self).__init__( source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True), @@ -101,31 +91,11 @@ class MemoryStore(DataStore): ) def save_to_file(self, *args, **kwargs): - """Write SITX objects from in-memory dictionary to JSON file, as a STIX - Bundle. - - Args: - file_path (str): file path to write STIX data to - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. - - """ + """See MemorySink.save_to_file() for documentation""" return self.sink.save_to_file(*args, **kwargs) def load_from_file(self, *args, **kwargs): - """Load STIX data from JSON file. - - File format is expected to be a single JSON - STIX object or JSON STIX bundle. - - Args: - file_path (str): file path to load STIX data from - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ + """See MemorySource.load_from_file() for documentation""" return self.source.load_from_file(*args, **kwargs) @@ -138,11 +108,12 @@ class MemorySink(DataSink): Args: stix_data (dict OR list): valid STIX 2.0 content in bundle or a list. - _store (bool): if the MemorySink is a part of a DataStore, + _store (bool): whether the MemorySink is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSource. Not user supplied - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom objects/properties + when exporting STIX content to file. + Default: True. Attributes: _data (dict): the in-memory dict that holds STIX objects. @@ -150,25 +121,34 @@ class MemorySink(DataSink): a MemorySource """ - def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): + def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): super(MemorySink, self).__init__() self._data = {} + self.allow_custom = allow_custom if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) - def add(self, stix_data, allow_custom=False, version=None): - _add(self, stix_data, allow_custom=allow_custom, version=version) + def add(self, stix_data, version=None): + _add(self, stix_data, version=version) add.__doc__ = _add.__doc__ - def save_to_file(self, file_path, allow_custom=False): + def save_to_file(self, file_path): + """Write SITX objects from in-memory dictionary to JSON file, as a STIX + Bundle. + + Args: + file_path (str): file path to write STIX data to + + """ file_path = os.path.abspath(file_path) + if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(list(self._data.values()), allow_custom=allow_custom))) + f.write(str(Bundle(list(self._data.values()), allow_custom=self.allow_custom))) save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ @@ -185,8 +165,9 @@ class MemorySource(DataSource): _store (bool): if the MemorySource is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSink. Not user supplied - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom objects/properties + when importing STIX content from file. + Default: True. Attributes: _data (dict): the in-memory dict that holds STIX objects. @@ -194,14 +175,15 @@ class MemorySource(DataSource): a MemorySink """ - def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): + def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): super(MemorySource, self).__init__() self._data = {} + self.allow_custom = allow_custom if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. @@ -257,6 +239,7 @@ class MemorySource(DataSource): is returned in the same form as it as added """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): @@ -298,15 +281,24 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path, allow_custom=False, version=None): - """ Load JSON formatted STIX content from file and add to Memory.""" - file_path = os.path.abspath(file_path) + def load_from_file(self, file_path, version=None): + """Load STIX data from JSON file. - # converting the STIX content to JSON encoded string before calling - # _add() so that the STIX content is added as python-stix2 objects - # to the in-memory dict. Otherwise, if you pass a dict to _add(), - # it gets stored as a dict. - stix_data = json.dumps(json.load(open(file_path, "r"))) + File format is expected to be a single JSON + STIX object or JSON STIX bundle. - _add(self, stix_data, allow_custom=allow_custom, version=version) + Args: + file_path (str): file path to load STIX data from + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ + stix_data = json.load(open(os.path.abspath(file_path), "r")) + + if stix_data["type"] == "bundle": + for stix_obj in stix_data["objects"]: + print(stix_obj) + _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"])) + else: + _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=version)) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 512fb58..5b5c53c 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -19,11 +19,23 @@ class TAXIICollectionStore(DataStore): Args: collection (taxii2.Collection): TAXII Collection instance + allow_custom (bool): whether to allow custom STIX content to be + pushed/retrieved. Defaults to True for TAXIICollectionSource + side(retrieving data) and False for TAXIICollectionSink + side(pushing data). However, when parameter is supplied, it will + be applied to both TAXIICollectionSource/Sink. + """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=None): + if not allow_custom: + allow_custom_source = True + allow_custom_sink = False + else: + allow_custom_sink = allow_custom_source = allow_custom + super(TAXIICollectionStore, self).__init__( - source=TAXIICollectionSource(collection), - sink=TAXIICollectionSink(collection) + source=TAXIICollectionSource(collection, allow_custom=allow_custom_source), + sink=TAXIICollectionSink(collection, allow_custom=allow_custom_sink) ) @@ -33,48 +45,49 @@ class TAXIICollectionSink(DataSink): Args: collection (taxii2.Collection): TAXII2 Collection instance + allow_custom (bool): Whether to allow custom STIX content to be + added to the TAXIICollectionSink. Default: False """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=False): super(TAXIICollectionSink, self).__init__() self.collection = collection + self.allow_custom = allow_custom - def add(self, stix_data, allow_custom=False, version=None): + def add(self, stix_data, version=None): """Add/push STIX content to TAXII Collection endpoint Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0 json encoded string, or list of any of the following - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. """ if isinstance(stix_data, _STIXBase): # adding python STIX object - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) elif isinstance(stix_data, dict): # adding python dict (of either Bundle or STIX obj) if stix_data["type"] == "bundle": bundle = stix_data else: - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj, allow_custom=allow_custom, version=version) + self.add(obj, version=version) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) + stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) else: raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") @@ -88,13 +101,16 @@ class TAXIICollectionSource(DataSource): Args: collection (taxii2.Collection): TAXII Collection instance + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSink. Default: True """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=True): super(TAXIICollectionSource, self).__init__() self.collection = collection + self.allow_custom = allow_custom - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote STIX Collection endpoint. @@ -102,8 +118,6 @@ class TAXIICollectionSource(DataSource): stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -131,7 +145,7 @@ class TAXIICollectionSource(DataSource): stix_obj = [] if len(stix_obj): - stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) + stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version) if stix_obj.id != stix_id: # check - was added to handle erroneous TAXII servers stix_obj = None @@ -140,7 +154,7 @@ class TAXIICollectionSource(DataSource): return stix_obj - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it @@ -148,8 +162,6 @@ class TAXIICollectionSource(DataSource): stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -163,17 +175,17 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json - all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] + all_data = [parse(stix_obj, allow_custom=self.allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + def query(self, query=None, version=None, _composite_filters=None): """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -184,8 +196,6 @@ class TAXIICollectionSource(DataSource): query (list): list of filters to search on _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -228,7 +238,7 @@ class TAXIICollectionSource(DataSource): all_data = [] # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 85f6966..729285a 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -340,7 +340,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id, allow_custom=True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -352,9 +352,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): allow_custom=True) bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle, allow_custom=True) + fs_store.add(bundle) - camp_r = fs_store.get(camp.id, allow_custom=True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -367,9 +367,9 @@ def test_filesystem_custom_object(fs_store): pass newobj = NewObj(property1='something') - fs_store.add(newobj, allow_custom=True) + fs_store.add(newobj) - newobj_r = fs_store.get(newobj.id, allow_custom=True) + newobj_r = fs_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index 6b1219e..d5651d0 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -184,64 +184,64 @@ def test_memory_store_save_load_file(mem_store): shutil.rmtree(os.path.dirname(filename)) +# Currently deprecated - removed functionality from Memory API +# def test_memory_store_add_stix_object_str(mem_store): +# # add stix object string +# camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" +# camp_name = "Aurelius" +# camp_alias = "Purple Robes" +# camp = """{ +# "name": "%s", +# "type": "campaign", +# "objective": "German and French Intelligence Services", +# "aliases": ["%s"], +# "id": "%s", +# "created": "2017-05-31T21:31:53.197755Z" +# }""" % (camp_name, camp_alias, camp_id) +# +# mem_store.add(camp) +# +# camp_r = mem_store.get(camp_id) +# assert camp_r["id"] == camp_id +# assert camp_r["name"] == camp_name +# assert camp_alias in camp_r["aliases"] -def test_memory_store_add_stix_object_str(mem_store): - # add stix object string - camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" - camp_name = "Aurelius" - camp_alias = "Purple Robes" - camp = """{ - "name": "%s", - "type": "campaign", - "objective": "German and French Intelligence Services", - "aliases": ["%s"], - "id": "%s", - "created": "2017-05-31T21:31:53.197755Z" - }""" % (camp_name, camp_alias, camp_id) - - mem_store.add(camp) - - camp_r = mem_store.get(camp_id) - assert camp_r["id"] == camp_id - assert camp_r["name"] == camp_name - assert camp_alias in camp_r["aliases"] - - -def test_memory_store_add_stix_bundle_str(mem_store): - # add stix bundle string - camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" - camp_name = "Atilla" - camp_alias = "Huns" - bund = """{ - "type": "bundle", - "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", - "spec_version": "2.0", - "objects": [ - { - "name": "%s", - "type": "campaign", - "objective": "Bulgarian, Albanian and Romanian Intelligence Services", - "aliases": ["%s"], - "id": "%s", - "created": "2017-05-31T21:31:53.197755Z" - } - ] - }""" % (camp_name, camp_alias, camp_id) - - mem_store.add(bund) - - camp_r = mem_store.get(camp_id) - assert camp_r["id"] == camp_id - assert camp_r["name"] == camp_name - assert camp_alias in camp_r["aliases"] +# Currently deprecated - removed functionality from Memory API +# def test_memory_store_add_stix_bundle_str(mem_store): +# # add stix bundle string +# camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" +# camp_name = "Atilla" +# camp_alias = "Huns" +# bund = """{ +# "type": "bundle", +# "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", +# "spec_version": "2.0", +# "objects": [ +# { +# "name": "%s", +# "type": "campaign", +# "objective": "Bulgarian, Albanian and Romanian Intelligence Services", +# "aliases": ["%s"], +# "id": "%s", +# "created": "2017-05-31T21:31:53.197755Z" +# } +# ] +# }""" % (camp_name, camp_alias, camp_id) +# +# mem_store.add(bund) +# +# camp_r = mem_store.get(camp_id) +# assert camp_r["id"] == camp_id +# assert camp_r["name"] == camp_name +# assert camp_alias in camp_r["aliases"] def test_memory_store_add_invalid_object(mem_store): ind = ('indicator', IND1) # tuple isn't valid with pytest.raises(TypeError) as excinfo: mem_store.add(ind) - assert 'stix_data must be' in str(excinfo.value) - assert 'a STIX object' in str(excinfo.value) + assert 'stix_data expected to be' in str(excinfo.value) + assert 'a python-stix2 object' in str(excinfo.value) assert 'JSON formatted STIX' in str(excinfo.value) assert 'JSON formatted STIX bundle' in str(excinfo.value) diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 8695a30..233587e 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -88,11 +88,15 @@ def test_versioning_error_bad_modified_value(): assert excinfo.value.cls == stix2.Campaign assert excinfo.value.prop_name == "modified" - assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \ + "It cannot be equal, as according to STIX 2 specification, objects that are different " \ + "but have the same id and modified timestamp do not have defined consumer behavior." msg = "Invalid value for {0} '{1}': {2}" msg = msg.format(stix2.Campaign.__name__, "modified", - "The new modified datetime cannot be before the current modified datatime.") + "The new modified datetime cannot be before than or equal to the current modified datetime." + "It cannot be equal, as according to STIX 2 specification, objects that are different " + "but have the same id and modified timestamp do not have defined consumer behavior.") assert str(excinfo.value) == msg @@ -153,7 +157,9 @@ def test_versioning_error_dict_bad_modified_value(): assert excinfo.value.cls == dict assert excinfo.value.prop_name == "modified" - assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \ + "It cannot be equal, as according to STIX 2 specification, objects that are different " \ + "but have the same id and modified timestamp do not have defined consumer behavior." def test_versioning_error_dict_no_modified_value(): @@ -206,3 +212,33 @@ def test_revoke_invalid_cls(): stix2.utils.revoke(campaign_v1) assert 'cannot revoke object of this type' in str(excinfo.value) + + +def test_remove_custom_stix_property(): + mal = stix2.Malware(name="ColePowers", + labels=["rootkit"], + x_custom="armada", + allow_custom=True) + + mal_nc = stix2.utils.remove_custom_stix(mal) + + assert "x_custom" not in mal_nc + assert stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], + precision="millisecond") + + +def test_remove_custom_stix_object(): + @stix2.CustomObject("x-animal", [ + ("species", stix2.properties.StringProperty(required=True)), + ("animal_class", stix2.properties.StringProperty()), + ]) + class Animal(object): + def __init__(self, animal_class=None, **kwargs): + if animal_class and animal_class not in ["mammal", "bird"]: + raise ValueError("Not a recognized class of animal") + + animal = Animal(species="lion", animal_class="mammal") + + nc = stix2.utils.remove_custom_stix(animal) + + assert nc is None diff --git a/stix2/utils.py b/stix2/utils.py index 974fd94..10de8b0 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -232,9 +232,9 @@ def new_version(data, **kwargs): new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') if new_modified_property <= old_modified_property: raise InvalidValueError(cls, 'modified', - "The new modified datetime cannot be before the current modified datetime. The new modified time can " - "also not be equal to the current modified datetime because if a consumer receives two objects that are " - "different, but have the same id and modified timestamp, it is not defined how the consumer handles the objects.") + "The new modified datetime cannot be before than or equal to the current modified datetime." + "It cannot be equal, as according to STIX 2 specification, objects that are different " + "but have the same id and modified timestamp do not have defined consumer behavior.") new_obj_inner.update(kwargs) # Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass return cls(**{k: v for k, v in new_obj_inner.items() if v is not None}) @@ -269,9 +269,12 @@ def remove_custom_stix(stix_obj): Args: stix_obj (dict OR python-stix obj): a single python-stix object or dict of a STIX object + + Returns: + A new version of the object with any custom content removed """ - if stix_obj["type"].startswith("x_"): + if stix_obj["type"].startswith("x-"): # if entire object is custom, discard return None From 52a052d4dd4efd4830eb516f9ac102a909d26bb3 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 Nov 2017 12:27:20 -0500 Subject: [PATCH 005/150] removing debug lines; attemting to fix a Travis test error --- stix2/sources/memory.py | 1 - stix2/test/test_versioning.py | 4 ++-- stix2/utils.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index f094f90..0c208a8 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -297,7 +297,6 @@ class MemorySource(DataSource): if stix_data["type"] == "bundle": for stix_obj in stix_data["objects"]: - print(stix_obj) _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"])) else: _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=version)) diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 233587e..dcbd3d3 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -223,8 +223,8 @@ def test_remove_custom_stix_property(): mal_nc = stix2.utils.remove_custom_stix(mal) assert "x_custom" not in mal_nc - assert stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], - precision="millisecond") + assert stix2.utils.parse_into_datetime(mal["modified"], precision="microsecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], + precision="microsecond") def test_remove_custom_stix_object(): diff --git a/stix2/utils.py b/stix2/utils.py index 10de8b0..3129a5c 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,5 +1,4 @@ """Utility functions and classes for the stix2 library.""" -# HACK from collections import Mapping import copy import datetime as dt From 742d249ee03cc7069f10c11a15b05996f3b45a76 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 29 Nov 2017 12:50:13 -0500 Subject: [PATCH 006/150] trying to fix timing bug --- stix2/test/test_versioning.py | 4 ++-- stix2/utils.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index dcbd3d3..233587e 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -223,8 +223,8 @@ def test_remove_custom_stix_property(): mal_nc = stix2.utils.remove_custom_stix(mal) assert "x_custom" not in mal_nc - assert stix2.utils.parse_into_datetime(mal["modified"], precision="microsecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], - precision="microsecond") + assert stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], + precision="millisecond") def test_remove_custom_stix_object(): diff --git a/stix2/utils.py b/stix2/utils.py index 3129a5c..701fe56 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -303,7 +303,14 @@ def remove_custom_stix(stix_obj): # add to set the custom properties we want to get rid of (with their value=None) props.extend(custom_props) - return new_version(stix_obj, **(dict(props))) + new_obj = new_version(stix_obj, **(dict(props))) + + while parse_into_datetime(new_obj["modified"]) == parse_into_datetime(stix_obj["modified"]): + # Prevents bug when fast computation allows multiple STIX object + # versions to be created in single unit of time + new_obj = new_version(stix_obj, **(dict(props))) + + return new_obj else: return stix_obj From 0c8dd7f47c524065ef49b2a626457fe2ac10a2c2 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 1 Dec 2017 11:30:41 -0500 Subject: [PATCH 007/150] added docstring warning for utility function --- stix2/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stix2/utils.py b/stix2/utils.py index 701fe56..86532b3 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -265,6 +265,15 @@ def get_class_hierarchy_names(obj): def remove_custom_stix(stix_obj): """remove any custom STIX objects or properties + Warning: This function is a best effort utility, in that + it will remove custom objects and properties based on the + type names; i.e. if "x-" prefixes object types, and "x_" + prefixes property types. According to the STIX2 spec, + those naming conventions are a SHOULDs not MUSTs, meaning + that valid custom STIX content may ignore those conventions + and in effect render this utility function invalid when used + on that STIX content. + Args: stix_obj (dict OR python-stix obj): a single python-stix object or dict of a STIX object From f1813f17e09adc6b6db26472299650de1e184d86 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 20 Feb 2018 12:16:15 -0500 Subject: [PATCH 008/150] Fix a couple links in documentation Also upgrade nbsphinx version. --- docs/api_ref.rst | 2 +- docs/guide/creating.ipynb | 2 +- requirements.txt | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/api_ref.rst b/docs/api_ref.rst index bb60081..ffc328c 100644 --- a/docs/api_ref.rst +++ b/docs/api_ref.rst @@ -7,6 +7,6 @@ functions in the ``stix2`` API, as given by the package's docstrings. .. note:: All the classes and functions detailed in the pages below are importable directly from `stix2`. See also: - :ref:`How imports will work `. + :ref:`How imports will work `. .. automodule:: stix2 diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 5836d1d..3e22061 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -381,7 +381,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To update the properties of an object, see the [Versioning](versioning.ipynb) section." + "To update the properties of an object, see the [Versioning](guide/versioning.ipynb) section." ] }, { diff --git a/requirements.txt b/requirements.txt index 0de31a7..d6abb63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ bumpversion -nbsphinx>=0.2.15 +ipython +nbsphinx>=0.3.0 pre-commit pytest pytest-cov From 3263c3848f3b1d34a4dbb68dc2d4f8497a1da14d Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 20 Feb 2018 14:48:53 -0500 Subject: [PATCH 009/150] List STIX objects' properties in documentation --- docs/conf.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index c337e79..50aef1d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,11 @@ import os import sys +from six import class_types +from sphinx.ext.autodoc import ClassDocumenter + +from stix2.base import _STIXBase + sys.path.insert(0, os.path.abspath('..')) extensions = [ @@ -50,3 +55,32 @@ latex_elements = {} latex_documents = [ (master_doc, 'stix2.tex', 'stix2 Documentation', 'OASIS', 'manual'), ] + +class STIXAttributeDocumenter(ClassDocumenter): + """Custom Sphinx extension to auto-document STIX properties. + + Needed because descendants of _STIXBase use `_properties` dictionaries + instead of instance variables for STIX 2 objects' properties. + + """ + objtype = "stixattr" + directivetype = "class" + priority = 999 + + @classmethod + def can_document_member(cls, member, membername, isattr, parent): + return isinstance(member, class_types) and \ + issubclass(member, _STIXBase) and \ + hasattr(member, '_properties') + + def add_content(self, more_content, no_docstring=False): + ClassDocumenter.add_content(self, more_content, no_docstring) + + obj = self.object + self.add_line(":Properties:", "") + for prop in obj._properties: + self.add_line(" - %s" % prop, "") + self.add_line("", "") + +def setup(app): + app.add_autodocumenter(STIXAttributeDocumenter) From 6f35978ba659399502d886134dba8ec4bddbd102 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 20 Feb 2018 16:41:19 -0500 Subject: [PATCH 010/150] Document STIX object properties' metadata (i.e. property type, whether it is required) --- docs/conf.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 50aef1d..afc2f68 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,5 @@ import os +import re import sys from six import class_types @@ -56,15 +57,25 @@ latex_documents = [ (master_doc, 'stix2.tex', 'stix2 Documentation', 'OASIS', 'manual'), ] +def get_property_type(prop): + try: + prop_class = prop.__name__ + except AttributeError: + prop_class = prop.__class__.__name__ + prop_class = prop_class.split('Property')[0] + split_camelcase = re.sub('(?!^)([A-Z][a-z]+)', r' \1', prop_class).split() + prop_class = ' '.join(split_camelcase) + return prop_class + class STIXAttributeDocumenter(ClassDocumenter): - """Custom Sphinx extension to auto-document STIX properties. + '''Custom Sphinx extension to auto-document STIX properties. Needed because descendants of _STIXBase use `_properties` dictionaries instead of instance variables for STIX 2 objects' properties. - """ - objtype = "stixattr" - directivetype = "class" + ''' + objtype = 'stixattr' + directivetype = 'class' priority = 999 @classmethod @@ -77,10 +88,17 @@ class STIXAttributeDocumenter(ClassDocumenter): ClassDocumenter.add_content(self, more_content, no_docstring) obj = self.object - self.add_line(":Properties:", "") - for prop in obj._properties: - self.add_line(" - %s" % prop, "") - self.add_line("", "") + self.add_line(':Properties:', '') + for prop_name, prop in obj._properties.items(): + # Add metadata about the property + prop_type = get_property_type(prop) + if prop_type == 'List': + prop_type = 'List of %ss' % get_property_type(prop.contained) + if prop.required: + prop_type += ', required' + prop_str = '**%s** (*%s*)' % (prop_name, prop_type) + self.add_line(' - %s' % prop_str, '') + self.add_line('', '') def setup(app): app.add_autodocumenter(STIXAttributeDocumenter) From 2886ae9961c026b0a3888de6b52b2e253f74d786 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 21 Feb 2018 16:42:25 -0500 Subject: [PATCH 011/150] Add links to specs in STIX Objects' documentation --- stix2/core.py | 3 ++ stix2/v20/common.py | 26 +++++++++- stix2/v20/observables.py | 108 +++++++++++++++++++++++++++++++++++++++ stix2/v20/sdo.py | 36 +++++++++++++ stix2/v20/sro.py | 7 +++ 5 files changed, 179 insertions(+), 1 deletion(-) diff --git a/stix2/core.py b/stix2/core.py index 658bb47..b6d295d 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -41,6 +41,9 @@ class STIXObjectProperty(Property): class Bundle(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'bundle' _properties = OrderedDict() diff --git a/stix2/v20/common.py b/stix2/v20/common.py index ef45060..e915df6 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -11,6 +11,10 @@ from ..utils import NOW, get_dict class ExternalReference(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _properties = OrderedDict() _properties.update([ ('source_name', StringProperty(required=True)), @@ -26,6 +30,10 @@ class ExternalReference(_STIXBase): class KillChainPhase(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _properties = OrderedDict() _properties.update([ ('kill_chain_name', StringProperty(required=True)), @@ -34,6 +42,10 @@ class KillChainPhase(_STIXBase): class GranularMarking(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _properties = OrderedDict() _properties.update([ ('marking_ref', ReferenceProperty(required=True, type="marking-definition")), @@ -42,15 +54,23 @@ class GranularMarking(_STIXBase): class TLPMarking(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + # TODO: don't allow the creation of any other TLPMarkings than the ones below _type = 'tlp' _properties = OrderedDict() _properties.update([ - ('tlp', Property(required=True)) + ('tlp', StringProperty(required=True)) ]) class StatementMarking(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _type = 'statement' _properties = OrderedDict() _properties.update([ @@ -78,6 +98,10 @@ class MarkingProperty(Property): class MarkingDefinition(_STIXBase, _MarkingsMixin): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _type = 'marking-definition' _properties = OrderedDict() _properties.update([ diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index a874df9..1adde90 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -74,6 +74,9 @@ class ExtensionsProperty(DictionaryProperty): class Artifact(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'artifact' _properties = OrderedDict() @@ -93,6 +96,9 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'autonomous-system' _properties = OrderedDict() @@ -106,6 +112,9 @@ class AutonomousSystem(_Observable): class Directory(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'directory' _properties = OrderedDict() @@ -123,6 +132,9 @@ class Directory(_Observable): class DomainName(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'domain-name' _properties = OrderedDict() @@ -135,6 +147,9 @@ class DomainName(_Observable): class EmailAddress(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'email-addr' _properties = OrderedDict() @@ -148,6 +163,9 @@ class EmailAddress(_Observable): class EmailMIMEComponent(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _properties = OrderedDict() _properties.update([ @@ -163,6 +181,9 @@ class EmailMIMEComponent(_STIXBase): class EmailMessage(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'email-message' _properties = OrderedDict() @@ -194,6 +215,9 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'archive-ext' _properties = OrderedDict() @@ -205,6 +229,9 @@ class ArchiveExt(_Extension): class AlternateDataStream(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _properties = OrderedDict() _properties.update([ @@ -215,6 +242,9 @@ class AlternateDataStream(_STIXBase): class NTFSExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'ntfs-ext' _properties = OrderedDict() @@ -225,6 +255,9 @@ class NTFSExt(_Extension): class PDFExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'pdf-ext' _properties = OrderedDict() @@ -238,6 +271,9 @@ class PDFExt(_Extension): class RasterImageExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'raster-image-ext' _properties = OrderedDict() @@ -251,6 +287,9 @@ class RasterImageExt(_Extension): class WindowsPEOptionalHeaderType(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _properties = OrderedDict() _properties.update([ @@ -293,6 +332,9 @@ class WindowsPEOptionalHeaderType(_STIXBase): class WindowsPESection(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _properties = OrderedDict() _properties.update([ @@ -304,6 +346,9 @@ class WindowsPESection(_STIXBase): class WindowsPEBinaryExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'windows-pebinary-ext' _properties = OrderedDict() @@ -324,6 +369,9 @@ class WindowsPEBinaryExt(_Extension): class File(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'file' _properties = OrderedDict() @@ -355,6 +403,9 @@ class File(_Observable): class IPv4Address(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'ipv4-addr' _properties = OrderedDict() @@ -368,6 +419,9 @@ class IPv4Address(_Observable): class IPv6Address(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'ipv6-addr' _properties = OrderedDict() @@ -381,6 +435,9 @@ class IPv6Address(_Observable): class MACAddress(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'mac-addr' _properties = OrderedDict() @@ -392,6 +449,9 @@ class MACAddress(_Observable): class Mutex(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'mutex' _properties = OrderedDict() @@ -403,6 +463,9 @@ class Mutex(_Observable): class HTTPRequestExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'http-request-ext' _properties = OrderedDict() @@ -417,6 +480,9 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'icmp-ext' _properties = OrderedDict() @@ -427,6 +493,9 @@ class ICMPExt(_Extension): class SocketExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'socket-ext' _properties = OrderedDict() @@ -465,6 +534,9 @@ class SocketExt(_Extension): class TCPExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'tcp-ext' _properties = OrderedDict() @@ -475,6 +547,9 @@ class TCPExt(_Extension): class NetworkTraffic(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'network-traffic' _properties = OrderedDict() @@ -506,6 +581,9 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'windows-process-ext' _properties = OrderedDict() @@ -520,6 +598,9 @@ class WindowsProcessExt(_Extension): class WindowsServiceExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'windows-service-ext' _properties = OrderedDict() @@ -555,6 +636,9 @@ class WindowsServiceExt(_Extension): class Process(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'process' _properties = OrderedDict() @@ -593,6 +677,9 @@ class Process(_Observable): class Software(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'software' _properties = OrderedDict() @@ -608,6 +695,9 @@ class Software(_Observable): class URL(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'url' _properties = OrderedDict() @@ -619,6 +709,9 @@ class URL(_Observable): class UNIXAccountExt(_Extension): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'unix-account-ext' _properties = OrderedDict() @@ -631,6 +724,9 @@ class UNIXAccountExt(_Extension): class UserAccount(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'user-account' _properties = OrderedDict() @@ -654,6 +750,9 @@ class UserAccount(_Observable): class WindowsRegistryValueType(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'windows-registry-value-type' _properties = OrderedDict() @@ -679,6 +778,9 @@ class WindowsRegistryValueType(_STIXBase): class WindowsRegistryKey(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'windows-registry-key' _properties = OrderedDict() @@ -700,6 +802,9 @@ class WindowsRegistryKey(_Observable): class X509V3ExtenstionsType(_STIXBase): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'x509-v3-extensions-type' _properties = OrderedDict() @@ -724,6 +829,9 @@ class X509V3ExtenstionsType(_STIXBase): class X509Certificate(_Observable): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'x509-certificate' _properties = OrderedDict() diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 2d36aaf..7ccc3e3 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -19,6 +19,9 @@ class STIXDomainObject(_STIXBase, _MarkingsMixin): class AttackPattern(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'attack-pattern' _properties = OrderedDict() @@ -40,6 +43,9 @@ class AttackPattern(STIXDomainObject): class Campaign(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'campaign' _properties = OrderedDict() @@ -64,6 +70,9 @@ class Campaign(STIXDomainObject): class CourseOfAction(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'course-of-action' _properties = OrderedDict() @@ -84,6 +93,9 @@ class CourseOfAction(STIXDomainObject): class Identity(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'identity' _properties = OrderedDict() @@ -107,6 +119,9 @@ class Identity(STIXDomainObject): class Indicator(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'indicator' _properties = OrderedDict() @@ -131,6 +146,9 @@ class Indicator(STIXDomainObject): class IntrusionSet(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'intrusion-set' _properties = OrderedDict() @@ -158,6 +176,9 @@ class IntrusionSet(STIXDomainObject): class Malware(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'malware' _properties = OrderedDict() @@ -179,6 +200,9 @@ class Malware(STIXDomainObject): class ObservedData(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'observed-data' _properties = OrderedDict() @@ -201,6 +225,9 @@ class ObservedData(STIXDomainObject): class Report(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'report' _properties = OrderedDict() @@ -223,6 +250,9 @@ class Report(STIXDomainObject): class ThreatActor(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'threat-actor' _properties = OrderedDict() @@ -251,6 +281,9 @@ class ThreatActor(STIXDomainObject): class Tool(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'tool' _properties = OrderedDict() @@ -273,6 +306,9 @@ class Tool(STIXDomainObject): class Vulnerability(STIXDomainObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'vulnerability' _properties = OrderedDict() diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index 7f05f5e..7d7d3ae 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -16,6 +16,9 @@ class STIXRelationshipObject(_STIXBase, _MarkingsMixin): class Relationship(STIXRelationshipObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ _type = 'relationship' _properties = OrderedDict() @@ -51,6 +54,10 @@ class Relationship(STIXRelationshipObject): class Sighting(STIXRelationshipObject): + """For more detailed information on this object's properties, see + `the STIX 2.0 specification `__. + """ + _type = 'sighting' _properties = OrderedDict() _properties.update([ From aaff2a3a048b145016dd89a90c903d232fe2a794 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 22 Feb 2018 09:55:15 -0500 Subject: [PATCH 012/150] Appease pycodestyle --- docs/conf.py | 12 +++++-- stix2/v20/observables.py | 72 ++++++++++++++++++++-------------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index afc2f68..dbed6a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,23 +57,30 @@ latex_documents = [ (master_doc, 'stix2.tex', 'stix2 Documentation', 'OASIS', 'manual'), ] + def get_property_type(prop): + """Convert property classname into pretty string name of property. + + """ try: prop_class = prop.__name__ except AttributeError: prop_class = prop.__class__.__name__ + # Remove 'Property' from the string prop_class = prop_class.split('Property')[0] + # Split camelcase with spaces split_camelcase = re.sub('(?!^)([A-Z][a-z]+)', r' \1', prop_class).split() prop_class = ' '.join(split_camelcase) return prop_class + class STIXAttributeDocumenter(ClassDocumenter): - '''Custom Sphinx extension to auto-document STIX properties. + """Custom Sphinx extension to auto-document STIX properties. Needed because descendants of _STIXBase use `_properties` dictionaries instead of instance variables for STIX 2 objects' properties. - ''' + """ objtype = 'stixattr' directivetype = 'class' priority = 999 @@ -100,5 +107,6 @@ class STIXAttributeDocumenter(ClassDocumenter): self.add_line(' - %s' % prop_str, '') self.add_line('', '') + def setup(app): app.add_autodocumenter(STIXAttributeDocumenter) diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 1adde90..83600b0 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -76,7 +76,7 @@ class ExtensionsProperty(DictionaryProperty): class Artifact(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'artifact' _properties = OrderedDict() @@ -98,7 +98,7 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'autonomous-system' _properties = OrderedDict() @@ -114,7 +114,7 @@ class AutonomousSystem(_Observable): class Directory(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'directory' _properties = OrderedDict() @@ -134,7 +134,7 @@ class Directory(_Observable): class DomainName(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'domain-name' _properties = OrderedDict() @@ -149,7 +149,7 @@ class DomainName(_Observable): class EmailAddress(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'email-addr' _properties = OrderedDict() @@ -165,7 +165,7 @@ class EmailAddress(_Observable): class EmailMIMEComponent(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _properties = OrderedDict() _properties.update([ @@ -183,7 +183,7 @@ class EmailMIMEComponent(_STIXBase): class EmailMessage(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'email-message' _properties = OrderedDict() @@ -217,7 +217,7 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'archive-ext' _properties = OrderedDict() @@ -231,7 +231,7 @@ class ArchiveExt(_Extension): class AlternateDataStream(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _properties = OrderedDict() _properties.update([ @@ -244,7 +244,7 @@ class AlternateDataStream(_STIXBase): class NTFSExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'ntfs-ext' _properties = OrderedDict() @@ -257,7 +257,7 @@ class NTFSExt(_Extension): class PDFExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'pdf-ext' _properties = OrderedDict() @@ -273,7 +273,7 @@ class PDFExt(_Extension): class RasterImageExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'raster-image-ext' _properties = OrderedDict() @@ -289,7 +289,7 @@ class RasterImageExt(_Extension): class WindowsPEOptionalHeaderType(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _properties = OrderedDict() _properties.update([ @@ -334,7 +334,7 @@ class WindowsPEOptionalHeaderType(_STIXBase): class WindowsPESection(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _properties = OrderedDict() _properties.update([ @@ -348,7 +348,7 @@ class WindowsPESection(_STIXBase): class WindowsPEBinaryExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'windows-pebinary-ext' _properties = OrderedDict() @@ -371,7 +371,7 @@ class WindowsPEBinaryExt(_Extension): class File(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'file' _properties = OrderedDict() @@ -405,7 +405,7 @@ class File(_Observable): class IPv4Address(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'ipv4-addr' _properties = OrderedDict() @@ -421,7 +421,7 @@ class IPv4Address(_Observable): class IPv6Address(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'ipv6-addr' _properties = OrderedDict() @@ -437,7 +437,7 @@ class IPv6Address(_Observable): class MACAddress(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'mac-addr' _properties = OrderedDict() @@ -451,7 +451,7 @@ class MACAddress(_Observable): class Mutex(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'mutex' _properties = OrderedDict() @@ -465,7 +465,7 @@ class Mutex(_Observable): class HTTPRequestExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'http-request-ext' _properties = OrderedDict() @@ -482,7 +482,7 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'icmp-ext' _properties = OrderedDict() @@ -495,7 +495,7 @@ class ICMPExt(_Extension): class SocketExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'socket-ext' _properties = OrderedDict() @@ -536,7 +536,7 @@ class SocketExt(_Extension): class TCPExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'tcp-ext' _properties = OrderedDict() @@ -549,7 +549,7 @@ class TCPExt(_Extension): class NetworkTraffic(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'network-traffic' _properties = OrderedDict() @@ -583,7 +583,7 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'windows-process-ext' _properties = OrderedDict() @@ -600,7 +600,7 @@ class WindowsProcessExt(_Extension): class WindowsServiceExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'windows-service-ext' _properties = OrderedDict() @@ -638,7 +638,7 @@ class WindowsServiceExt(_Extension): class Process(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'process' _properties = OrderedDict() @@ -679,7 +679,7 @@ class Process(_Observable): class Software(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'software' _properties = OrderedDict() @@ -697,7 +697,7 @@ class Software(_Observable): class URL(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'url' _properties = OrderedDict() @@ -711,7 +711,7 @@ class URL(_Observable): class UNIXAccountExt(_Extension): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'unix-account-ext' _properties = OrderedDict() @@ -726,7 +726,7 @@ class UNIXAccountExt(_Extension): class UserAccount(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'user-account' _properties = OrderedDict() @@ -752,7 +752,7 @@ class UserAccount(_Observable): class WindowsRegistryValueType(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'windows-registry-value-type' _properties = OrderedDict() @@ -780,7 +780,7 @@ class WindowsRegistryValueType(_STIXBase): class WindowsRegistryKey(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'windows-registry-key' _properties = OrderedDict() @@ -804,7 +804,7 @@ class WindowsRegistryKey(_Observable): class X509V3ExtenstionsType(_STIXBase): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'x509-v3-extensions-type' _properties = OrderedDict() @@ -831,7 +831,7 @@ class X509V3ExtenstionsType(_STIXBase): class X509Certificate(_Observable): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. - """ + """ # noqa _type = 'x509-certificate' _properties = OrderedDict() From e23d93720419838e90e5361125669f263810606d Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 23 Feb 2018 09:37:14 -0500 Subject: [PATCH 013/150] Fix isort --- .isort.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/.isort.cfg b/.isort.cfg index d535851..0fadb83 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -8,6 +8,7 @@ known_third_party = requests, simplejson six, + sphinx, stix2patterns, taxii2client, known_first_party = stix2 From df57f4d1593649e6627cc7cc31c15635c4ce581f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 26 Feb 2018 16:56:24 -0500 Subject: [PATCH 014/150] Fix typos in datasources --- docs/guide/filesystem.ipynb | 8 ++++---- docs/guide/memory.ipynb | 8 ++++---- docs/guide/taxii.ipynb | 4 ++-- stix2/sources/filesystem.py | 2 +- stix2/sources/memory.py | 6 ++---- stix2/sources/taxii.py | 2 +- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index 4b5bd6f..f494e6e 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -82,7 +82,7 @@ " /STIX2 Domain Object type\n", "```\n", "\n", - "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", + "The master STIX2 content directory contains subdirectories, each of which aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\", etc.). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", "\n", "```\n", "stix2_content/\n", @@ -109,11 +109,11 @@ "\n", "[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n", "\n", - "Use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", + "For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", "\n", "### FileSystem API\n", "\n", - "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query). The format of the STIX2 content targeted by the FileSystem suite is JSON files. When STIX2 content (in JSON) is retrieved by the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", + "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n", "\n", "A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", "\n", @@ -264,7 +264,7 @@ "source": [ "from stix2 import FileSystemSource\n", "\"\"\"\n", - "Working with FileSystemSource for retrieveing STIX content.\n", + "Working with FileSystemSource for retrieving STIX content.\n", "\"\"\"\n", "\n", "# create FileSystemSource\n", diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index 1119a64..d651525 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -62,11 +62,11 @@ "\n", "\n", "### Memory API\n", - "A note on adding and retreiving STIX content to the Memory suite. As mentioned, under the hood is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects. While (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", + "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", "\n", - "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) . For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", + "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file): For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", "\n", - "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", + "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", "\n", "### Memory Examples\n", "\n", @@ -517,7 +517,7 @@ "from stix2 import Filter\n", "\n", "# add json formatted string to MemoryStore\n", - "# Again, would NOT manual create json-formatted string\n", + "# Again, would NOT manually create json-formatted string\n", "# but taken as an output form from another source\n", "report = '{\"type\": \"report\",\"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\"labels\": [\"threat-report\"], \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\", \"published\": \"2017-05-08T10:24:11.011Z\", \"object_refs\":[\"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"], \"created\": \"2017-05-08T18:34:08.042Z\", \"modified\": \"2017-05-08T18:34:08.042Z\"}'\n", "\n", diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 2f8905b..b0f0cea 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -58,9 +58,9 @@ "source": [ "## TAXIICollection\n", "\n", - "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) for pushing and retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) for retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", + "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) pushes and retrieves STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) retrieves STIX content from local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) pushes STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", "\n", - "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", + "A note on TAXII2 searching/filtering of STIX content: TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", "\n", "### TAXIICollection API\n", "\n", diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 050ca23..f4311be 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -34,7 +34,7 @@ class FileSystemStore(DataStore): """ def __init__(self, stix_dir, allow_custom=None, bundlify=False): - if not allow_custom: + if allow_custom is None: allow_custom_source = True allow_custom_sink = False else: diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0c208a8..b1e61e4 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -117,8 +117,7 @@ class MemorySink(DataSink): Attributes: _data (dict): the in-memory dict that holds STIX objects. - If apart of a MemoryStore, dict is shared between with - a MemorySource + If part of a MemoryStore, the dict is shared with a MemorySource """ def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): @@ -171,8 +170,7 @@ class MemorySource(DataSource): Attributes: _data (dict): the in-memory dict that holds STIX objects. - If apart of a MemoryStore, dict is shared between with - a MemorySink + If part of a MemoryStore, the dict is shared with a MemorySink """ def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 5b5c53c..2d54725 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -27,7 +27,7 @@ class TAXIICollectionStore(DataStore): """ def __init__(self, collection, allow_custom=None): - if not allow_custom: + if allow_custom is None: allow_custom_source = True allow_custom_sink = False else: From 7bdf786f89c8ade623b89a9668cc986583e19566 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 27 Feb 2018 12:17:26 -0500 Subject: [PATCH 015/150] Remove deprecated tests --- stix2/test/test_memory.py | 51 --------------------------------------- 1 file changed, 51 deletions(-) diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index e290ca9..ad78611 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -202,57 +202,6 @@ def test_memory_store_save_load_file(mem_store): shutil.rmtree(os.path.dirname(filename)) -# Currently deprecated - removed functionality from Memory API -# def test_memory_store_add_stix_object_str(mem_store): -# # add stix object string -# camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" -# camp_name = "Aurelius" -# camp_alias = "Purple Robes" -# camp = """{ -# "name": "%s", -# "type": "campaign", -# "objective": "German and French Intelligence Services", -# "aliases": ["%s"], -# "id": "%s", -# "created": "2017-05-31T21:31:53.197755Z" -# }""" % (camp_name, camp_alias, camp_id) -# -# mem_store.add(camp) -# -# camp_r = mem_store.get(camp_id) -# assert camp_r["id"] == camp_id -# assert camp_r["name"] == camp_name -# assert camp_alias in camp_r["aliases"] - -# Currently deprecated - removed functionality from Memory API -# def test_memory_store_add_stix_bundle_str(mem_store): -# # add stix bundle string -# camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" -# camp_name = "Atilla" -# camp_alias = "Huns" -# bund = """{ -# "type": "bundle", -# "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", -# "spec_version": "2.0", -# "objects": [ -# { -# "name": "%s", -# "type": "campaign", -# "objective": "Bulgarian, Albanian and Romanian Intelligence Services", -# "aliases": ["%s"], -# "id": "%s", -# "created": "2017-05-31T21:31:53.197755Z" -# } -# ] -# }""" % (camp_name, camp_alias, camp_id) -# -# mem_store.add(bund) -# -# camp_r = mem_store.get(camp_id) -# assert camp_r["id"] == camp_id -# assert camp_r["name"] == camp_name -# assert camp_alias in camp_r["aliases"] - def test_memory_store_add_invalid_object(mem_store): ind = ('indicator', IND1) # tuple isn't valid From b6f14cdd4dea62576f27234f0810e8ae5104a115 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 27 Feb 2018 16:53:05 -0500 Subject: [PATCH 016/150] Fix MemoryStore docstrings --- stix2/sources/memory.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index b1e61e4..5d08d7c 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -91,11 +91,27 @@ class MemoryStore(DataStore): ) def save_to_file(self, *args, **kwargs): - """See MemorySink.save_to_file() for documentation""" + """Write SITX objects from in-memory dictionary to JSON file, as a STIX + Bundle. + + Args: + file_path (str): file path to write STIX data to + + """ return self.sink.save_to_file(*args, **kwargs) def load_from_file(self, *args, **kwargs): - """See MemorySource.load_from_file() for documentation""" + """Load STIX data from JSON file. + + File format is expected to be a single JSON + STIX object or JSON STIX bundle. + + Args: + file_path (str): file path to load STIX data from + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ return self.source.load_from_file(*args, **kwargs) @@ -135,13 +151,6 @@ class MemorySink(DataSink): add.__doc__ = _add.__doc__ def save_to_file(self, file_path): - """Write SITX objects from in-memory dictionary to JSON file, as a STIX - Bundle. - - Args: - file_path (str): file path to write STIX data to - - """ file_path = os.path.abspath(file_path) if not os.path.exists(os.path.dirname(file_path)): @@ -280,17 +289,6 @@ class MemorySource(DataSource): return all_data def load_from_file(self, file_path, version=None): - """Load STIX data from JSON file. - - File format is expected to be a single JSON - STIX object or JSON STIX bundle. - - Args: - file_path (str): file path to load STIX data from - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ stix_data = json.load(open(os.path.abspath(file_path), "r")) if stix_data["type"] == "bundle": From 5f703509be280c896d42a68abd1d4996c9a65aa3 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 1 Mar 2018 09:04:42 -0500 Subject: [PATCH 017/150] Rename "sources" subpackage to "datastore" Fix #114. --- .../datastore/stix2.datastore.filesystem.rst | 5 ++++ .../api/datastore/stix2.datastore.filters.rst | 5 ++++ docs/api/datastore/stix2.datastore.memory.rst | 5 ++++ docs/api/datastore/stix2.datastore.taxii.rst | 5 ++++ docs/api/sources/stix2.sources.filesystem.rst | 5 ---- docs/api/sources/stix2.sources.filters.rst | 5 ---- docs/api/sources/stix2.sources.memory.rst | 5 ---- docs/api/sources/stix2.sources.taxii.rst | 5 ---- docs/api/stix2.datastore.rst | 5 ++++ docs/api/stix2.sources.rst | 5 ---- docs/guide/datastore.ipynb | 24 +++++++++---------- docs/guide/environment.ipynb | 10 ++++---- docs/guide/filesystem.ipynb | 10 ++++---- docs/guide/memory.ipynb | 8 +++---- docs/guide/taxii.ipynb | 4 ++-- examples/taxii_example.py | 2 +- stix2/__init__.py | 16 ++++++------- stix2/{sources => datastore}/__init__.py | 6 ++--- stix2/{sources => datastore}/filesystem.py | 4 ++-- stix2/{sources => datastore}/filters.py | 0 stix2/{sources => datastore}/memory.py | 4 ++-- stix2/{sources => datastore}/taxii.py | 4 ++-- stix2/environment.py | 2 +- ...test_data_sources.py => test_datastore.py} | 8 +++---- stix2/test/test_memory.py | 2 +- 25 files changed, 77 insertions(+), 77 deletions(-) create mode 100644 docs/api/datastore/stix2.datastore.filesystem.rst create mode 100644 docs/api/datastore/stix2.datastore.filters.rst create mode 100644 docs/api/datastore/stix2.datastore.memory.rst create mode 100644 docs/api/datastore/stix2.datastore.taxii.rst delete mode 100644 docs/api/sources/stix2.sources.filesystem.rst delete mode 100644 docs/api/sources/stix2.sources.filters.rst delete mode 100644 docs/api/sources/stix2.sources.memory.rst delete mode 100644 docs/api/sources/stix2.sources.taxii.rst create mode 100644 docs/api/stix2.datastore.rst delete mode 100644 docs/api/stix2.sources.rst rename stix2/{sources => datastore}/__init__.py (99%) rename stix2/{sources => datastore}/filesystem.py (99%) rename stix2/{sources => datastore}/filters.py (100%) rename stix2/{sources => datastore}/memory.py (98%) rename stix2/{sources => datastore}/taxii.py (98%) rename stix2/test/{test_data_sources.py => test_datastore.py} (98%) diff --git a/docs/api/datastore/stix2.datastore.filesystem.rst b/docs/api/datastore/stix2.datastore.filesystem.rst new file mode 100644 index 0000000..665df66 --- /dev/null +++ b/docs/api/datastore/stix2.datastore.filesystem.rst @@ -0,0 +1,5 @@ +filesystem +========================== + +.. automodule:: stix2.datastore.filesystem + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.filters.rst b/docs/api/datastore/stix2.datastore.filters.rst new file mode 100644 index 0000000..b556754 --- /dev/null +++ b/docs/api/datastore/stix2.datastore.filters.rst @@ -0,0 +1,5 @@ +filters +======================= + +.. automodule:: stix2.datastore.filters + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.memory.rst b/docs/api/datastore/stix2.datastore.memory.rst new file mode 100644 index 0000000..b0521c7 --- /dev/null +++ b/docs/api/datastore/stix2.datastore.memory.rst @@ -0,0 +1,5 @@ +memory +====================== + +.. automodule:: stix2.datastore.memory + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.taxii.rst b/docs/api/datastore/stix2.datastore.taxii.rst new file mode 100644 index 0000000..68389a0 --- /dev/null +++ b/docs/api/datastore/stix2.datastore.taxii.rst @@ -0,0 +1,5 @@ +taxii +===================== + +.. automodule:: stix2.datastore.taxii + :members: \ No newline at end of file diff --git a/docs/api/sources/stix2.sources.filesystem.rst b/docs/api/sources/stix2.sources.filesystem.rst deleted file mode 100644 index ca0cfce..0000000 --- a/docs/api/sources/stix2.sources.filesystem.rst +++ /dev/null @@ -1,5 +0,0 @@ -filesystem -======================== - -.. automodule:: stix2.sources.filesystem - :members: \ No newline at end of file diff --git a/docs/api/sources/stix2.sources.filters.rst b/docs/api/sources/stix2.sources.filters.rst deleted file mode 100644 index 43dea51..0000000 --- a/docs/api/sources/stix2.sources.filters.rst +++ /dev/null @@ -1,5 +0,0 @@ -filters -===================== - -.. automodule:: stix2.sources.filters - :members: \ No newline at end of file diff --git a/docs/api/sources/stix2.sources.memory.rst b/docs/api/sources/stix2.sources.memory.rst deleted file mode 100644 index 1e70f2e..0000000 --- a/docs/api/sources/stix2.sources.memory.rst +++ /dev/null @@ -1,5 +0,0 @@ -memory -==================== - -.. automodule:: stix2.sources.memory - :members: \ No newline at end of file diff --git a/docs/api/sources/stix2.sources.taxii.rst b/docs/api/sources/stix2.sources.taxii.rst deleted file mode 100644 index 42303fe..0000000 --- a/docs/api/sources/stix2.sources.taxii.rst +++ /dev/null @@ -1,5 +0,0 @@ -taxii -=================== - -.. automodule:: stix2.sources.taxii - :members: \ No newline at end of file diff --git a/docs/api/stix2.datastore.rst b/docs/api/stix2.datastore.rst new file mode 100644 index 0000000..4af05a9 --- /dev/null +++ b/docs/api/stix2.datastore.rst @@ -0,0 +1,5 @@ +datastore +=============== + +.. automodule:: stix2.datastore + :members: \ No newline at end of file diff --git a/docs/api/stix2.sources.rst b/docs/api/stix2.sources.rst deleted file mode 100644 index aaf9f89..0000000 --- a/docs/api/stix2.sources.rst +++ /dev/null @@ -1,5 +0,0 @@ -sources -============= - -.. automodule:: stix2.sources - :members: \ No newline at end of file diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 31635ba..de6c810 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -71,9 +71,9 @@ "source": [ "# DataStore API\n", "\n", - "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) constructs: a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) for both pulling and pushing.\n", + "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore), [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) constructs: a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) for both pulling and pushing.\n", "\n", - "The DataStore, [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource), [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites." + "The DataStore, [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource), [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites." ] }, { @@ -82,13 +82,13 @@ "source": [ "## CompositeDataSource\n", "\n", - "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is an available controller that can be used as a single interface to a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). The purpose of this controller is allow for the grouping of [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) and making `get()`/`query()` calls to a set of DataSources in one API call. [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can be used to organize/group [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource), federate `get()`/`all_versions()`/`query()` calls, and reduce user code.\n", + "[CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) is an available controller that can be used as a single interface to a set of defined [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). The purpose of this controller is allow for the grouping of [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) and making `get()`/`query()` calls to a set of DataSources in one API call. [CompositeDataSources](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) can be used to organize/group [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource), federate `get()`/`all_versions()`/`query()` calls, and reduce user code.\n", "\n", - "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is just a wrapper around a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) (e.g. [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource)) that federates `get()`/`all_versions()`/`query()` calls individually to each of the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) , collects the results from each [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and returns them.\n", + "[CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) is just a wrapper around a set of defined [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) (e.g. [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource)) that federates `get()`/`all_versions()`/`query()` calls individually to each of the attached [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) , collects the results from each [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and returns them.\n", "\n", - "Filters can be attached to [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore) and [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). In addition, those [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource). \n", + "Filters can be attached to [CompositeDataSources](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStore) and [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). In addition, those [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource). \n", "\n", - "A [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can also be attached to a [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) for multiple layers of grouped [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource).\n", + "A [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) can also be attached to a [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) for multiple layers of grouped [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource).\n", "\n", "\n", "### CompositeDataSource API\n", @@ -353,7 +353,7 @@ "source": [ "## Filters\n", "\n", - "The CTI Python STIX2 DataStore suites - [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst) - all use the [Filters](../api/sources/stix2.sources.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) so that every future query placed to that [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore).\n", + "The CTI Python STIX2 DataStore suites - [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst) - all use the [Filters](../api/datastore/stix2.datastore.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) so that every future query placed to that [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStore).\n", "\n", "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX2 objects can be filtered on. In addition, TAXII2 Filtering parameters for fields can also be used in filters.\n", "\n", @@ -418,7 +418,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For Filters to be applied to a query, they must be either supplied with the query call or attached a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), more specifically to a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) whether that [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) is a part of a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) or stands by itself. " + "For Filters to be applied to a query, they must be either supplied with the query call or attached a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore), more specifically to a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) whether that [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) is a part of a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) or stands by itself. " ] }, { @@ -461,7 +461,7 @@ "source": [ "## De-Referencing Relationships\n", "\n", - "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) and add some objects and relationships." + "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) and add some objects and relationships." ] }, { @@ -487,7 +487,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of) method to retrieve the [Identity](../api/stix2.v20.sdo.rst#stix2.v20.sdo.Identity) object that created it." + "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.datastore.rst#stix2.datastore.DataSource.creator_of) method to retrieve the [Identity](../api/stix2.v20.sdo.rst#stix2.v20.sdo.Identity) object that created it." ] }, { @@ -593,7 +593,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Use the [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) method to retrieve all the relationship objects that reference a STIX object." + "Use the [relationships()](../api/stix2.datastore.rst#stix2.datastore.DataSource.relationships) method to retrieve all the relationship objects that reference a STIX object." ] }, { @@ -703,7 +703,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Finally, you can retrieve all STIX objects related to a given STIX object using [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to). This calls [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) but then performs the extra step of getting the objects that these Relationships point to. [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to) takes all the same arguments that [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) does." + "Finally, you can retrieve all STIX objects related to a given STIX object using [related_to()](../api/stix2.datastore.rst#stix2.datastore.DataSource.related_to). This calls [relationships()](../api/stix2.datastore.rst#stix2.datastore.DataSource.relationships) but then performs the extra step of getting the objects that these Relationships point to. [related_to()](../api/stix2.datastore.rst#stix2.datastore.DataSource.related_to) takes all the same arguments that [relationships()](../api/stix2.datastore.rst#stix2.datastore.DataSource.relationships) does." ] }, { diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb index 0cb5796..e3be92f 100644 --- a/docs/guide/environment.ipynb +++ b/docs/guide/environment.ipynb @@ -62,7 +62,7 @@ "\n", "### Storing and Retrieving STIX Content\n", "\n", - "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) if you want to store and retrieve STIX content from the same place. " + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) if you want to store and retrieve STIX content from the same place. " ] }, { @@ -82,7 +82,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If desired, you can instead set up an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with different data sources and sinks. In the following example we set up an environment that retrieves objects from [memory](../api/sources/stix2.sources.memory.rst) and a directory on the [filesystem](../api/sources/stix2.sources.filesystem.rst), and stores objects in a different directory on the filesystem." + "If desired, you can instead set up an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with different data sources and sinks. In the following example we set up an environment that retrieves objects from [memory](../api/datastore/stix2.datastore.memory.rst) and a directory on the [filesystem](../api/datastore/stix2.datastore.filesystem.rst), and stores objects in a different directory on the filesystem." ] }, { @@ -105,7 +105,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) you can store some STIX content in its [DataSinks](../api/stix2.sources.rst#stix2.sources.DataSink) with [add()](../api/stix2.environment.rst#stix2.environment.Environment.add):" + "Once you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) you can store some STIX content in its [DataSinks](../api/stix2.datastore.rst#stix2.datastore.DataSink) with [add()](../api/stix2.environment.rst#stix2.environment.Environment.add):" ] }, { @@ -128,7 +128,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can retrieve STIX objects from the [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of), [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to), and [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) just as you would for a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource)." + "You can retrieve STIX objects from the [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), [creator_of()](../api/stix2.datastore.rst#stix2.datastore.DataSource.creator_of), [related_to()](../api/stix2.datastore.rst#stix2.datastore.DataSource.related_to), and [relationships()](../api/stix2.datastore.rst#stix2.datastore.DataSource.relationships) just as you would for a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource)." ] }, { @@ -609,7 +609,7 @@ "collapsed": true }, "source": [ - "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore)/[Source](../api/stix2.sources.rst#stix2.sources.DataSource)/[Sink](../api/stix2.sources.rst#stix2.sources.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" + "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore)/[Source](../api/stix2.datastore.rst#stix2.datastore.DataSource)/[Sink](../api/stix2.datastore.rst#stix2.datastore.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" ] }, { diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index f494e6e..cff73d6 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -58,7 +58,7 @@ "source": [ "## FileSystem \n", "\n", - "The FileSystem suite contains [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore), [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", + "The FileSystem suite contains [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore), [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", "\n", "The directory and file structure of the intended STIX2 content should be:\n", "\n", @@ -107,15 +107,15 @@ " /vulnerability\n", "```\n", "\n", - "[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n", + "[FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) that point the same file directory.\n", "\n", - "For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", + "For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", "\n", "### FileSystem API\n", "\n", - "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n", + "A note on [get()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.get), [all_versions()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.all_versions), and [query()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n", "\n", - "A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", + "A note on [add()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", "\n", "### FileSystem Examples\n", "\n", diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index d651525..79a8f33 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -58,15 +58,15 @@ "source": [ "## Memory\n", "\n", - "The Memory suite consists of [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore), [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource), and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink). Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is a just a wrapper around a paired [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink); as there is quite limited uses for just a [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) or a [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink), it is recommended to always use [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). The [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is encompassed within the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore). However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. [MemoryStore.save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file) allows for saving all the STIX content that is in memory to a json file. [MemoryStore.load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) allows for loading STIX content from a JSON-formatted file. \n", + "The Memory suite consists of [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore), [MemorySource](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySource), and [MemorySink](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySink). Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) is a just a wrapper around a paired [MemorySource](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySource) and [MemorySink](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySink); as there is quite limited uses for just a [MemorySource](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySource) or a [MemorySink](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemorySink), it is recommended to always use [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore). The [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is encompassed within the [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore). However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. [MemoryStore.save_to_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.save_to_file) allows for saving all the STIX content that is in memory to a json file. [MemoryStore.load_from_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.load_from_file) allows for loading STIX content from a JSON-formatted file. \n", "\n", "\n", "### Memory API\n", - "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", + "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", "\n", - "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file): For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", + "A note on [load_from_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.load_from_file): For [load_from_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", "\n", - "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", + "A note on [save_to_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore). \n", "\n", "### Memory Examples\n", "\n", diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index b0f0cea..ad06093 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -58,9 +58,9 @@ "source": [ "## TAXIICollection\n", "\n", - "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) pushes and retrieves STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) retrieves STIX content from local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) pushes STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", + "The TAXIICollection suite contains [TAXIICollectionStore](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionStore) pushes and retrieves STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionSource) retrieves STIX content from local/remote TAXII Collection(s). [TAXIICollectionSink](../api/datastore/stix2.datastore.taxii.rst#stix2.datastore.taxii.TAXIICollectionSink) pushes STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/datastore/stix2.datastore.taxii.rst) API calls will be executed through that Collection instance.\n", "\n", - "A note on TAXII2 searching/filtering of STIX content: TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", + "A note on TAXII2 searching/filtering of STIX content: TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/datastore/stix2.datastore.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/datastore/stix2.datastore.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/datastore/stix2.datastore.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/datastore/stix2.datastore.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/datastore/stix2.datastore.taxii.rst) API call. \n", "\n", "### TAXIICollection API\n", "\n", diff --git a/examples/taxii_example.py b/examples/taxii_example.py index 166d028..e102b17 100644 --- a/examples/taxii_example.py +++ b/examples/taxii_example.py @@ -1,6 +1,6 @@ import json -from stix2.sources.taxii import TAXIIDataSource +from stix2.datastore.taxii import TAXIIDataSource # Flask TAXII server - developmental ROOT = 'http://localhost:5000' diff --git a/stix2/__init__.py b/stix2/__init__.py index 9ce9378..884e861 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -6,13 +6,13 @@ v20.common core environment + datastore exceptions markings v20.observables patterns properties v20.sdo - sources v20.sro utils """ @@ -20,6 +20,13 @@ # flake8: noqa from .core import Bundle, _collect_stix2_obj_maps, _register_type, parse +from .datastore import CompositeDataSource +from .datastore.filesystem import (FileSystemSink, FileSystemSource, + FileSystemStore) +from .datastore.filters import Filter +from .datastore.memory import MemorySink, MemorySource, MemoryStore +from .datastore.taxii import (TAXIICollectionSink, TAXIICollectionSource, + TAXIICollectionStore) from .environment import Environment, ObjectFactory from .markings import (add_markings, clear_markings, get_markings, is_marked, remove_markings, set_markings) @@ -41,13 +48,6 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) -from .sources import CompositeDataSource -from .sources.filesystem import (FileSystemSink, FileSystemSource, - FileSystemStore) -from .sources.filters import Filter -from .sources.memory import MemorySink, MemorySource, MemoryStore -from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource, - TAXIICollectionStore) from .utils import get_dict, new_version, revoke from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ diff --git a/stix2/sources/__init__.py b/stix2/datastore/__init__.py similarity index 99% rename from stix2/sources/__init__.py rename to stix2/datastore/__init__.py index adc6def..0386bf7 100644 --- a/stix2/sources/__init__.py +++ b/stix2/datastore/__init__.py @@ -1,7 +1,7 @@ -"""Python STIX 2.0 Sources +"""Python STIX 2.0 DataStore API .. autosummary:: - :toctree: sources + :toctree: datastore filesystem filters @@ -16,7 +16,7 @@ import uuid from six import with_metaclass -from stix2.sources.filters import Filter +from stix2.datastore.filters import Filter from stix2.utils import deduplicate diff --git a/stix2/sources/filesystem.py b/stix2/datastore/filesystem.py similarity index 99% rename from stix2/sources/filesystem.py rename to stix2/datastore/filesystem.py index f4311be..c0401d7 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -7,8 +7,8 @@ import json import os from stix2.core import Bundle, parse -from stix2.sources import DataSink, DataSource, DataStore -from stix2.sources.filters import Filter, apply_common_filters +from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore.filters import Filter, apply_common_filters from stix2.utils import deduplicate, get_class_hierarchy_names diff --git a/stix2/sources/filters.py b/stix2/datastore/filters.py similarity index 100% rename from stix2/sources/filters.py rename to stix2/datastore/filters.py diff --git a/stix2/sources/memory.py b/stix2/datastore/memory.py similarity index 98% rename from stix2/sources/memory.py rename to stix2/datastore/memory.py index 5d08d7c..1e18721 100644 --- a/stix2/sources/memory.py +++ b/stix2/datastore/memory.py @@ -17,8 +17,8 @@ import os from stix2.base import _STIXBase from stix2.core import Bundle, parse -from stix2.sources import DataSink, DataSource, DataStore -from stix2.sources.filters import Filter, apply_common_filters +from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore.filters import Filter, apply_common_filters def _add(store, stix_data=None, version=None): diff --git a/stix2/sources/taxii.py b/stix2/datastore/taxii.py similarity index 98% rename from stix2/sources/taxii.py rename to stix2/datastore/taxii.py index 2d54725..41dfd0c 100644 --- a/stix2/sources/taxii.py +++ b/stix2/datastore/taxii.py @@ -5,8 +5,8 @@ from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse -from stix2.sources import DataSink, DataSource, DataStore -from stix2.sources.filters import Filter, apply_common_filters +from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore.filters import Filter, apply_common_filters from stix2.utils import deduplicate TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] diff --git a/stix2/environment.py b/stix2/environment.py index ab16b09..c278bfb 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -1,7 +1,7 @@ import copy from .core import parse as _parse -from .sources import CompositeDataSource, DataStore +from .datastore import CompositeDataSource, DataStore class ObjectFactory(object): diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_datastore.py similarity index 98% rename from stix2/test/test_data_sources.py rename to stix2/test/test_datastore.py index d7f238a..3490d6a 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_datastore.py @@ -2,9 +2,9 @@ import pytest from taxii2client import Collection from stix2 import Filter, MemorySink, MemorySource -from stix2.sources import (CompositeDataSource, DataSink, DataSource, make_id, - taxii) -from stix2.sources.filters import apply_common_filters +from stix2.datastore import (CompositeDataSource, DataSink, DataSource, + make_id, taxii) +from stix2.datastore.filters import apply_common_filters from stix2.utils import deduplicate COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -476,7 +476,7 @@ def test_add_remove_composite_datasource(): with pytest.raises(TypeError) as excinfo: cds.add_data_sources([ds1, ds2, ds1, ds3]) assert str(excinfo.value) == ("DataSource (to be added) is not of type " - "stix2.DataSource. DataSource type is ''") + "stix2.DataSource. DataSource type is ''") cds.add_data_sources([ds1, ds2, ds1]) diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index ad78611..2384848 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -6,7 +6,7 @@ import pytest from stix2 import (Bundle, Campaign, CustomObject, Filter, Identity, Indicator, Malware, MemorySource, MemoryStore, Relationship, properties) -from stix2.sources import make_id +from stix2.datastore import make_id from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, From eb2d87ce7193b93844d20f40ae60f531b969af40 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 1 Mar 2018 11:27:37 -0500 Subject: [PATCH 018/150] Rename "DataStore" class to "DataStoreMixin" Fix #116. --- docs/guide/datastore.ipynb | 10 +++++----- docs/guide/environment.ipynb | 4 ++-- stix2/datastore/__init__.py | 8 ++++---- stix2/datastore/filesystem.py | 4 ++-- stix2/datastore/memory.py | 8 ++++---- stix2/datastore/taxii.py | 4 ++-- stix2/environment.py | 20 ++++++++++---------- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index de6c810..ac27a82 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -71,7 +71,7 @@ "source": [ "# DataStore API\n", "\n", - "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore), [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) constructs: a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) for both pulling and pushing.\n", + "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin), [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) constructs: a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) for both pulling and pushing.\n", "\n", "The DataStore, [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource), [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites." ] @@ -86,7 +86,7 @@ "\n", "[CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) is just a wrapper around a set of defined [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) (e.g. [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource)) that federates `get()`/`all_versions()`/`query()` calls individually to each of the attached [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) , collects the results from each [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and returns them.\n", "\n", - "Filters can be attached to [CompositeDataSources](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStore) and [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). In addition, those [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource). \n", + "Filters can be attached to [CompositeDataSources](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) and [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource). In addition, those [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource). \n", "\n", "A [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) can also be attached to a [CompositeDataSource](../api/stix2.datastore.rst#stix2.datastore.CompositeDataSource) for multiple layers of grouped [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource).\n", "\n", @@ -353,7 +353,7 @@ "source": [ "## Filters\n", "\n", - "The CTI Python STIX2 DataStore suites - [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst) - all use the [Filters](../api/datastore/stix2.datastore.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) so that every future query placed to that [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStore).\n", + "The CTI Python STIX2 DataStore suites - [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst) - all use the [Filters](../api/datastore/stix2.datastore.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) so that every future query placed to that [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin).\n", "\n", "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX2 objects can be filtered on. In addition, TAXII2 Filtering parameters for fields can also be used in filters.\n", "\n", @@ -418,7 +418,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For Filters to be applied to a query, they must be either supplied with the query call or attached a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore), more specifically to a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) whether that [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) is a part of a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) or stands by itself. " + "For Filters to be applied to a query, they must be either supplied with the query call or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin), more specifically to a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) whether that [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) is a part of a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) or stands by itself. " ] }, { @@ -461,7 +461,7 @@ "source": [ "## De-Referencing Relationships\n", "\n", - "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) and add some objects and relationships." + "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) and add some objects and relationships." ] }, { diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb index e3be92f..3ece7c4 100644 --- a/docs/guide/environment.ipynb +++ b/docs/guide/environment.ipynb @@ -62,7 +62,7 @@ "\n", "### Storing and Retrieving STIX Content\n", "\n", - "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore) if you want to store and retrieve STIX content from the same place. " + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) if you want to store and retrieve STIX content from the same place. " ] }, { @@ -609,7 +609,7 @@ "collapsed": true }, "source": [ - "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStore)/[Source](../api/stix2.datastore.rst#stix2.datastore.DataSource)/[Sink](../api/stix2.datastore.rst#stix2.datastore.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" + "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin)/[Source](../api/stix2.datastore.rst#stix2.datastore.DataSource)/[Sink](../api/stix2.datastore.rst#stix2.datastore.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" ] }, { diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py index 0386bf7..78f7555 100644 --- a/stix2/datastore/__init__.py +++ b/stix2/datastore/__init__.py @@ -24,9 +24,9 @@ def make_id(): return str(uuid.uuid4()) -class DataStore(object): - """An implementer can subclass to create custom behavior from - this class for the specific DataStores. +class DataStoreMixin(object): + """Provides mechanisms for storing and retrieving STIX data. The specific + behavior can be customized by subclasses. Args: source (DataSource): An existing DataSource to use @@ -41,7 +41,7 @@ class DataStore(object): """ def __init__(self, source=None, sink=None): - super(DataStore, self).__init__() + super(DataStoreMixin, self).__init__() self.id = make_id() self.source = source self.sink = sink diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index c0401d7..26d0c58 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -7,12 +7,12 @@ import json import os from stix2.core import Bundle, parse -from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, apply_common_filters from stix2.utils import deduplicate, get_class_hierarchy_names -class FileSystemStore(DataStore): +class FileSystemStore(DataStoreMixin): """Interface to a file directory of STIX objects. FileSystemStore is a wrapper around a paired FileSystemSink diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index 1e18721..e057271 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -17,7 +17,7 @@ import os from stix2.base import _STIXBase from stix2.core import Bundle, parse -from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, apply_common_filters @@ -56,7 +56,7 @@ def _add(store, stix_data=None, version=None): " or a JSON formatted STIX bundle. stix_data was of type: " + str(type(stix_data))) -class MemoryStore(DataStore): +class MemoryStore(DataStoreMixin): """Interface to an in-memory dictionary of STIX objects. MemoryStore is a wrapper around a paired MemorySink and MemorySource. @@ -124,7 +124,7 @@ class MemorySink(DataSink): Args: stix_data (dict OR list): valid STIX 2.0 content in bundle or a list. - _store (bool): whether the MemorySink is a part of a DataStore, + _store (bool): whether the MemorySink is a part of a MemoryStore, in which case "stix_data" is a direct reference to shared memory with DataSource. Not user supplied allow_custom (bool): whether to allow custom objects/properties @@ -170,7 +170,7 @@ class MemorySource(DataSource): Args: stix_data (dict OR list OR STIX object): valid STIX 2.0 content in bundle or list. - _store (bool): if the MemorySource is a part of a DataStore, + _store (bool): if the MemorySource is a part of a MemoryStore, in which case "stix_data" is a direct reference to shared memory with DataSink. Not user supplied allow_custom (bool): whether to allow custom objects/properties diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 41dfd0c..5af4354 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -5,14 +5,14 @@ from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse -from stix2.datastore import DataSink, DataSource, DataStore +from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, apply_common_filters from stix2.utils import deduplicate TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] -class TAXIICollectionStore(DataStore): +class TAXIICollectionStore(DataStoreMixin): """Provides an interface to a local/remote TAXII Collection of STIX data. TAXIICollectionStore is a wrapper around a paired TAXIICollectionSink and TAXIICollectionSource. diff --git a/stix2/environment.py b/stix2/environment.py index c278bfb..eb5583e 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -1,7 +1,7 @@ import copy from .core import parse as _parse -from .datastore import CompositeDataSource, DataStore +from .datastore import CompositeDataSource, DataStoreMixin class ObjectFactory(object): @@ -75,7 +75,7 @@ class ObjectFactory(object): return cls(**properties) -class Environment(object): +class Environment(DataStoreMixin): """Abstract away some of the nasty details of working with STIX content. Args: @@ -86,6 +86,14 @@ class Environment(object): source (DataSource, optional): Source for retrieving STIX objects. sink (DataSink, optional): Destination for saving STIX objects. Invalid if `store` is also provided. + + .. automethod:: get + .. automethod:: all_versions + .. automethod:: query + .. automethod:: creator_of + .. automethod:: relationships + .. automethod:: related_to + .. automethod:: add """ def __init__(self, factory=ObjectFactory(), store=None, source=None, sink=None): @@ -105,14 +113,6 @@ class Environment(object): return self.factory.create(*args, **kwargs) create.__doc__ = ObjectFactory.create.__doc__ - get = DataStore.__dict__['get'] - all_versions = DataStore.__dict__['all_versions'] - query = DataStore.__dict__['query'] - creator_of = DataStore.__dict__['creator_of'] - relationships = DataStore.__dict__['relationships'] - related_to = DataStore.__dict__['related_to'] - add = DataStore.__dict__['add'] - def add_filters(self, *args, **kwargs): try: return self.source.filters.update(*args, **kwargs) From 307a85c8b169d58a9d31b20e3f6dff0fe32680a9 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 1 Mar 2018 13:15:02 -0600 Subject: [PATCH 019/150] Reorder modules for API documentation --- stix2/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index 884e861..401d44b 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,18 +3,18 @@ .. autosummary:: :toctree: api - v20.common core - environment datastore + environment exceptions markings - v20.observables patterns properties + utils + v20.common + v20.observables v20.sdo v20.sro - utils """ # flake8: noqa From 1eab9b2832487447aee270d27d07d002954ccb3f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 2 Mar 2018 10:21:51 -0500 Subject: [PATCH 020/150] Assume custom properties allowable in add_markings --- stix2/markings/object_markings.py | 2 +- stix2/test/test_custom.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index c0375c3..563d92b 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -37,7 +37,7 @@ def add_markings(obj, marking): object_markings = set(obj.get("object_marking_refs", []) + marking) - return new_version(obj, object_marking_refs=list(object_markings)) + return new_version(obj, object_marking_refs=list(object_markings), allow_custom=True) def remove_markings(obj, marking): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 7c1832b..df0db79 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -2,7 +2,7 @@ import pytest import stix2 -from .constants import FAKE_TIME +from .constants import FAKE_TIME, MARKING_DEFINITION_ID def test_identity_custom_property(): @@ -94,6 +94,25 @@ def test_custom_property_in_bundled_object(): assert '"x_foo": "bar"' in str(bundle) +def test_identity_custom_property_add_marking(): + identity = stix2.Identity( + id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", + created="2015-12-21T19:59:11Z", + modified="2015-12-21T19:59:11Z", + name="John Smith", + identity_class="individual", + x_foo="bar", + allow_custom=True, + ) + marking_definition = stix2.MarkingDefinition( + id=MARKING_DEFINITION_ID, + definition_type="statement", + definition=stix2.StatementMarking(statement="Copyright 2016, Example Corp") + ) + identity2 = identity.add_markings(marking_definition) + assert identity2.x_foo == "bar" + + def test_custom_marking_no_init_1(): @stix2.CustomMarking('x-new-obj', [ ('property1', stix2.properties.StringProperty(required=True)), From 5a71ef2e64f613418a9d03b5560a0b630a3a6174 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 2 Mar 2018 11:32:07 -0500 Subject: [PATCH 021/150] Fix allow_custom in functions calling new_version --- stix2/markings/granular_markings.py | 10 +++--- stix2/markings/object_markings.py | 6 ++-- stix2/test/test_custom.py | 48 +++++++++++++++++------------ stix2/utils.py | 2 +- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index be5d258..7c227d9 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -116,9 +116,9 @@ def remove_markings(obj, marking, selectors): granular_markings = utils.compress_markings(granular_markings) if granular_markings: - return new_version(obj, granular_markings=granular_markings) + return new_version(obj, granular_markings=granular_markings, allow_custom=True) else: - return new_version(obj, granular_markings=None) + return new_version(obj, granular_markings=None, allow_custom=True) def add_markings(obj, marking, selectors): @@ -152,7 +152,7 @@ def add_markings(obj, marking, selectors): granular_marking = utils.expand_markings(granular_marking) granular_marking = utils.compress_markings(granular_marking) - return new_version(obj, granular_markings=granular_marking) + return new_version(obj, granular_markings=granular_marking, allow_custom=True) def clear_markings(obj, selectors): @@ -207,9 +207,9 @@ def clear_markings(obj, selectors): granular_markings = utils.compress_markings(granular_markings) if granular_markings: - return new_version(obj, granular_markings=granular_markings) + return new_version(obj, granular_markings=granular_markings, allow_custom=True) else: - return new_version(obj, granular_markings=None) + return new_version(obj, granular_markings=None, allow_custom=True) def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index 563d92b..a169fe3 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -69,9 +69,9 @@ def remove_markings(obj, marking): new_markings = [x for x in object_markings if x not in marking] if new_markings: - return new_version(obj, object_marking_refs=new_markings) + return new_version(obj, object_marking_refs=new_markings, allow_custom=True) else: - return new_version(obj, object_marking_refs=None) + return new_version(obj, object_marking_refs=None, allow_custom=True) def set_markings(obj, marking): @@ -103,7 +103,7 @@ def clear_markings(obj): A new version of the given SDO or SRO with object_marking_refs cleared. """ - return new_version(obj, object_marking_refs=None) + return new_version(obj, object_marking_refs=None, allow_custom=True) def is_marked(obj, marking=None): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index df0db79..76ad61b 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -4,6 +4,13 @@ import stix2 from .constants import FAKE_TIME, MARKING_DEFINITION_ID +IDENTITY_CUSTOM_PROP = stix2.Identity( + name="John Smith", + identity_class="individual", + x_foo="bar", + allow_custom=True, +) + def test_identity_custom_property(): with pytest.raises(ValueError) as excinfo: @@ -82,35 +89,36 @@ def test_parse_identity_custom_property(data): def test_custom_property_in_bundled_object(): - identity = stix2.Identity( - name="John Smith", - identity_class="individual", - x_foo="bar", - allow_custom=True, - ) - bundle = stix2.Bundle(identity, allow_custom=True) + bundle = stix2.Bundle(IDENTITY_CUSTOM_PROP, allow_custom=True) assert bundle.objects[0].x_foo == "bar" assert '"x_foo": "bar"' in str(bundle) -def test_identity_custom_property_add_marking(): - identity = stix2.Identity( - id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", - created="2015-12-21T19:59:11Z", - modified="2015-12-21T19:59:11Z", - name="John Smith", - identity_class="individual", - x_foo="bar", - allow_custom=True, - ) - marking_definition = stix2.MarkingDefinition( +def test_identity_custom_property_revoke(): + identity = IDENTITY_CUSTOM_PROP.revoke() + assert identity.x_foo == "bar" + + +def test_identity_custom_property_edit_markings(): + marking_obj = stix2.MarkingDefinition( id=MARKING_DEFINITION_ID, definition_type="statement", definition=stix2.StatementMarking(statement="Copyright 2016, Example Corp") ) - identity2 = identity.add_markings(marking_definition) - assert identity2.x_foo == "bar" + marking_obj2 = stix2.MarkingDefinition( + id=MARKING_DEFINITION_ID, + definition_type="statement", + definition=stix2.StatementMarking(statement="Another one") + ) + + # None of the following should throw exceptions + identity = IDENTITY_CUSTOM_PROP.add_markings(marking_obj) + identity2 = identity.add_markings(marking_obj2, ['x_foo']) + identity2.remove_markings(marking_obj.id) + identity2.remove_markings(marking_obj2.id, ['x_foo']) + identity2.clear_markings() + identity2.clear_markings('x_foo') def test_custom_marking_no_init_1(): diff --git a/stix2/utils.py b/stix2/utils.py index 73337d0..37ff166 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -251,7 +251,7 @@ def revoke(data): if data.get("revoked"): raise RevokeError("revoke") - return new_version(data, revoked=True) + return new_version(data, revoked=True, allow_custom=True) def get_class_hierarchy_names(obj): From 0e70e61451657969117e45478ac1d6ec95b30bd0 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 6 Mar 2018 10:16:04 -0500 Subject: [PATCH 022/150] Update CHANGELOG for v0.5.0 --- CHANGELOG | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index c5ffabf..2862633 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,15 @@ CHANGELOG ========= +0.5.0 - 2018-03-06 + +* Adds functions to dereference relationships. +* Adds a function to get an object's type from its ID. +* Reorganizes DataStore api, renaming: stix2.sources to stix2.datastore. +* Fixes various bugs involving CustomMarking, creating new versions of objects + with custom properties, and a missing IntrusionSet property. +* Drops Python 3.3 support. + 0.4.0 - 2017-11-13 * Adds `creator_of` function to easily get the Identity that created an object, From 35679e6a6b3de84a7cf0cbbdbcb84b0c51aaa23e Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 6 Mar 2018 10:16:57 -0500 Subject: [PATCH 023/150] =?UTF-8?q?Bump=20version:=200.4.0=20=E2=86=92=200?= =?UTF-8?q?.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/conf.py | 4 ++-- setup.cfg | 2 +- stix2/version.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index dbed6a3..a3fb3f8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,8 +34,8 @@ project = 'stix2' copyright = '2017, OASIS Open' author = 'OASIS Open' -version = '0.4.0' -release = '0.4.0' +version = '0.5.0' +release = '0.5.0' language = None exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints'] diff --git a/setup.cfg b/setup.cfg index 76162b5..0cd287c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.0 +current_version = 0.5.0 commit = True tag = True diff --git a/stix2/version.py b/stix2/version.py index 6a9beea..3d18726 100644 --- a/stix2/version.py +++ b/stix2/version.py @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.5.0" From b613e2a66fb96cf52b0c0c60205df32efbf616a0 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 6 Mar 2018 12:05:25 -0500 Subject: [PATCH 024/150] Update CHANGELOG for v0.5.1 --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 2862633..7f77b25 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,10 @@ CHANGELOG ========= +0.5.1 - 2018-03-06 + +* Fixes issue with PyPI. + 0.5.0 - 2018-03-06 * Adds functions to dereference relationships. From f9523f24bf1243e7081f2e94d55e4a371189b593 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 6 Mar 2018 12:05:53 -0500 Subject: [PATCH 025/150] =?UTF-8?q?Bump=20version:=200.5.0=20=E2=86=92=200?= =?UTF-8?q?.5.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/conf.py | 4 ++-- setup.cfg | 2 +- stix2/version.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a3fb3f8..e53829b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,8 +34,8 @@ project = 'stix2' copyright = '2017, OASIS Open' author = 'OASIS Open' -version = '0.5.0' -release = '0.5.0' +version = '0.5.1' +release = '0.5.1' language = None exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints'] diff --git a/setup.cfg b/setup.cfg index 0cd287c..b252746 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.5.1 commit = True tag = True diff --git a/stix2/version.py b/stix2/version.py index 3d18726..dd9b22c 100644 --- a/stix2/version.py +++ b/stix2/version.py @@ -1 +1 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" From 955cad83858b41b158f94448f838ab6dcc1da5b5 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 13 Mar 2018 16:01:42 -0400 Subject: [PATCH 026/150] Clarify serialization in documentation --- docs/guide/serializing.ipynb | 151 ++++++++++++++++++++++++++++++----- stix2/base.py | 2 +- 2 files changed, 134 insertions(+), 19 deletions(-) diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb index 4499677..1046d9f 100644 --- a/docs/guide/serializing.ipynb +++ b/docs/guide/serializing.ipynb @@ -4,7 +4,6 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true, "nbsphinx": "hidden" }, "outputs": [], @@ -23,9 +22,8 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { - "collapsed": true, "nbsphinx": "hidden" }, "outputs": [], @@ -68,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -144,12 +142,12 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674",\n",
-       "    "created": "2017-11-09T19:21:06.285Z",\n",
-       "    "modified": "2017-11-09T19:21:06.285Z",\n",
+       "    "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48",\n",
+       "    "created": "2018-03-13T19:49:53.392Z",\n",
+       "    "modified": "2018-03-13T19:49:53.392Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-11-09T19:21:06.285451Z",\n",
+       "    "valid_from": "2018-03-13T19:49:53.392627Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ]\n",
@@ -160,7 +158,7 @@
        ""
       ]
      },
-     "execution_count": 2,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -179,12 +177,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "However, the string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's serialize() function:"
+    "However, the string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's `serialize()` function:"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -258,14 +256,14 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"valid_from": "2017-11-09T19:21:06.285451Z", "name": "File hash for malware variant", "created": "2017-11-09T19:21:06.285Z", "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "labels": ["malicious-activity"], "modified": "2017-11-09T19:21:06.285Z", "type": "indicator", "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674"}\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"name": "File hash for malware variant", "labels": ["malicious-activity"], "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "type": "indicator", "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48", "created": "2018-03-13T19:49:53.392Z", "modified": "2018-03-13T19:49:53.392Z", "valid_from": "2018-03-13T19:49:53.392627Z"}\n",
        "
\n"
       ],
       "text/plain": [
        ""
       ]
      },
-     "execution_count": 6,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -273,25 +271,142 @@
    "source": [
     "print(indicator.serialize())"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you need performance but also need human-readable output, you can pass the `indent` keyword argument to `serialize()`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "name": "File hash for malware variant",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ],\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48",\n",
+       "    "created": "2018-03-13T19:49:53.392Z",\n",
+       "    "modified": "2018-03-13T19:49:53.392Z",\n",
+       "    "valid_from": "2018-03-13T19:49:53.392627Z"\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(indicator.serialize(indent=4))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The only difference between this and the string representation from using `str()` is that this will not sort the keys. This works because the keyword arguments are passed to `json.dumps()` internally."
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/stix2/base.py b/stix2/base.py
index fc13094..6bc3dd4 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -187,7 +187,7 @@ class _STIXBase(collections.Mapping):
         Args:
             pretty (bool): If True, output properties following the STIX specs
                 formatting. This includes indentation. Refer to notes for more
-                details.
+                details. (Default: ``False``)
             **kwargs: The arguments for a json.dumps() call.
 
         Returns:

From 5820fa0845761851f4829ccba45b2afc31b51c57 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Wed, 14 Mar 2018 10:06:03 -0500
Subject: [PATCH 027/150] GH-138: ignore invalid JSON files.

---
 stix2/datastore/filesystem.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index 26d0c58..a00c6e1 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -303,7 +303,11 @@ class FileSystemSource(DataSource):
                 for file_ in files:
                     if not id_ or id_ == file_.split(".")[0]:
                         # have to load into memory regardless to evaluate other filters
-                        stix_obj = json.load(open(os.path.join(root, file_)))
+                        try:
+                            stix_obj = json.load(open(os.path.join(root, file_)))
+                        except UnicodeDecodeError:  # likely not a JSON file
+                            # TODO: log a warning somehow? (os.path.abspath(file_)))
+                            continue
                         if stix_obj.get('type', '') == 'bundle':
                             stix_obj = stix_obj['objects'][0]
                         # check against other filters, add if match

From af14cd4f881241dfd7373618c3352142faba3c02 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 14 Mar 2018 16:28:44 -0400
Subject: [PATCH 028/150] more type checking of filesystem json files; added
 corresponding tests

---
 stix2/datastore/filesystem.py | 20 +++++++++----
 stix2/test/test_filesystem.py | 56 ++++++++++++++++++++++++++++++++++-
 2 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index a00c6e1..438e706 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -301,15 +301,25 @@ class FileSystemSource(DataSource):
         for path in include_paths:
             for root, dirs, files in os.walk(path):
                 for file_ in files:
+                    if not file_.endswith(".json"):
+                        # skip non '.json' files as more likely to be random non-STIX files
+                        continue
+
                     if not id_ or id_ == file_.split(".")[0]:
                         # have to load into memory regardless to evaluate other filters
                         try:
                             stix_obj = json.load(open(os.path.join(root, file_)))
-                        except UnicodeDecodeError:  # likely not a JSON file
-                            # TODO: log a warning somehow? (os.path.abspath(file_)))
-                            continue
-                        if stix_obj.get('type', '') == 'bundle':
-                            stix_obj = stix_obj['objects'][0]
+
+                            if stix_obj["type"] == "bundle":
+                                stix_obj = stix_obj["objects"][0]
+
+                            stix_obj["type"]
+                            stix_obj["id"]
+
+                        except (UnicodeDecodeError, ValueError, KeyError) as e:  # likely not a JSON file
+                            print("filesytem TypeError raised")
+                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
+
                         # check against other filters, add if match
                         all_data.extend(apply_common_filters([stix_obj], query))
 
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 020fee5..64d0e31 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -1,4 +1,5 @@
 import os
+import json
 import shutil
 
 import pytest
@@ -44,6 +45,39 @@ def fs_sink():
     # remove campaign dir
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
+@pytest.fixture
+def bad_json_files():
+    # create erroneous JSON files for tests to make sure handled gracefully
+
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"), "w") as f:
+        f.write("Im not a JSON file")
+
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"), "w") as f:
+        f.write("Im not a JSON formatted file")
+
+    yield True # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"))
+    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"))
+
+@pytest.fixture
+def bad_stix_files():
+    # create erroneous STIX JSON files for tests to make sure handled correctly
+
+    # bad STIX object
+    stix_obj = {
+        "id": "indicator--test-bad-stix",
+        "spec_version": "2.0"
+        # no "type" field
+    }
+
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"), "w") as f:
+        f.write(json.dumps(stix_obj))
+
+    yield True # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"))
+
 
 @pytest.fixture(scope='module')
 def rel_fs_store():
@@ -76,6 +110,26 @@ def test_filesystem_sink_nonexistent_folder():
     assert "for STIX data does not exist" in str(excinfo)
 
 
+def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
+    # this tests the handling of two bad json files
+    #  - one file should just be skipped (silently) as its a ".txt" extension
+    #  - one file should be parsed and raise Exception bc its not JSON
+    try:
+        bad_json_indicator = fs_source.get("indicator--test-bad-json")
+    except TypeError as e:
+        assert "indicator--test-bad-json" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
+def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
+    # this tests handling of bad STIX json object
+    try:
+        bad_stix_indicator = fs_source.get("indicator--test-non-stix")
+    except TypeError as e:
+        assert "indicator--test-non-stix" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
 def test_filesytem_source_get_object(fs_source):
     # get object
     mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")
@@ -470,4 +524,4 @@ def test_related_to_by_target(rel_fs_store):
 
     assert len(resp) == 2
     assert any(x['id'] == CAMPAIGN_ID for x in resp)
-    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
\ No newline at end of file

From 6f762e7ea053ee6db90e91d1da19199307584155 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 14 Mar 2018 16:32:31 -0400
Subject: [PATCH 029/150] woops forgot file, recommit of: more type checking of
 filesystem json files; added corresponding tests

---
 stix2/datastore/filesystem.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index 438e706..35e3a54 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -313,10 +313,11 @@ class FileSystemSource(DataSource):
                             if stix_obj["type"] == "bundle":
                                 stix_obj = stix_obj["objects"][0]
 
+                            # naive STIX check
                             stix_obj["type"]
                             stix_obj["id"]
 
-                        except (UnicodeDecodeError, ValueError, KeyError) as e:  # likely not a JSON file
+                        except (ValueError, KeyError) as e:  # likely not a JSON file
                             print("filesytem TypeError raised")
                             raise TypeError("STIX JSON object at '{0}' could either not be parsed to JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
 

From f4558c09587734cce269b07bdb398842634407da Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 14 Mar 2018 19:09:25 -0400
Subject: [PATCH 030/150] pre-commit errors

---
 stix2/datastore/filesystem.py |  7 ++++---
 stix2/test/test_filesystem.py | 20 +++++++++++---------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index 35e3a54..b525932 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -313,13 +313,14 @@ class FileSystemSource(DataSource):
                             if stix_obj["type"] == "bundle":
                                 stix_obj = stix_obj["objects"][0]
 
-                            # naive STIX check
+                            # naive STIX type checking
                             stix_obj["type"]
                             stix_obj["id"]
 
-                        except (ValueError, KeyError) as e:  # likely not a JSON file
+                        except (ValueError, KeyError):  # likely not a JSON file
                             print("filesytem TypeError raised")
-                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
+                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to "
+                                            "JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
 
                         # check against other filters, add if match
                         all_data.extend(apply_common_filters([stix_obj], query))
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 64d0e31..4176322 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -1,5 +1,5 @@
-import os
 import json
+import os
 import shutil
 
 import pytest
@@ -45,21 +45,23 @@ def fs_sink():
     # remove campaign dir
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
+
 @pytest.fixture
 def bad_json_files():
     # create erroneous JSON files for tests to make sure handled gracefully
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"), "w") as f:
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"), "w+") as f:
         f.write("Im not a JSON file")
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"), "w") as f:
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"), "w+") as f:
         f.write("Im not a JSON formatted file")
 
-    yield True # dummy yield so can have teardown
+    yield True  # dummy yield so can have teardown
 
     os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"))
     os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"))
 
+
 @pytest.fixture
 def bad_stix_files():
     # create erroneous STIX JSON files for tests to make sure handled correctly
@@ -71,10 +73,10 @@ def bad_stix_files():
         # no "type" field
     }
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"), "w") as f:
+    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"), "w+") as f:
         f.write(json.dumps(stix_obj))
 
-    yield True # dummy yield so can have teardown
+    yield True  # dummy yield so can have teardown
 
     os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"))
 
@@ -115,7 +117,7 @@ def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
     #  - one file should just be skipped (silently) as its a ".txt" extension
     #  - one file should be parsed and raise Exception bc its not JSON
     try:
-        bad_json_indicator = fs_source.get("indicator--test-bad-json")
+        fs_source.get("indicator--test-bad-json")
     except TypeError as e:
         assert "indicator--test-bad-json" in str(e)
         assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
@@ -124,7 +126,7 @@ def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
 def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
     # this tests handling of bad STIX json object
     try:
-        bad_stix_indicator = fs_source.get("indicator--test-non-stix")
+        fs_source.get("indicator--test-non-stix")
     except TypeError as e:
         assert "indicator--test-non-stix" in str(e)
         assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
@@ -524,4 +526,4 @@ def test_related_to_by_target(rel_fs_store):
 
     assert len(resp) == 2
     assert any(x['id'] == CAMPAIGN_ID for x in resp)
-    assert any(x['id'] == INDICATOR_ID for x in resp)
\ No newline at end of file
+    assert any(x['id'] == INDICATOR_ID for x in resp)

From 2fbde05e6c20effa0272b9017f38a2341189627b Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 14 Mar 2018 19:34:07 -0400
Subject: [PATCH 031/150] putting test files in stix type folder that exists on
 git (bc there are files in it)

---
 stix2/test/test_filesystem.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 4176322..f59136e 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -50,16 +50,16 @@ def fs_sink():
 def bad_json_files():
     # create erroneous JSON files for tests to make sure handled gracefully
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"), "w+") as f:
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f:
         f.write("Im not a JSON file")
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"), "w+") as f:
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f:
         f.write("Im not a JSON formatted file")
 
     yield True  # dummy yield so can have teardown
 
-    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-json.txt"))
-    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-bad-json.json"))
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"))
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"))
 
 
 @pytest.fixture
@@ -68,17 +68,17 @@ def bad_stix_files():
 
     # bad STIX object
     stix_obj = {
-        "id": "indicator--test-bad-stix",
+        "id": "intrusion-set--test-bad-stix",
         "spec_version": "2.0"
         # no "type" field
     }
 
-    with open(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"), "w+") as f:
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f:
         f.write(json.dumps(stix_obj))
 
     yield True  # dummy yield so can have teardown
 
-    os.remove(os.path.join(FS_PATH, "indicator", "indicator--test-non-stix.json"))
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"))
 
 
 @pytest.fixture(scope='module')
@@ -117,18 +117,18 @@ def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
     #  - one file should just be skipped (silently) as its a ".txt" extension
     #  - one file should be parsed and raise Exception bc its not JSON
     try:
-        fs_source.get("indicator--test-bad-json")
+        fs_source.get("intrusion-set--test-bad-json")
     except TypeError as e:
-        assert "indicator--test-bad-json" in str(e)
+        assert "intrusion-set--test-bad-json" in str(e)
         assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
 
 
 def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
     # this tests handling of bad STIX json object
     try:
-        fs_source.get("indicator--test-non-stix")
+        fs_source.get("intrusion-set--test-non-stix")
     except TypeError as e:
-        assert "indicator--test-non-stix" in str(e)
+        assert "intrusion-set--test-non-stix" in str(e)
         assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
 
 

From 7f9a3c49ea3076f37ea56a940c47068050d0c155 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Thu, 15 Mar 2018 13:51:47 -0400
Subject: [PATCH 032/150] Update TAXIICollectionSink.add() to use serialize().
 Call parse() on dict objects. Pass UTF-8 strings to taxii2client. closes #134

---
 stix2/datastore/taxii.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 5af4354..ee3af35 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -67,14 +67,17 @@ class TAXIICollectionSink(DataSink):
         """
         if isinstance(stix_data, _STIXBase):
             # adding python STIX object
-            bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
+            if stix_data["type"] == "bundle":
+                bundle = stix_data.serialize(encoding="utf-8")
+            else:
+                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
 
         elif isinstance(stix_data, dict):
             # adding python dict (of either Bundle or STIX obj)
             if stix_data["type"] == "bundle":
-                bundle = stix_data
+                bundle = parse(stix_data, allow_custom=self.allow_custom, version=version).serialize(encoding="utf-8")
             else:
-                bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
+                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
 
         elif isinstance(stix_data, list):
             # adding list of something - recurse on each
@@ -85,9 +88,9 @@ class TAXIICollectionSink(DataSink):
             # adding json encoded string of STIX content
             stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version)
             if stix_data["type"] == "bundle":
-                bundle = dict(stix_data)
+                bundle = stix_data.serialize(encoding="utf-8")
             else:
-                bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom))
+                bundle = Bundle(stix_data, allow_custom=self.allow_custom).serialize(encoding="utf-8")
 
         else:
             raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle")

From 017df285f9472275482597b1e0e3a6ee8124e8fa Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 15 Mar 2018 16:11:22 -0400
Subject: [PATCH 033/150] so pip installs will not include test data directory

---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index fa68616..b234473 100644
--- a/setup.py
+++ b/setup.py
@@ -54,4 +54,7 @@ setup(
         'stix2-patterns',
         'taxii2-client',
     ],
+    exclude_package_data={
+        'test': ['stix2_data']
+    },
 )

From bc72f934247fa340f41f7ad45a2e4e19cb296bb2 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 15 Mar 2018 18:06:08 -0400
Subject: [PATCH 034/150] Add warning about TAXIICollection bug

Related: #125.
---
 docs/guide/taxii.ipynb | 50 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb
index ad06093..4045a98 100644
--- a/docs/guide/taxii.ipynb
+++ b/docs/guide/taxii.ipynb
@@ -261,25 +261,65 @@
     "\n",
     "tc_store.add(ind)\n"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Bug and Workaround\n",
+    "\n",
+    "You may get an error similar to the following when adding STIX objects to a TAXIICollectionStore or TAXIICollectionSink:\n",
+    "\n",
+    "```\n",
+    "TypeError: Object of type ThreatActor is not JSON serializable\n",
+    "```\n",
+    "\n",
+    "This is a known bug and we are working to fix it. For more information, see [this GitHub issue](https://github.com/oasis-open/cti-python-stix2/issues/125) In the meantime, try this workaround:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tc_sink.add(json.loads(Bundle(ta).serialize()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Or bypass the TAXIICollection altogether and interact with the collection itself:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "collection.add_objects(json.loads(Bundle(ta).serialize()))"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,

From 1f5876d4209fa3b04d4e8447a5c3c7620b01a322 Mon Sep 17 00:00:00 2001
From: Michael K 
Date: Fri, 16 Mar 2018 10:57:31 -0400
Subject: [PATCH 035/150] Undo last commit

Removing directive that is not needed, test packages are already excluded correctly.
---
 setup.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup.py b/setup.py
index b234473..fa68616 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,4 @@ setup(
         'stix2-patterns',
         'taxii2-client',
     ],
-    exclude_package_data={
-        'test': ['stix2_data']
-    },
 )

From c98abb18f54e2f0fd1dae2df8c20339ef5387279 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 16 Mar 2018 14:26:41 -0400
Subject: [PATCH 036/150] Fix docs

---
 stix2/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/base.py b/stix2/base.py
index 6bc3dd4..898f489 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -191,7 +191,7 @@ class _STIXBase(collections.Mapping):
             **kwargs: The arguments for a json.dumps() call.
 
         Returns:
-            dict: The serialized JSON object.
+            str: The serialized JSON object.
 
         Note:
             The argument ``pretty=True`` will output the STIX object following

From 198b4ecffa131f2b01945cda818bf48195bfbeb9 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 16 Mar 2018 16:10:53 -0400
Subject: [PATCH 037/150] Tweak property autodocumenter

- Skip the 'type' property, since it shouldn't be overridden.
- Add info on default values (currently this is only used for timestamp
properties, setting them to the current time).
- Rename it to STIXPropertyDocumenter since they're called properties in
STIX instead of attributes.
---
 docs/conf.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index e53829b..49416a0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -74,7 +74,7 @@ def get_property_type(prop):
     return prop_class
 
 
-class STIXAttributeDocumenter(ClassDocumenter):
+class STIXPropertyDocumenter(ClassDocumenter):
     """Custom Sphinx extension to auto-document STIX properties.
 
     Needed because descendants of _STIXBase use `_properties` dictionaries
@@ -97,16 +97,23 @@ class STIXAttributeDocumenter(ClassDocumenter):
         obj = self.object
         self.add_line(':Properties:', '')
         for prop_name, prop in obj._properties.items():
+            # Skip 'type'
+            if prop_name == 'type':
+                continue
+
             # Add metadata about the property
             prop_type = get_property_type(prop)
             if prop_type == 'List':
                 prop_type = 'List of %ss' % get_property_type(prop.contained)
             if prop.required:
                 prop_type += ', required'
+            if 'Timestamp' in prop_type and hasattr(prop, 'default'):
+                prop_type += ', default: current date/time'
             prop_str = '**%s** (*%s*)' % (prop_name, prop_type)
             self.add_line('    - %s' % prop_str, '')
+
         self.add_line('', '')
 
 
 def setup(app):
-    app.add_autodocumenter(STIXAttributeDocumenter)
+    app.add_autodocumenter(STIXPropertyDocumenter)

From 4e5eb8845807c0c9d3c4cf7707ad6f7caba5ad68 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 16 Mar 2018 16:38:29 -0400
Subject: [PATCH 038/150] Increase versioning code coverage slightly

---
 stix2/test/test_versioning.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py
index 233587e..fa3bddb 100644
--- a/stix2/test/test_versioning.py
+++ b/stix2/test/test_versioning.py
@@ -233,12 +233,19 @@ def test_remove_custom_stix_object():
         ("animal_class", stix2.properties.StringProperty()),
     ])
     class Animal(object):
-        def __init__(self, animal_class=None, **kwargs):
-            if animal_class and animal_class not in ["mammal", "bird"]:
-                raise ValueError("Not a recognized class of animal")
+        pass
 
     animal = Animal(species="lion", animal_class="mammal")
 
     nc = stix2.utils.remove_custom_stix(animal)
 
     assert nc is None
+
+
+def test_remove_custom_stix_no_custom():
+    campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS)
+    campaign_v2 = stix2.utils.remove_custom_stix(campaign_v1)
+
+    assert len(campaign_v1.keys()) == len(campaign_v2.keys())
+    assert campaign_v1.id == campaign_v2.id
+    assert campaign_v1.description == campaign_v2.description

From 32cb62cca24382572f0b767cb0153828bfab8724 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 19 Mar 2018 11:00:09 -0400
Subject: [PATCH 039/150] Make sure we return if there is nothing to add to the
 collection

---
 stix2/datastore/taxii.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index ee3af35..d1cca1e 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -83,6 +83,8 @@ class TAXIICollectionSink(DataSink):
             # adding list of something - recurse on each
             for obj in stix_data:
                 self.add(obj, version=version)
+            else:
+                return  # If the list is empty just return.
 
         elif isinstance(stix_data, str):
             # adding json encoded string of STIX content

From eeb34ebbef90c45fe197c4d63a3339bfa4620138 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 19 Mar 2018 12:03:25 -0400
Subject: [PATCH 040/150] Change logic for case `isinstance(stix_data, list)``

---
 stix2/datastore/taxii.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index d1cca1e..cb136d9 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -83,8 +83,7 @@ class TAXIICollectionSink(DataSink):
             # adding list of something - recurse on each
             for obj in stix_data:
                 self.add(obj, version=version)
-            else:
-                return  # If the list is empty just return.
+            return
 
         elif isinstance(stix_data, str):
             # adding json encoded string of STIX content

From 7880e4a89b1a527c43d17531ad068ee602633724 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 21 Mar 2018 17:19:05 -0400
Subject: [PATCH 041/150] passing TAXII query params in correct format to
 taxii2client; patching bug where TAXII query filters were being applied again
 locally to STIX objects via Filters (which doesnt work)

---
 stix2/datastore/taxii.py     | 46 +++++++++++++++++++++---------------
 stix2/test/test_datastore.py | 14 +++++------
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index cb136d9..0a58763 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -224,18 +224,21 @@ class TAXIICollectionSource(DataSource):
         if _composite_filters:
             query.update(_composite_filters)
 
-        # separate taxii query terms (can be done remotely)
+        # parse taxii query params (that can be applied remotely)
         taxii_filters = self._parse_taxii_filters(query)
 
+        # taxii2client requires query params as keywords
+        taxii_filters_dict = dict((f.property, f.value) for f in taxii_filters)
+
         # query TAXII collection
         try:
-            all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
+            all_data = self.collection.get_objects(**taxii_filters_dict)["objects"]
 
             # deduplicate data (before filtering as reduces wasted filtering)
             all_data = deduplicate(all_data)
 
-            # apply local (CompositeDataSource, TAXIICollectionSource and query filters)
-            all_data = list(apply_common_filters(all_data, query))
+            # apply local (CompositeDataSource, TAXIICollectionSource and query) filters
+            all_data = list(apply_common_filters(all_data, (query - taxii_filters)))
 
         except HTTPError:
             # if resources not found or access is denied from TAXII server, return empty list
@@ -247,30 +250,35 @@ class TAXIICollectionSource(DataSource):
         return stix_objs
 
     def _parse_taxii_filters(self, query):
-        """Parse out TAXII filters that the TAXII server can filter on.
+        """Parse out TAXII filters that the TAXII server can filter on
 
-        Note:
-            For instance - "?match[type]=indicator,sighting" should be in a
-            query dict as follows:
+        Does not put in TAXII spec format as the TAXII2Client (that we use)
+        does this for us.
+
+        NOTE:
+            Currently, the TAXII2Client can handle TAXII filters where the
+            filter value is list, as both a comma-seperated string or python list
+
+            For instance - "?match[type]=indicator,sighting" can be in a
+            filter in any of these formats:
+
+            Filter("type", "", "indicator,sighting")
+
+            Filter("type", "", ["indicator", "sighting"])
 
-            Filter("type", "=", "indicator,sighting")
 
         Args:
-            query (list): list of filters to extract which ones are TAXII
+            query (set): set of filters to extract which ones are TAXII
                 specific.
 
         Returns:
-            params (dict): dict of the TAXII filters but in format required
-                for 'requests.get()'.
+            taxii_filters (set): set of the TAXII filters
 
         """
-        params = {}
+        taxii_filters = set()
 
         for filter_ in query:
             if filter_.property in TAXII_FILTERS:
-                if filter_.property == "added_after":
-                    params[filter_.property] = filter_.value
-                else:
-                    taxii_field = "match[%s]" % filter_.property
-                    params[taxii_field] = filter_.value
-        return params
+                taxii_filters.add(filter_)
+
+        return taxii_filters
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 3490d6a..a164b2f 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -148,18 +148,18 @@ def test_parse_taxii_filters():
         Filter("created_by_ref", "=", "Bane"),
     ]
 
-    expected_params = {
-        "added_after": "2016-02-01T00:00:01.000Z",
-        "match[id]": "taxii stix object ID",
-        "match[type]": "taxii stix object ID",
-        "match[version]": "first"
-    }
+    taxii_filters_expected = set([
+        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
+        Filter("id", "=", "taxii stix object ID"),
+        Filter("type", "=", "taxii stix object ID"),
+        Filter("version", "=", "first")
+    ])
 
     ds = taxii.TAXIICollectionSource(collection)
 
     taxii_filters = ds._parse_taxii_filters(query)
 
-    assert taxii_filters == expected_params
+    assert taxii_filters == taxii_filters_expected
 
 
 def test_add_get_remove_filter():

From 383bf5755e7b40c82617ea073c1a34cc010b074d Mon Sep 17 00:00:00 2001
From: Michael K 
Date: Wed, 21 Mar 2018 22:06:46 -0400
Subject: [PATCH 042/150] remove debug line

---
 stix2/datastore/filesystem.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index b525932..a6f31cf 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -318,7 +318,6 @@ class FileSystemSource(DataSource):
                             stix_obj["id"]
 
                         except (ValueError, KeyError):  # likely not a JSON file
-                            print("filesytem TypeError raised")
                             raise TypeError("STIX JSON object at '{0}' could either not be parsed to "
                                             "JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
 

From 536e56836c1dae63259bd5ab3e97bb5fb711a43f Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 22 Mar 2018 10:59:07 -0400
Subject: [PATCH 043/150] adding Filter check when the Filter property is
 'type', checks the value has no underscores (issue #136)

---
 stix2/datastore/filters.py   |  6 +++-
 stix2/test/test_datastore.py | 53 +++++++++++++++++++++++++-----------
 2 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 5af48cd..9065b61 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -35,7 +35,11 @@ def _check_filter_components(prop, op, value):
 
     if type(value) not in FILTER_VALUE_TYPES:
         # check filter value type is supported
-        raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value))
+        raise TypeError("Filter value of '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value))
+
+    if prop == "type" and "_" in value:
+        # check filter where the property is type, value (type name) cannot have underscores
+        raise ValueError("Filter for property 'type' cannot have its value '%s' include underscores" % value)
 
     return True
 
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index a164b2f..af9176f 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -172,22 +172,6 @@ def test_add_get_remove_filter():
         Filter('labels', 'in', ["heartbleed", "malicious-activity"]),
     ]
 
-    # Invalid filters - wont pass creation
-    # these filters will not be allowed to be created
-    # check proper errors are raised when trying to create them
-
-    with pytest.raises(ValueError) as excinfo:
-        # create Filter that has an operator that is not allowed
-        Filter('modified', '*', 'not supported operator - just place holder')
-    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
-
-    with pytest.raises(TypeError) as excinfo:
-        # create Filter that has a value type that is not allowed
-        Filter('created', '=', object())
-    # On Python 2, the type of object() is `` On Python 3, it's ``.
-    assert str(excinfo.value).startswith("Filter value type")
-    assert str(excinfo.value).endswith("is not supported. The type must be a Python immutable type or dictionary")
-
     assert len(ds.filters) == 0
 
     ds.filters.add(valid_filters[0])
@@ -212,6 +196,43 @@ def test_add_get_remove_filter():
     ds.filters.update(valid_filters)
 
 
+def test_filter_ops_check():
+    # invalid filters - non supported operators
+
+    with pytest.raises(ValueError) as excinfo:
+        # create Filter that has an operator that is not allowed
+        Filter('modified', '*', 'not supported operator')
+    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
+
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "%", "4")
+    assert "Filter operator '%' not supported for specified property" in str(excinfo.value)
+
+
+def test_filter_value_type_check():
+    # invalid filters - non supported value types
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter('created', '=', object())
+    # On Python 2, the type of object() is `` On Python 3, it's ``.
+    assert "Filter value of '' is not supported" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", complex(2, -1))
+    assert "Filter value of '' is not supported" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", set([16, 23]))
+    assert "Filter value of '' is not supported" in str(excinfo.value)
+
+
+def test_filter_type_underscore_check():
+    # check that Filters where property="type", value (name) doesnt have underscores
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "=", "oh_underscore")
+    assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
+
+
 def test_apply_common_filters():
     stix_objs = [
         {

From 9f19245c8aa3f94e86005902d90ef6e8de26fdc5 Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 22 Mar 2018 11:23:48 -0400
Subject: [PATCH 044/150] modify tests for python 3.x acceptance

---
 stix2/test/test_datastore.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index af9176f..70deb38 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -215,7 +215,8 @@ def test_filter_value_type_check():
     with pytest.raises(TypeError) as excinfo:
         Filter('created', '=', object())
     # On Python 2, the type of object() is `` On Python 3, it's ``.
-    assert "Filter value of '' is not supported" in str(excinfo.value)
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
 
     with pytest.raises(TypeError) as excinfo:
         Filter("type", "=", complex(2, -1))

From b1579ae26517422f2d1b35b31bc27341587eb9bc Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 22 Mar 2018 11:36:35 -0400
Subject: [PATCH 045/150] same as last commit

---
 stix2/test/test_datastore.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 70deb38..ff27cce 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -220,7 +220,8 @@ def test_filter_value_type_check():
 
     with pytest.raises(TypeError) as excinfo:
         Filter("type", "=", complex(2, -1))
-    assert "Filter value of '' is not supported" in str(excinfo.value)
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
 
     with pytest.raises(TypeError) as excinfo:
         Filter("type", "=", set([16, 23]))

From 51dcd6c184740215e47ca15f9077338a0e787d56 Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 22 Mar 2018 11:46:49 -0400
Subject: [PATCH 046/150] same as last commit

---
 stix2/test/test_datastore.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index ff27cce..e80e8d8 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -225,7 +225,8 @@ def test_filter_value_type_check():
 
     with pytest.raises(TypeError) as excinfo:
         Filter("type", "=", set([16, 23]))
-    assert "Filter value of '' is not supported" in str(excinfo.value)
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
 
 
 def test_filter_type_underscore_check():

From 89cf4bc38f6b65769a6a989880124ab0905b8626 Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 29 Mar 2018 11:49:30 -0400
Subject: [PATCH 047/150] WIP:allow unknown custom objects to be processed by
 parse; WIP: splitting up parse utility into components; found bug in tests
 that wasnt providing for proper teardown cleaning, fixed

---
 stix2/core.py             | 42 +++++++++++++++++++++++++++++++--------
 stix2/test/test_memory.py | 23 ++++++++++++++++-----
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/stix2/core.py b/stix2/core.py
index b6d295d..64307ff 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -77,14 +77,36 @@ def parse(data, allow_custom=False, version=None):
 
     Args:
         data (str, dict, file-like object): The STIX 2 content to be parsed.
-        allow_custom (bool): Whether to allow custom properties or not.
-            Default: False.
+        allow_custom (bool): Whether to allow custom properties as well unknown
+            custom objects. Note that unknown custom objects cannot be parsed
+            into STIX objects, and will be returned as is. Default: False.
         version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
             None, use latest version.
 
     Returns:
         An instantiated Python STIX object.
 
+    """
+    # convert STIX object to dict, if not already
+    obj = get_dict(data)
+
+    # convert dict to full python-stix2 obj
+    obj = dict_to_stix2(obj, allow_custom, version)
+
+    return obj
+
+
+def dict_to_stix2(stix_dict, allow_custom=False, version=None):
+    """convert dictionary to full python-stix2 object
+
+        Args:
+            stix_dict (dict): a python dictionary of a STIX object
+                that (presumably) is semantically correct to be parsed
+                into a full python-stix2 obj
+            allow_custom (bool): Whether to allow custom properties as well unknown
+                custom objects. Note that unknown custom objects cannot be parsed
+                into STIX objects, and will be returned as is. Default: False.
+
     """
     if not version:
         # Use latest version
@@ -93,16 +115,20 @@ def parse(data, allow_custom=False, version=None):
         v = 'v' + version.replace('.', '')
 
     OBJ_MAP = STIX2_OBJ_MAPS[v]
-    obj = get_dict(data)
 
-    if 'type' not in obj:
-        raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(obj))
+    if 'type' not in stix_dict:
+        raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(stix_dict))
 
     try:
-        obj_class = OBJ_MAP[obj['type']]
+        obj_class = OBJ_MAP[stix_dict['type']]
     except KeyError:
-        raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj['type'])
-    return obj_class(allow_custom=allow_custom, **obj)
+        if allow_custom:
+            # flag allows for unknown custom objects too, but will not
+            # be parsed into STIX object, returned as is
+            return stix_dict
+        raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % stix_dict['type'])
+
+    return obj_class(allow_custom=allow_custom, **stix_dict)
 
 
 def _register_type(new_type, version=None):
diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index 2384848..284c43e 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -136,6 +136,19 @@ def rel_mem_store():
     yield MemoryStore(stix_objs)
 
 
+@pytest.fixture
+def fs_mem_store(request, mem_store):
+    filename = 'memory_test/mem_store.json'
+    mem_store.save_to_file(filename)
+
+    def fin():
+        # teardown, excecuted regardless of exception
+        shutil.rmtree(os.path.dirname(filename))
+    request.addfinalizer(fin)
+
+    return filename
+
+
 def test_memory_source_get(mem_source):
     resp = mem_source.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")
     assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
@@ -187,9 +200,11 @@ def test_memory_store_query_multiple_filters(mem_store):
     assert len(resp) == 1
 
 
-def test_memory_store_save_load_file(mem_store):
-    filename = 'memory_test/mem_store.json'
-    mem_store.save_to_file(filename)
+def test_memory_store_save_load_file(mem_store, fs_mem_store):
+    filename = fs_mem_store  # the fixture fs_mem_store yields filename where the memory store was written to
+
+    # STIX2 contents of mem_store have already been written to file
+    # (this is done in fixture 'fs_mem_store'), so can already read-in here
     contents = open(os.path.abspath(filename)).read()
 
     assert '"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",' in contents
@@ -200,8 +215,6 @@ def test_memory_store_save_load_file(mem_store):
     assert mem_store2.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")
     assert mem_store2.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
 
-    shutil.rmtree(os.path.dirname(filename))
-
 
 def test_memory_store_add_invalid_object(mem_store):
     ind = ('indicator', IND1)  # tuple isn't valid

From aeff8f4bc09f36d9dc744a5781b3c61fae17dc75 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 29 Nov 2017 08:58:01 -0500
Subject: [PATCH 048/150] Create Workbench layer

Contains a default implicit Environment and functions to get all objects
a specific type.
---
 stix2/test/constants.py      |  57 +++++++++++---
 stix2/test/test_workbench.py | 139 +++++++++++++++++++++++++++++++++++
 stix2/workbench.py           |  72 ++++++++++++++++++
 3 files changed, 259 insertions(+), 9 deletions(-)
 create mode 100644 stix2/test/test_workbench.py
 create mode 100644 stix2/workbench.py

diff --git a/stix2/test/constants.py b/stix2/test/constants.py
index 3db39d6..ab7fcf3 100644
--- a/stix2/test/constants.py
+++ b/stix2/test/constants.py
@@ -34,14 +34,18 @@ RELATIONSHIP_IDS = [
     'relationship--a0cbb21c-8daf-4a7f-96aa-7155a4ef8f70'
 ]
 
-# All required args for a Campaign instance
+# *_KWARGS contains all required arguments to create an instance of that STIX object
+# *_MORE_KWARGS contains all the required arguments, plus some optional ones
+
+ATTACK_PATTERN_KWARGS = dict(
+    name="Phishing",
+)
+
 CAMPAIGN_KWARGS = dict(
     name="Green Group Attacks Against Finance",
     description="Campaign by Green Group against a series of targets in the financial services sector.",
 )
 
-
-# All required args for a Campaign instance, plus some optional args
 CAMPAIGN_MORE_KWARGS = dict(
     type='campaign',
     id=CAMPAIGN_ID,
@@ -52,25 +56,29 @@ CAMPAIGN_MORE_KWARGS = dict(
     description="Campaign by Green Group against a series of targets in the financial services sector.",
 )
 
-# Minimum required args for an Identity instance
+COURSE_OF_ACTION_KWARGS = dict(
+    name="Block",
+)
+
 IDENTITY_KWARGS = dict(
     name="John Smith",
     identity_class="individual",
 )
 
-# Minimum required args for an Indicator instance
 INDICATOR_KWARGS = dict(
     labels=['malicious-activity'],
     pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']",
 )
 
-# Minimum required args for a Malware instance
+INTRUSION_SET_KWARGS = dict(
+    name="Bobcat Breakin",
+)
+
 MALWARE_KWARGS = dict(
     labels=['ransomware'],
     name="Cryptolocker",
 )
 
-# All required args for a Malware instance, plus some optional args
 MALWARE_MORE_KWARGS = dict(
     type='malware',
     id=MALWARE_ID,
@@ -81,14 +89,45 @@ MALWARE_MORE_KWARGS = dict(
     description="A ransomware related to ..."
 )
 
-# Minimum required args for a Relationship instance
+OBSERVED_DATA_KWARGS = dict(
+    first_observed=FAKE_TIME,
+    last_observed=FAKE_TIME,
+    number_observed=1,
+    objects={
+        "0": {
+            "type": "windows-registry-key",
+            "key": "HKEY_LOCAL_MACHINE\\System\\Foo\\Bar",
+        }
+    }
+)
+
+REPORT_KWARGS = dict(
+    labels=["campaign"],
+    name="Bad Cybercrime",
+    published=FAKE_TIME,
+    object_refs=[INDICATOR_ID],
+)
+
 RELATIONSHIP_KWARGS = dict(
     relationship_type="indicates",
     source_ref=INDICATOR_ID,
     target_ref=MALWARE_ID,
 )
 
-# Minimum required args for a Sighting instance
 SIGHTING_KWARGS = dict(
     sighting_of_ref=INDICATOR_ID,
 )
+
+THREAT_ACTOR_KWARGS = dict(
+    labels=["crime-syndicate"],
+    name="Evil Org",
+)
+
+TOOL_KWARGS = dict(
+    labels=["remote-access"],
+    name="VNC",
+)
+
+VULNERABILITY_KWARGS = dict(
+    name="Heartbleed",
+)
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
new file mode 100644
index 0000000..5e2809b
--- /dev/null
+++ b/stix2/test/test_workbench.py
@@ -0,0 +1,139 @@
+import stix2
+from stix2.workbench import (add, all_versions, attack_patterns, campaigns,
+                             courses_of_action, create, get, identities,
+                             indicators, intrusion_sets, malware,
+                             observed_data, query, reports, threat_actors,
+                             tools, vulnerabilities)
+
+from .constants import (ATTACK_PATTERN_ID, ATTACK_PATTERN_KWARGS, CAMPAIGN_ID,
+                        CAMPAIGN_KWARGS, COURSE_OF_ACTION_ID,
+                        COURSE_OF_ACTION_KWARGS, IDENTITY_ID, IDENTITY_KWARGS,
+                        INDICATOR_ID, INDICATOR_KWARGS, INTRUSION_SET_ID,
+                        INTRUSION_SET_KWARGS, MALWARE_ID, MALWARE_KWARGS,
+                        OBSERVED_DATA_ID, OBSERVED_DATA_KWARGS, REPORT_ID,
+                        REPORT_KWARGS, THREAT_ACTOR_ID, THREAT_ACTOR_KWARGS,
+                        TOOL_ID, TOOL_KWARGS, VULNERABILITY_ID,
+                        VULNERABILITY_KWARGS)
+
+
+def test_workbench_environment():
+
+    # Create a STIX object
+    ind = create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
+    add(ind)
+
+    resp = get(INDICATOR_ID)
+    assert resp['labels'][0] == 'malicious-activity'
+
+    resp = all_versions(INDICATOR_ID)
+    assert len(resp) == 1
+
+    # Search on something other than id
+    q = [stix2.Filter('type', '=', 'vulnerability')]
+    resp = query(q)
+    assert len(resp) == 0
+
+
+def test_workbench_get_all_attack_patterns():
+    mal = stix2.AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS)
+    add(mal)
+
+    resp = attack_patterns()
+    assert len(resp) == 1
+    assert resp[0].id == ATTACK_PATTERN_ID
+
+
+def test_workbench_get_all_campaigns():
+    cam = stix2.Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    add(cam)
+
+    resp = campaigns()
+    assert len(resp) == 1
+    assert resp[0].id == CAMPAIGN_ID
+
+
+def test_workbench_get_all_courses_of_action():
+    coa = stix2.CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS)
+    add(coa)
+
+    resp = courses_of_action()
+    assert len(resp) == 1
+    assert resp[0].id == COURSE_OF_ACTION_ID
+
+
+def test_workbench_get_all_identities():
+    idty = stix2.Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    add(idty)
+
+    resp = identities()
+    assert len(resp) == 1
+    assert resp[0].id == IDENTITY_ID
+
+
+def test_workbench_get_all_indicators():
+    resp = indicators()
+    assert len(resp) == 1
+    assert resp[0].id == INDICATOR_ID
+
+
+def test_workbench_get_all_intrusion_sets():
+    ins = stix2.IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS)
+    add(ins)
+
+    resp = intrusion_sets()
+    assert len(resp) == 1
+    assert resp[0].id == INTRUSION_SET_ID
+
+
+def test_workbench_get_all_malware():
+    mal = stix2.Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    add(mal)
+
+    resp = malware()
+    assert len(resp) == 1
+    assert resp[0].id == MALWARE_ID
+
+
+def test_workbench_get_all_observed_data():
+    od = stix2.ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS)
+    add(od)
+
+    resp = observed_data()
+    assert len(resp) == 1
+    assert resp[0].id == OBSERVED_DATA_ID
+
+
+def test_workbench_get_all_reports():
+    rep = stix2.Report(id=REPORT_ID, **REPORT_KWARGS)
+    add(rep)
+
+    resp = reports()
+    assert len(resp) == 1
+    assert resp[0].id == REPORT_ID
+
+
+def test_workbench_get_all_threat_actors():
+    thr = stix2.ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS)
+    add(thr)
+
+    resp = threat_actors()
+    assert len(resp) == 1
+    assert resp[0].id == THREAT_ACTOR_ID
+
+
+def test_workbench_get_all_tools():
+    tool = stix2.Tool(id=TOOL_ID, **TOOL_KWARGS)
+    add(tool)
+
+    resp = tools()
+    assert len(resp) == 1
+    assert resp[0].id == TOOL_ID
+
+
+def test_workbench_get_all_vulnerabilities():
+    vuln = stix2.Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS)
+    add(vuln)
+
+    resp = vulnerabilities()
+    assert len(resp) == 1
+    assert resp[0].id == VULNERABILITY_ID
diff --git a/stix2/workbench.py b/stix2/workbench.py
new file mode 100644
index 0000000..55c8009
--- /dev/null
+++ b/stix2/workbench.py
@@ -0,0 +1,72 @@
+"""Functions and class wrappers for interacting with STIX data at a high level.
+"""
+
+from .environment import Environment
+from .sources.filters import Filter
+from .sources.memory import MemoryStore
+
+_environ = Environment(store=MemoryStore())
+
+create = _environ.create
+get = _environ.get
+all_versions = _environ.all_versions
+query = _environ.query
+creator_of = _environ.creator_of
+relationships = _environ.relationships
+related_to = _environ.related_to
+add = _environ.add
+add_filters = _environ.add_filters
+add_filter = _environ.add_filter
+parse = _environ.parse
+add_data_source = _environ.source.add_data_source
+
+
+# Functions to get all objects of a specific type
+
+
+def attack_patterns():
+    return query(Filter('type', '=', 'attack-pattern'))
+
+
+def campaigns():
+    return query(Filter('type', '=', 'campaign'))
+
+
+def courses_of_action():
+    return query(Filter('type', '=', 'course-of-action'))
+
+
+def identities():
+    return query(Filter('type', '=', 'identity'))
+
+
+def indicators():
+    return query(Filter('type', '=', 'indicator'))
+
+
+def intrusion_sets():
+    return query(Filter('type', '=', 'intrusion-set'))
+
+
+def malware():
+    return query(Filter('type', '=', 'malware'))
+
+
+def observed_data():
+    return query(Filter('type', '=', 'observed-data'))
+
+
+def reports():
+    return query(Filter('type', '=', 'report'))
+
+
+def threat_actors():
+    return query(Filter('type', '=', 'threat-actor'))
+
+
+def tools():
+    return query(Filter('type', '=', 'tool'))
+
+
+def vulnerabilities():
+    return query(Filter('type', '=', 'vulnerability'))

From b2613ca62c0c552b37900c1efc4a8b413e2391cb Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 29 Nov 2017 14:12:54 -0500
Subject: [PATCH 049/150] Add Workbench wrapper functions

---
 stix2/test/test_workbench.py | 31 +++++++++++++++++++++++++++++++
 stix2/workbench.py           | 28 ++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 5e2809b..7f3c9fc 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -137,3 +137,34 @@ def test_workbench_get_all_vulnerabilities():
     resp = vulnerabilities()
     assert len(resp) == 1
     assert resp[0].id == VULNERABILITY_ID
+
+
+def test_workbench_relationships():
+    rel = stix2.Relationship(INDICATOR_ID, 'indicates', MALWARE_ID)
+    add(rel)
+
+    ind = get(INDICATOR_ID)
+    resp = ind.relationships()
+    assert len(resp) == 1
+    assert resp[0].relationship_type == 'indicates'
+    assert resp[0].source_ref == INDICATOR_ID
+    assert resp[0].target_ref == MALWARE_ID
+
+
+def test_workbench_created_by():
+    intset = stix2.IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID)
+    add(intset)
+    creator = intset.created_by()
+    assert creator.id == IDENTITY_ID
+
+
+def test_workbench_related():
+    rel1 = stix2.Relationship(MALWARE_ID, 'targets', IDENTITY_ID)
+    rel2 = stix2.Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID)
+    add([rel1, rel2])
+
+    resp = get(MALWARE_ID).related()
+    assert len(resp) == 3
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == IDENTITY_ID for x in resp)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 55c8009..4069309 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -1,6 +1,9 @@
 """Functions and class wrappers for interacting with STIX data at a high level.
 """
 
+from . import (AttackPattern, Campaign, CourseOfAction, CustomObject, Identity,
+               Indicator, IntrusionSet, Malware, ObservedData, Report,
+               ThreatActor, Tool, Vulnerability)
 from .environment import Environment
 from .sources.filters import Filter
 from .sources.memory import MemoryStore
@@ -21,6 +24,31 @@ parse = _environ.parse
 add_data_source = _environ.source.add_data_source
 
 
+# Wrap SDOs with helper functions
+
+
+def created_by_wrapper(self, *args, **kwargs):
+    return _environ.creator_of(self, *args, **kwargs)
+
+
+def relationships_wrapper(self, *args, **kwargs):
+    return _environ.relationships(self, *args, **kwargs)
+
+
+def related_wrapper(self, *args, **kwargs):
+    return _environ.related_to(self, *args, **kwargs)
+
+
+STIX_OBJS = [AttackPattern, Campaign, CourseOfAction, CustomObject, Identity,
+             Indicator, IntrusionSet, Malware, ObservedData, Report,
+             ThreatActor, Tool, Vulnerability]
+
+for obj_type in STIX_OBJS:
+    obj_type.created_by = created_by_wrapper
+    obj_type.relationships = relationships_wrapper
+    obj_type.related = related_wrapper
+
+
 # Functions to get all objects of a specific type
 
 

From 5285934034d575d57dd08e7b77f1de17f52a1941 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 29 Nov 2017 19:25:52 -0500
Subject: [PATCH 050/150] Make Workbench use implicit ObjectFactory

This is needed to implement functions like `set_default_creator`.

The changes to Tox are so that the wrapping we do in workbench doesn't
affect the rest of our tests. If we test them all in one go, pytest will
import all the tests before running any of them. This will cause the
workbench versions of the SDO classes to be used in all tests.
---
 stix2/test/test_tool.py      |  6 +++++
 stix2/test/test_workbench.py | 31 +++++++++++++------------
 stix2/workbench.py           | 44 ++++++++++++++++++++++++++++--------
 tox.ini                      |  3 ++-
 4 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py
index 21ece24..ce99fb8 100644
--- a/stix2/test/test_tool.py
+++ b/stix2/test/test_tool.py
@@ -58,4 +58,10 @@ def test_parse_tool(data):
     assert tool.labels == ["remote-access"]
     assert tool.name == "VNC"
 
+
+def test_tool_no_workbench_wrappers():
+    tool = stix2.Tool(name='VNC', labels=['remote-access'])
+    with pytest.raises(AttributeError):
+        tool.created_by()
+
 # TODO: Add other examples
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 7f3c9fc..6a33f11 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -1,5 +1,8 @@
 import stix2
-from stix2.workbench import (add, all_versions, attack_patterns, campaigns,
+from stix2.workbench import (AttackPattern, Campaign, CourseOfAction, Identity,
+                             Indicator, IntrusionSet, Malware, ObservedData,
+                             Report, ThreatActor, Tool, Vulnerability, add,
+                             all_versions, attack_patterns, campaigns,
                              courses_of_action, create, get, identities,
                              indicators, intrusion_sets, malware,
                              observed_data, query, reports, threat_actors,
@@ -19,7 +22,7 @@ from .constants import (ATTACK_PATTERN_ID, ATTACK_PATTERN_KWARGS, CAMPAIGN_ID,
 def test_workbench_environment():
 
     # Create a STIX object
-    ind = create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
+    ind = create(Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
     add(ind)
 
     resp = get(INDICATOR_ID)
@@ -35,7 +38,7 @@ def test_workbench_environment():
 
 
 def test_workbench_get_all_attack_patterns():
-    mal = stix2.AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS)
+    mal = AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS)
     add(mal)
 
     resp = attack_patterns()
@@ -44,7 +47,7 @@ def test_workbench_get_all_attack_patterns():
 
 
 def test_workbench_get_all_campaigns():
-    cam = stix2.Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
     add(cam)
 
     resp = campaigns()
@@ -53,7 +56,7 @@ def test_workbench_get_all_campaigns():
 
 
 def test_workbench_get_all_courses_of_action():
-    coa = stix2.CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS)
+    coa = CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS)
     add(coa)
 
     resp = courses_of_action()
@@ -62,7 +65,7 @@ def test_workbench_get_all_courses_of_action():
 
 
 def test_workbench_get_all_identities():
-    idty = stix2.Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    idty = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
     add(idty)
 
     resp = identities()
@@ -77,7 +80,7 @@ def test_workbench_get_all_indicators():
 
 
 def test_workbench_get_all_intrusion_sets():
-    ins = stix2.IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS)
+    ins = IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS)
     add(ins)
 
     resp = intrusion_sets()
@@ -86,7 +89,7 @@ def test_workbench_get_all_intrusion_sets():
 
 
 def test_workbench_get_all_malware():
-    mal = stix2.Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
     add(mal)
 
     resp = malware()
@@ -95,7 +98,7 @@ def test_workbench_get_all_malware():
 
 
 def test_workbench_get_all_observed_data():
-    od = stix2.ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS)
+    od = ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS)
     add(od)
 
     resp = observed_data()
@@ -104,7 +107,7 @@ def test_workbench_get_all_observed_data():
 
 
 def test_workbench_get_all_reports():
-    rep = stix2.Report(id=REPORT_ID, **REPORT_KWARGS)
+    rep = Report(id=REPORT_ID, **REPORT_KWARGS)
     add(rep)
 
     resp = reports()
@@ -113,7 +116,7 @@ def test_workbench_get_all_reports():
 
 
 def test_workbench_get_all_threat_actors():
-    thr = stix2.ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS)
+    thr = ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS)
     add(thr)
 
     resp = threat_actors()
@@ -122,7 +125,7 @@ def test_workbench_get_all_threat_actors():
 
 
 def test_workbench_get_all_tools():
-    tool = stix2.Tool(id=TOOL_ID, **TOOL_KWARGS)
+    tool = Tool(id=TOOL_ID, **TOOL_KWARGS)
     add(tool)
 
     resp = tools()
@@ -131,7 +134,7 @@ def test_workbench_get_all_tools():
 
 
 def test_workbench_get_all_vulnerabilities():
-    vuln = stix2.Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS)
+    vuln = Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS)
     add(vuln)
 
     resp = vulnerabilities()
@@ -152,7 +155,7 @@ def test_workbench_relationships():
 
 
 def test_workbench_created_by():
-    intset = stix2.IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID)
+    intset = IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID)
     add(intset)
     creator = intset.created_by()
     assert creator.id == IDENTITY_ID
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 4069309..f5f2d32 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -1,9 +1,18 @@
 """Functions and class wrappers for interacting with STIX data at a high level.
 """
 
-from . import (AttackPattern, Campaign, CourseOfAction, CustomObject, Identity,
-               Indicator, IntrusionSet, Malware, ObservedData, Report,
-               ThreatActor, Tool, Vulnerability)
+from . import AttackPattern as _AttackPattern
+from . import Campaign as _Campaign
+from . import CourseOfAction as _CourseOfAction
+from . import Identity as _Identity
+from . import Indicator as _Indicator
+from . import IntrusionSet as _IntrusionSet
+from . import Malware as _Malware
+from . import ObservedData as _ObservedData
+from . import Report as _Report
+from . import ThreatActor as _ThreatActor
+from . import Tool as _Tool
+from . import Vulnerability as _Vulnerability
 from .environment import Environment
 from .sources.filters import Filter
 from .sources.memory import MemoryStore
@@ -27,6 +36,11 @@ add_data_source = _environ.source.add_data_source
 # Wrap SDOs with helper functions
 
 
+STIX_OBJS = [_AttackPattern, _Campaign, _CourseOfAction, _Identity,
+             _Indicator, _IntrusionSet, _Malware, _ObservedData, _Report,
+             _ThreatActor, _Tool, _Vulnerability]
+
+
 def created_by_wrapper(self, *args, **kwargs):
     return _environ.creator_of(self, *args, **kwargs)
 
@@ -39,14 +53,26 @@ def related_wrapper(self, *args, **kwargs):
     return _environ.related_to(self, *args, **kwargs)
 
 
-STIX_OBJS = [AttackPattern, Campaign, CourseOfAction, CustomObject, Identity,
-             Indicator, IntrusionSet, Malware, ObservedData, Report,
-             ThreatActor, Tool, Vulnerability]
+def constructor_wrapper(obj_type):
+    # Use an intermediate wrapper class so the implicit environment will create objects that have our wrapper functions
+    wrapped_type = type(obj_type.__name__, obj_type.__bases__, dict(
+        created_by=created_by_wrapper,
+        relationships=relationships_wrapper,
+        related=related_wrapper,
+        **obj_type.__dict__
+    ))
 
+    @staticmethod
+    def new_constructor(cls, *args, **kwargs):
+        return _environ.create(wrapped_type, *args, **kwargs)
+    return new_constructor
+
+
+# Create wrapper classes whose constructors call the implicit environment's create()
 for obj_type in STIX_OBJS:
-    obj_type.created_by = created_by_wrapper
-    obj_type.relationships = relationships_wrapper
-    obj_type.related = related_wrapper
+    new_class = type(obj_type.__name__, (), {})
+    new_class.__new__ = constructor_wrapper(obj_type)
+    globals()[obj_type.__name__] = new_class
 
 
 # Functions to get all objects of a specific type
diff --git a/tox.ini b/tox.ini
index bfc8c1b..97d9519 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,8 @@ deps =
     pytest-cov
     coverage
 commands =
-    py.test --cov=stix2 stix2/test/ --cov-report term-missing
+    py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing --cov-append
+    py.test stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
 
 passenv = CI TRAVIS TRAVIS_*
 

From e91b71f3009788f69c7303e22aca6a852c4cacf0 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 14 Mar 2018 12:47:28 -0400
Subject: [PATCH 051/150] Test adding a data source to the workbench

---
 stix2/test/test_workbench.py | 24 +++++++++++++++++++++---
 stix2/workbench.py           |  6 ++++--
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 6a33f11..b84b529 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -1,10 +1,12 @@
+import os
+
 import stix2
 from stix2.workbench import (AttackPattern, Campaign, CourseOfAction, Identity,
                              Indicator, IntrusionSet, Malware, ObservedData,
                              Report, ThreatActor, Tool, Vulnerability, add,
-                             all_versions, attack_patterns, campaigns,
-                             courses_of_action, create, get, identities,
-                             indicators, intrusion_sets, malware,
+                             add_data_source, all_versions, attack_patterns,
+                             campaigns, courses_of_action, create, get,
+                             identities, indicators, intrusion_sets, malware,
                              observed_data, query, reports, threat_actors,
                              tools, vulnerabilities)
 
@@ -171,3 +173,19 @@ def test_workbench_related():
     assert any(x['id'] == CAMPAIGN_ID for x in resp)
     assert any(x['id'] == INDICATOR_ID for x in resp)
     assert any(x['id'] == IDENTITY_ID for x in resp)
+
+    resp = get(MALWARE_ID).related(relationship_type='indicates')
+    assert len(resp) == 1
+
+
+def test_add_data_source():
+    fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
+    fs = stix2.FileSystemSource(fs_path)
+    add_data_source(fs)
+
+    resp = tools()
+    assert len(resp) == 3
+    resp_ids = [tool.id for tool in resp]
+    assert TOOL_ID in resp_ids
+    assert 'tool--03342581-f790-4f03-ba41-e82e67392e23' in resp_ids
+    assert 'tool--242f3da3-4425-4d11-8f5c-b842886da966' in resp_ids
diff --git a/stix2/workbench.py b/stix2/workbench.py
index f5f2d32..60d1165 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -13,10 +13,11 @@ from . import Report as _Report
 from . import ThreatActor as _ThreatActor
 from . import Tool as _Tool
 from . import Vulnerability as _Vulnerability
+from .datastore.filters import Filter
+from .datastore.memory import MemoryStore
 from .environment import Environment
-from .sources.filters import Filter
-from .sources.memory import MemoryStore
 
+# Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
 
 create = _environ.create
@@ -31,6 +32,7 @@ add_filters = _environ.add_filters
 add_filter = _environ.add_filter
 parse = _environ.parse
 add_data_source = _environ.source.add_data_source
+add_data_sources = _environ.source.add_data_sources
 
 
 # Wrap SDOs with helper functions

From 44248092256a0e58902d5089bb387ee1c5e809cb Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 14 Mar 2018 14:19:14 -0400
Subject: [PATCH 052/150] Fix tox config (coverage was incorrectly reported)

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 97d9519..46d88c1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,7 @@ deps =
     pytest-cov
     coverage
 commands =
-    py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing --cov-append
+    py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
     py.test stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
 
 passenv = CI TRAVIS TRAVIS_*

From 53c2d4fadfa0fffc63a05e89fc86ba6394271e98 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 14 Mar 2018 14:33:45 -0400
Subject: [PATCH 053/150] Allow add'l filters in workbench query functions

---
 stix2/test/test_workbench.py | 11 +++++++
 stix2/workbench.py           | 59 +++++++++++++++++++++---------------
 2 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index b84b529..324011b 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -189,3 +189,14 @@ def test_add_data_source():
     assert TOOL_ID in resp_ids
     assert 'tool--03342581-f790-4f03-ba41-e82e67392e23' in resp_ids
     assert 'tool--242f3da3-4425-4d11-8f5c-b842886da966' in resp_ids
+
+
+def test_additional_filter():
+    resp = tools(stix2.Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'))
+    assert len(resp) == 2
+
+
+def test_additional_filters_list():
+    resp = tools([stix2.Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'),
+                  stix2.Filter('name', '=', 'Windows Credential Editor')])
+    assert len(resp) == 1
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 60d1165..49e41c9 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -80,49 +80,60 @@ for obj_type in STIX_OBJS:
 # Functions to get all objects of a specific type
 
 
-def attack_patterns():
-    return query(Filter('type', '=', 'attack-pattern'))
+def query_by_type(obj_type='indicator', filters=None):
+    filter_list = [Filter('type', '=', obj_type)]
+    if filters:
+        if isinstance(filters, list):
+            filter_list += filters
+        else:
+            filter_list.append(filters)
+
+    return query(filter_list)
 
 
-def campaigns():
-    return query(Filter('type', '=', 'campaign'))
+def attack_patterns(filters=None):
+    return query_by_type('attack-pattern', filters)
 
 
-def courses_of_action():
-    return query(Filter('type', '=', 'course-of-action'))
+def campaigns(filters=None):
+    return query_by_type('campaign', filters)
 
 
-def identities():
-    return query(Filter('type', '=', 'identity'))
+def courses_of_action(filters=None):
+    return query_by_type('course-of-action', filters)
 
 
-def indicators():
-    return query(Filter('type', '=', 'indicator'))
+def identities(filters=None):
+    return query_by_type('identity', filters)
 
 
-def intrusion_sets():
-    return query(Filter('type', '=', 'intrusion-set'))
+def indicators(filters=None):
+    return query_by_type('indicator', filters)
 
 
-def malware():
-    return query(Filter('type', '=', 'malware'))
+def intrusion_sets(filters=None):
+    return query_by_type('intrusion-set', filters)
 
 
-def observed_data():
-    return query(Filter('type', '=', 'observed-data'))
+def malware(filters=None):
+    return query_by_type('malware', filters)
 
 
-def reports():
-    return query(Filter('type', '=', 'report'))
+def observed_data(filters=None):
+    return query_by_type('observed-data', filters)
 
 
-def threat_actors():
-    return query(Filter('type', '=', 'threat-actor'))
+def reports(filters=None):
+    return query_by_type('report', filters)
 
 
-def tools():
-    return query(Filter('type', '=', 'tool'))
+def threat_actors(filters=None):
+    return query_by_type('threat-actor', filters)
 
 
-def vulnerabilities():
-    return query(Filter('type', '=', 'vulnerability'))
+def tools(filters=None):
+    return query_by_type('tool', filters)
+
+
+def vulnerabilities(filters=None):
+    return query_by_type('vulnerability', filters)

From eeb94562f9ec19f642d45eaa2bf01dad1fed7028 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 16 Mar 2018 11:40:46 -0400
Subject: [PATCH 054/150] Clean up DataStore return value documentation

---
 stix2/datastore/__init__.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 78f7555..5920673 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -73,7 +73,7 @@ class DataStoreMixin(object):
             stix_id (str): the id of the STIX object to retrieve.
 
         Returns:
-            stix_objs (list): a list of STIX objects
+            list: All versions of the specified STIX object.
 
         """
         try:
@@ -91,7 +91,7 @@ class DataStoreMixin(object):
                 to conduct search on.
 
         Returns:
-            stix_objs (list): a list of STIX objects
+            list: The STIX objects matching the query.
 
         """
         try:
@@ -136,7 +136,7 @@ class DataStoreMixin(object):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of Relationship objects involving the given STIX object.
+            list: The Relationship objects involving the given STIX object.
 
         """
         try:
@@ -164,7 +164,7 @@ class DataStoreMixin(object):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of STIX objects related to the given STIX object.
+            list: The STIX objects related to the given STIX object.
 
         """
         try:
@@ -240,7 +240,7 @@ class DataSource(with_metaclass(ABCMeta)):
                 specified by the "id".
 
         Returns:
-            stix_obj: the STIX object
+            stix_obj: The STIX object.
 
         """
 
@@ -258,7 +258,7 @@ class DataSource(with_metaclass(ABCMeta)):
                 specified by the "id".
 
         Returns:
-            stix_objs (list): a list of STIX objects
+            list: All versions of the specified STIX object.
 
         """
 
@@ -273,7 +273,7 @@ class DataSource(with_metaclass(ABCMeta)):
                 to conduct search on.
 
         Returns:
-            stix_objs (list): a list of STIX objects
+            list: The STIX objects that matched the query.
 
         """
 
@@ -311,7 +311,7 @@ class DataSource(with_metaclass(ABCMeta)):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of Relationship objects involving the given STIX object.
+            list: The Relationship objects involving the given STIX object.
 
         """
         results = []
@@ -356,7 +356,7 @@ class DataSource(with_metaclass(ABCMeta)):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of STIX objects related to the given STIX object.
+            list: The STIX objects related to the given STIX object.
 
         """
         results = []
@@ -425,7 +425,7 @@ class CompositeDataSource(DataSource):
                 to another parent CompositeDataSource), not user supplied.
 
         Returns:
-            stix_obj: the STIX object to be returned.
+            stix_obj: The STIX object to be returned.
 
         """
         if not self.has_data_sources():
@@ -471,7 +471,7 @@ class CompositeDataSource(DataSource):
                 attached to a parent CompositeDataSource), not user supplied.
 
         Returns:
-            all_data (list): list of STIX objects that have the specified id
+            list: The STIX objects that have the specified id.
 
         """
         if not self.has_data_sources():
@@ -510,7 +510,7 @@ class CompositeDataSource(DataSource):
                 attached to a parent CompositeDataSource), not user supplied.
 
         Returns:
-            all_data (list): list of STIX objects to be returned
+            list: The STIX objects to be returned.
 
         """
         if not self.has_data_sources():
@@ -561,7 +561,7 @@ class CompositeDataSource(DataSource):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of Relationship objects involving the given STIX object.
+            list: The Relationship objects involving the given STIX object.
 
         """
         if not self.has_data_sources():
@@ -599,7 +599,7 @@ class CompositeDataSource(DataSource):
                 object is the target_ref. Default: False.
 
         Returns:
-            (list): List of STIX objects related to the given STIX object.
+            list: The STIX objects related to the given STIX object.
 
         """
         if not self.has_data_sources():

From fd6d9f74e955793103a059fdd56b26176e2c39f1 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 16 Mar 2018 15:41:08 -0400
Subject: [PATCH 055/150] Move query_by_type() to DataStoreMixin

---
 stix2/datastore/__init__.py    | 71 ++++++++++++++++++++++++++++++++++
 stix2/environment.py           |  2 +
 stix2/test/test_environment.py |  4 ++
 stix2/workbench.py             | 13 +------
 4 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 5920673..e482288 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -99,6 +99,25 @@ class DataStoreMixin(object):
         except AttributeError:
             raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
+    def query_by_type(self, *args, **kwargs):
+        """Retrieve all objects of the given STIX object type.
+
+        Translate query_by_type() call to the appropriate DataSource call.
+
+        Args:
+            obj_type (str): The STIX object type to retrieve.
+            filters (list, optional): A list of additional filters to apply to
+                the query.
+
+        Returns:
+            list: The STIX objects that matched the query.
+
+        """
+        try:
+            return self.source.query_by_type(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
+
     def creator_of(self, *args, **kwargs):
         """Retrieve the Identity refered to by the object's `created_by_ref`.
 
@@ -277,6 +296,29 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
+    def query_by_type(self, obj_type='indicator', filters=None):
+        """Retrieve all objects of the given STIX object type.
+
+        This helper function is a shortcut that calls query() under the hood.
+
+        Args:
+            obj_type (str): The STIX object type to retrieve.
+            filters (list, optional): A list of additional filters to apply to
+                the query.
+
+        Returns:
+            list: The STIX objects that matched the query.
+
+        """
+        filter_list = [Filter('type', '=', obj_type)]
+        if filters:
+            if isinstance(filters, list):
+                filter_list += filters
+            else:
+                filter_list.append(filters)
+
+        return self.query(filter_list)
+
     def creator_of(self, obj):
         """Retrieve the Identity refered to by the object's `created_by_ref`.
 
@@ -542,6 +584,35 @@ class CompositeDataSource(DataSource):
 
         return all_data
 
+    def query_by_type(self, *args, **kwargs):
+        """Retrieve all objects of the given STIX object type.
+
+        Federate the query to all DataSources attached to the
+        Composite Data Source.
+
+        Args:
+            obj_type (str): The STIX object type to retrieve.
+            filters (list, optional): A list of additional filters to apply to
+                the query.
+
+        Returns:
+            list: The STIX objects that matched the query.
+
+        """
+        if not self.has_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
+        results = []
+        for ds in self.data_sources:
+            results.extend(ds.query_by_type(*args, **kwargs))
+
+        # remove exact duplicates (where duplicates are STIX 2.0
+        # objects with the same 'id' and 'modified' values)
+        if len(results) > 0:
+            results = deduplicate(results)
+
+        return results
+
     def relationships(self, *args, **kwargs):
         """Retrieve Relationships involving the given STIX object.
 
diff --git a/stix2/environment.py b/stix2/environment.py
index eb5583e..95f73e6 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -90,10 +90,12 @@ class Environment(DataStoreMixin):
     .. automethod:: get
     .. automethod:: all_versions
     .. automethod:: query
+    .. automethod:: query_by_type
     .. automethod:: creator_of
     .. automethod:: relationships
     .. automethod:: related_to
     .. automethod:: add
+
     """
 
     def __init__(self, factory=ObjectFactory(), store=None, source=None, sink=None):
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 84ca803..456ec25 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -164,6 +164,10 @@ def test_environment_no_datastore():
         env.query(INDICATOR_ID)
     assert 'Environment has no data source' in str(excinfo.value)
 
+    with pytest.raises(AttributeError) as excinfo:
+        env.query_by_type('indicator')
+    assert 'Environment has no data source' in str(excinfo.value)
+
     with pytest.raises(AttributeError) as excinfo:
         env.relationships(INDICATOR_ID)
     assert 'Environment has no data source' in str(excinfo.value)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 49e41c9..7988329 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -13,7 +13,6 @@ from . import Report as _Report
 from . import ThreatActor as _ThreatActor
 from . import Tool as _Tool
 from . import Vulnerability as _Vulnerability
-from .datastore.filters import Filter
 from .datastore.memory import MemoryStore
 from .environment import Environment
 
@@ -24,6 +23,7 @@ create = _environ.create
 get = _environ.get
 all_versions = _environ.all_versions
 query = _environ.query
+query_by_type = _environ.query_by_type
 creator_of = _environ.creator_of
 relationships = _environ.relationships
 related_to = _environ.related_to
@@ -80,17 +80,6 @@ for obj_type in STIX_OBJS:
 # Functions to get all objects of a specific type
 
 
-def query_by_type(obj_type='indicator', filters=None):
-    filter_list = [Filter('type', '=', obj_type)]
-    if filters:
-        if isinstance(filters, list):
-            filter_list += filters
-        else:
-            filter_list.append(filters)
-
-    return query(filter_list)
-
-
 def attack_patterns(filters=None):
     return query_by_type('attack-pattern', filters)
 

From 61733ad89981bf9c3a50bb109ce992b3b6e41c89 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Mar 2018 13:32:02 -0400
Subject: [PATCH 056/150] Add functions to set ObjectFactory default values

---
 stix2/environment.py           | 54 +++++++++++++++++++++++++++++-----
 stix2/test/test_environment.py |  2 +-
 stix2/test/test_workbench.py   | 43 ++++++++++++++++++++++++++-
 stix2/workbench.py             |  4 +++
 4 files changed, 94 insertions(+), 9 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 95f73e6..69a394f 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -30,19 +30,43 @@ class ObjectFactory(object):
 
         self._defaults = {}
         if created_by_ref:
-            self._defaults['created_by_ref'] = created_by_ref
+            self.set_default_creator(created_by_ref)
         if created:
-            self._defaults['created'] = created
-            # If the user provides a default "created" time, we also want to use
-            # that as the modified time.
-            self._defaults['modified'] = created
+            self.set_default_created(created)
         if external_references:
-            self._defaults['external_references'] = external_references
+            self.set_default_external_refs(external_references)
         if object_marking_refs:
-            self._defaults['object_marking_refs'] = object_marking_refs
+            self.set_default_object_marking_refs(object_marking_refs)
         self._list_append = list_append
         self._list_properties = ['external_references', 'object_marking_refs']
 
+    def set_default_creator(self, creator=None):
+        """Set default value for the `created_by_ref` property.
+
+        """
+        self._defaults['created_by_ref'] = creator
+
+    def set_default_created(self, created=None):
+        """Set default value for the `created` property.
+
+        """
+        self._defaults['created'] = created
+        # If the user provides a default "created" time, we also want to use
+        # that as the modified time.
+        self._defaults['modified'] = created
+
+    def set_default_external_refs(self, external_references=None):
+        """Set default external references.
+
+        """
+        self._defaults['external_references'] = external_references
+
+    def set_default_object_marking_refs(self, object_marking_refs=None):
+        """Set default object markings.
+
+        """
+        self._defaults['object_marking_refs'] = object_marking_refs
+
     def create(self, cls, **kwargs):
         """Create a STIX object using object factory defaults.
 
@@ -115,6 +139,22 @@ class Environment(DataStoreMixin):
         return self.factory.create(*args, **kwargs)
     create.__doc__ = ObjectFactory.create.__doc__
 
+    def set_default_creator(self, *args, **kwargs):
+        return self.factory.set_default_creator(*args, **kwargs)
+    set_default_creator.__doc__ = ObjectFactory.set_default_creator.__doc__
+
+    def set_default_created(self, *args, **kwargs):
+        return self.factory.set_default_created(*args, **kwargs)
+    set_default_created.__doc__ = ObjectFactory.set_default_created.__doc__
+
+    def set_default_external_refs(self, *args, **kwargs):
+        return self.factory.set_default_external_refs(*args, **kwargs)
+    set_default_external_refs.__doc__ = ObjectFactory.set_default_external_refs.__doc__
+
+    def set_default_object_marking_refs(self, *args, **kwargs):
+        return self.factory.set_default_object_marking_refs(*args, **kwargs)
+    set_default_object_marking_refs.__doc__ = ObjectFactory.set_default_object_marking_refs.__doc__
+
     def add_filters(self, *args, **kwargs):
         try:
             return self.source.filters.update(*args, **kwargs)
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 456ec25..8764e25 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -47,7 +47,7 @@ def test_object_factory_created():
     assert ind.modified == FAKE_TIME
 
 
-def test_object_factory_external_resource():
+def test_object_factory_external_reference():
     ext_ref = stix2.ExternalReference(source_name="ACME Threat Intel",
                                       description="Threat report")
     factory = stix2.ObjectFactory(external_references=ext_ref)
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 324011b..c25cf1a 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -7,7 +7,10 @@ from stix2.workbench import (AttackPattern, Campaign, CourseOfAction, Identity,
                              add_data_source, all_versions, attack_patterns,
                              campaigns, courses_of_action, create, get,
                              identities, indicators, intrusion_sets, malware,
-                             observed_data, query, reports, threat_actors,
+                             observed_data, query, reports,
+                             set_default_created, set_default_creator,
+                             set_default_external_refs,
+                             set_default_object_marking_refs, threat_actors,
                              tools, vulnerabilities)
 
 from .constants import (ATTACK_PATTERN_ID, ATTACK_PATTERN_KWARGS, CAMPAIGN_ID,
@@ -200,3 +203,41 @@ def test_additional_filters_list():
     resp = tools([stix2.Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'),
                   stix2.Filter('name', '=', 'Windows Credential Editor')])
     assert len(resp) == 1
+
+
+def test_default_creator():
+    set_default_creator(IDENTITY_ID)
+    campaign = Campaign(**CAMPAIGN_KWARGS)
+
+    assert 'created_by_ref' not in CAMPAIGN_KWARGS
+    assert campaign.created_by_ref == IDENTITY_ID
+
+
+def test_default_created_timestamp():
+    timestamp = "2018-03-19T01:02:03.000Z"
+    set_default_created(timestamp)
+    campaign = Campaign(**CAMPAIGN_KWARGS)
+
+    assert 'created' not in CAMPAIGN_KWARGS
+    assert stix2.utils.format_datetime(campaign.created) == timestamp
+    assert stix2.utils.format_datetime(campaign.modified) == timestamp
+
+
+def test_default_external_refs():
+    ext_ref = stix2.ExternalReference(source_name="ACME Threat Intel",
+                                      description="Threat report")
+    set_default_external_refs(ext_ref)
+    campaign = Campaign(**CAMPAIGN_KWARGS)
+
+    assert campaign.external_references[0].source_name == "ACME Threat Intel"
+    assert campaign.external_references[0].description == "Threat report"
+
+
+def test_default_object_marking_refs():
+    stmt_marking = stix2.StatementMarking("Copyright 2016, Example Corp")
+    mark_def = stix2.MarkingDefinition(definition_type="statement",
+                                       definition=stmt_marking)
+    set_default_object_marking_refs(mark_def)
+    campaign = Campaign(**CAMPAIGN_KWARGS)
+
+    assert campaign.object_marking_refs[0] == mark_def.id
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 7988329..4436609 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -20,6 +20,10 @@ from .environment import Environment
 _environ = Environment(store=MemoryStore())
 
 create = _environ.create
+set_default_creator = _environ.set_default_creator
+set_default_created = _environ.set_default_created
+set_default_external_refs = _environ.set_default_external_refs
+set_default_object_marking_refs = _environ.set_default_object_marking_refs
 get = _environ.get
 all_versions = _environ.all_versions
 query = _environ.query

From 4fb24f14de7e277f57695edb5b66c678c119272c Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Mar 2018 15:56:20 -0400
Subject: [PATCH 057/150] Allow passing add'l filters to related_to()

---
 stix2/datastore/__init__.py  | 16 +++++++++++++---
 stix2/test/test_workbench.py | 13 +++++++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index e482288..f02d773 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -313,7 +313,7 @@ class DataSource(with_metaclass(ABCMeta)):
         filter_list = [Filter('type', '=', obj_type)]
         if filters:
             if isinstance(filters, list):
-                filter_list += filters
+                filter_list.extend(filters)
             else:
                 filter_list.append(filters)
 
@@ -380,7 +380,7 @@ class DataSource(with_metaclass(ABCMeta)):
 
         return results
 
-    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False):
+    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=None):
         """Retrieve STIX Objects that have a Relationship involving the given
         STIX object.
 
@@ -396,6 +396,8 @@ class DataSource(with_metaclass(ABCMeta)):
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this
                 object is the target_ref. Default: False.
+            filters (list): list of additional filters the related objects must
+                match.
 
         Returns:
             list: The STIX objects related to the given STIX object.
@@ -416,8 +418,16 @@ class DataSource(with_metaclass(ABCMeta)):
             ids.update((r.source_ref, r.target_ref))
         ids.remove(obj_id)
 
+        # Assemble filters
+        filter_list = []
+        if filters:
+            if isinstance(filters, list):
+                filter_list.extend(filters)
+            else:
+                filter_list.append(filters)
+
         for i in ids:
-            results.append(self.get(i))
+            results.extend(self.query(filter_list + [Filter('id', '=', i)]))
 
         return results
 
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index c25cf1a..a8edfbc 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -181,6 +181,19 @@ def test_workbench_related():
     assert len(resp) == 1
 
 
+def test_workbench_related_with_filters():
+    malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID)
+    rel = stix2.Relationship(malware.id, 'variant-of', MALWARE_ID)
+    add([malware, rel])
+
+    filters = [stix2.Filter('created_by_ref', '=', IDENTITY_ID)]
+    resp = get(MALWARE_ID).related(filters=filters)
+
+    assert len(resp) == 1
+    assert resp[0].name == malware.name
+    assert resp[0].created_by_ref == IDENTITY_ID
+
+
 def test_add_data_source():
     fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
     fs = stix2.FileSystemSource(fs_path)

From e48e0886a8741746a9cfad2ef160a5f3fd43fb9c Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Mar 2018 17:41:16 -0400
Subject: [PATCH 058/150] Improve code coverage slightly

Environment will always have a CompositeDataSource, so the try/catches
in add_filter/s did not make sense.
---
 stix2/environment.py         | 10 ++--------
 stix2/test/test_workbench.py |  4 ++++
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 69a394f..9f55a7e 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -156,16 +156,10 @@ class Environment(DataStoreMixin):
     set_default_object_marking_refs.__doc__ = ObjectFactory.set_default_object_marking_refs.__doc__
 
     def add_filters(self, *args, **kwargs):
-        try:
-            return self.source.filters.update(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
+        return self.source.filters.update(*args, **kwargs)
 
     def add_filter(self, *args, **kwargs):
-        try:
-            return self.source.filters.add(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
+        return self.source.filters.add(*args, **kwargs)
 
     def parse(self, *args, **kwargs):
         return _parse(*args, **kwargs)
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index a8edfbc..13aad23 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -193,6 +193,10 @@ def test_workbench_related_with_filters():
     assert resp[0].name == malware.name
     assert resp[0].created_by_ref == IDENTITY_ID
 
+    # filters arg can also be single filter
+    resp = get(MALWARE_ID).related(filters=filters[0])
+    assert len(resp) == 1
+
 
 def test_add_data_source():
     fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")

From efede514539a4a0310f4276643ad50f838e7f957 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 21 Mar 2018 13:56:50 -0400
Subject: [PATCH 059/150] Skip documenting some workbench stuff

---
 docs/conf.py       | 11 +++++++++++
 stix2/workbench.py | 17 +++++++++--------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 49416a0..0764454 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -115,5 +115,16 @@ class STIXPropertyDocumenter(ClassDocumenter):
         self.add_line('', '')
 
 
+def autodoc_skipper(app, what, name, obj, skip, options):
+    """Customize Sphinx to skip some member we don't want documented.
+
+    Skips anything containing ':autodoc-skip:' in its docstring.
+    """
+    if obj.__doc__ and ':autodoc-skip:' in obj.__doc__:
+        return skip or True
+    return skip
+
+
 def setup(app):
     app.add_autodocumenter(STIXPropertyDocumenter)
+    app.connect('autodoc-skip-member', autodoc_skipper)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 4436609..3a26a64 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -47,24 +47,24 @@ STIX_OBJS = [_AttackPattern, _Campaign, _CourseOfAction, _Identity,
              _ThreatActor, _Tool, _Vulnerability]
 
 
-def created_by_wrapper(self, *args, **kwargs):
+def _created_by_wrapper(self, *args, **kwargs):
     return _environ.creator_of(self, *args, **kwargs)
 
 
-def relationships_wrapper(self, *args, **kwargs):
+def _relationships_wrapper(self, *args, **kwargs):
     return _environ.relationships(self, *args, **kwargs)
 
 
-def related_wrapper(self, *args, **kwargs):
+def _related_wrapper(self, *args, **kwargs):
     return _environ.related_to(self, *args, **kwargs)
 
 
-def constructor_wrapper(obj_type):
+def _constructor_wrapper(obj_type):
     # Use an intermediate wrapper class so the implicit environment will create objects that have our wrapper functions
     wrapped_type = type(obj_type.__name__, obj_type.__bases__, dict(
-        created_by=created_by_wrapper,
-        relationships=relationships_wrapper,
-        related=related_wrapper,
+        created_by=_created_by_wrapper,
+        relationships=_relationships_wrapper,
+        related=_related_wrapper,
         **obj_type.__dict__
     ))
 
@@ -77,7 +77,8 @@ def constructor_wrapper(obj_type):
 # Create wrapper classes whose constructors call the implicit environment's create()
 for obj_type in STIX_OBJS:
     new_class = type(obj_type.__name__, (), {})
-    new_class.__new__ = constructor_wrapper(obj_type)
+    new_class.__new__ = _constructor_wrapper(obj_type)
+    new_class.__doc__ = ':autodoc-skip:'
     globals()[obj_type.__name__] = new_class
 
 

From b9bbd03481c3184af4a6a0606e14b21aca2082bf Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 22 Mar 2018 10:55:10 -0400
Subject: [PATCH 060/150] Update workbench imports and documentation

Import a bunch of stuff so users can just "from stix2.workbench import *" and
not need to import other stuff (e.g. MarkingDefinition, Cyber Observable
Object classes, etc.) from stix2.
---
 .isort.cfg                   |   1 +
 docs/api/stix2.workbench.rst |   5 ++
 stix2/__init__.py            |   1 +
 stix2/test/test_workbench.py |  48 +++++------
 stix2/workbench.py           | 149 +++++++++++++++++++++++++++++++++--
 5 files changed, 176 insertions(+), 28 deletions(-)
 create mode 100644 docs/api/stix2.workbench.rst

diff --git a/.isort.cfg b/.isort.cfg
index 0fadb83..cca9d19 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,5 +1,6 @@
 [settings]
 not_skip = __init__.py
+skip = workbench.py
 known_third_party =
     dateutil,
     ordereddict,
diff --git a/docs/api/stix2.workbench.rst b/docs/api/stix2.workbench.rst
new file mode 100644
index 0000000..19345f0
--- /dev/null
+++ b/docs/api/stix2.workbench.rst
@@ -0,0 +1,5 @@
+workbench
+===============
+
+.. automodule:: stix2.workbench
+   :members:
\ No newline at end of file
diff --git a/stix2/__init__.py b/stix2/__init__.py
index 401d44b..89043ec 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -11,6 +11,7 @@
    patterns
    properties
    utils
+   workbench
    v20.common
    v20.observables
    v20.sdo
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 13aad23..a2016cc 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -1,14 +1,16 @@
 import os
 
 import stix2
-from stix2.workbench import (AttackPattern, Campaign, CourseOfAction, Identity,
-                             Indicator, IntrusionSet, Malware, ObservedData,
-                             Report, ThreatActor, Tool, Vulnerability, add,
-                             add_data_source, all_versions, attack_patterns,
-                             campaigns, courses_of_action, create, get,
-                             identities, indicators, intrusion_sets, malware,
-                             observed_data, query, reports,
-                             set_default_created, set_default_creator,
+from stix2.workbench import (AttackPattern, Campaign, CourseOfAction,
+                             ExternalReference, FileSystemSource, Filter,
+                             Identity, Indicator, IntrusionSet, Malware,
+                             MarkingDefinition, ObservedData, Relationship,
+                             Report, StatementMarking, ThreatActor, Tool,
+                             Vulnerability, add, add_data_source, all_versions,
+                             attack_patterns, campaigns, courses_of_action,
+                             create, get, identities, indicators,
+                             intrusion_sets, malware, observed_data, query,
+                             reports, set_default_created, set_default_creator,
                              set_default_external_refs,
                              set_default_object_marking_refs, threat_actors,
                              tools, vulnerabilities)
@@ -37,7 +39,7 @@ def test_workbench_environment():
     assert len(resp) == 1
 
     # Search on something other than id
-    q = [stix2.Filter('type', '=', 'vulnerability')]
+    q = [Filter('type', '=', 'vulnerability')]
     resp = query(q)
     assert len(resp) == 0
 
@@ -148,7 +150,7 @@ def test_workbench_get_all_vulnerabilities():
 
 
 def test_workbench_relationships():
-    rel = stix2.Relationship(INDICATOR_ID, 'indicates', MALWARE_ID)
+    rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID)
     add(rel)
 
     ind = get(INDICATOR_ID)
@@ -167,8 +169,8 @@ def test_workbench_created_by():
 
 
 def test_workbench_related():
-    rel1 = stix2.Relationship(MALWARE_ID, 'targets', IDENTITY_ID)
-    rel2 = stix2.Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID)
+    rel1 = Relationship(MALWARE_ID, 'targets', IDENTITY_ID)
+    rel2 = Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID)
     add([rel1, rel2])
 
     resp = get(MALWARE_ID).related()
@@ -183,10 +185,10 @@ def test_workbench_related():
 
 def test_workbench_related_with_filters():
     malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID)
-    rel = stix2.Relationship(malware.id, 'variant-of', MALWARE_ID)
+    rel = Relationship(malware.id, 'variant-of', MALWARE_ID)
     add([malware, rel])
 
-    filters = [stix2.Filter('created_by_ref', '=', IDENTITY_ID)]
+    filters = [Filter('created_by_ref', '=', IDENTITY_ID)]
     resp = get(MALWARE_ID).related(filters=filters)
 
     assert len(resp) == 1
@@ -200,7 +202,7 @@ def test_workbench_related_with_filters():
 
 def test_add_data_source():
     fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
-    fs = stix2.FileSystemSource(fs_path)
+    fs = FileSystemSource(fs_path)
     add_data_source(fs)
 
     resp = tools()
@@ -212,13 +214,13 @@ def test_add_data_source():
 
 
 def test_additional_filter():
-    resp = tools(stix2.Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'))
+    resp = tools(Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'))
     assert len(resp) == 2
 
 
 def test_additional_filters_list():
-    resp = tools([stix2.Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'),
-                  stix2.Filter('name', '=', 'Windows Credential Editor')])
+    resp = tools([Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'),
+                  Filter('name', '=', 'Windows Credential Editor')])
     assert len(resp) == 1
 
 
@@ -241,8 +243,8 @@ def test_default_created_timestamp():
 
 
 def test_default_external_refs():
-    ext_ref = stix2.ExternalReference(source_name="ACME Threat Intel",
-                                      description="Threat report")
+    ext_ref = ExternalReference(source_name="ACME Threat Intel",
+                                description="Threat report")
     set_default_external_refs(ext_ref)
     campaign = Campaign(**CAMPAIGN_KWARGS)
 
@@ -251,9 +253,9 @@ def test_default_external_refs():
 
 
 def test_default_object_marking_refs():
-    stmt_marking = stix2.StatementMarking("Copyright 2016, Example Corp")
-    mark_def = stix2.MarkingDefinition(definition_type="statement",
-                                       definition=stmt_marking)
+    stmt_marking = StatementMarking("Copyright 2016, Example Corp")
+    mark_def = MarkingDefinition(definition_type="statement",
+                                 definition=stmt_marking)
     set_default_object_marking_refs(mark_def)
     campaign = Campaign(**CAMPAIGN_KWARGS)
 
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 3a26a64..91d626d 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -1,4 +1,24 @@
 """Functions and class wrappers for interacting with STIX data at a high level.
+
+.. autofunction:: create
+.. autofunction:: set_default_creator
+.. autofunction:: set_default_created
+.. autofunction:: set_default_external_refs
+.. autofunction:: set_default_object_marking_refs
+.. autofunction:: get
+.. autofunction:: all_versions
+.. autofunction:: query
+.. autofunction:: query_by_type
+.. autofunction:: creator_of
+.. autofunction:: relationships
+.. autofunction:: related_to
+.. autofunction:: add
+.. autofunction:: add_filters
+.. autofunction:: add_filter
+.. autofunction:: parse
+.. autofunction:: add_data_source
+.. autofunction:: add_data_sources
+
 """
 
 from . import AttackPattern as _AttackPattern
@@ -13,8 +33,21 @@ from . import Report as _Report
 from . import ThreatActor as _ThreatActor
 from . import Tool as _Tool
 from . import Vulnerability as _Vulnerability
-from .datastore.memory import MemoryStore
-from .environment import Environment
+from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # noqa: F401
+               Bundle, CustomExtension, CustomMarking, CustomObservable,
+               Directory, DomainName, EmailAddress, EmailMessage,
+               EmailMIMEComponent, Environment, ExtensionsProperty,
+               ExternalReference, File, FileSystemSource, Filter,
+               GranularMarking, HTTPRequestExt, ICMPExt, IPv4Address,
+               IPv6Address, KillChainPhase, MACAddress, MarkingDefinition,
+               MemoryStore, Mutex, NetworkTraffic, NTFSExt, parse_observable,
+               PDFExt, Process, RasterImageExt, Relationship, Sighting,
+               SocketExt, Software, StatementMarking, TAXIICollectionSource,
+               TCPExt, TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, TLPMarking,
+               UNIXAccountExt, URL, UserAccount, WindowsPEBinaryExt,
+               WindowsPEOptionalHeaderType, WindowsPESection,
+               WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
+               WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -46,6 +79,24 @@ STIX_OBJS = [_AttackPattern, _Campaign, _CourseOfAction, _Identity,
              _Indicator, _IntrusionSet, _Malware, _ObservedData, _Report,
              _ThreatActor, _Tool, _Vulnerability]
 
+STIX_OBJ_DOCS = """
+
+.. method:: created_by(*args, **kwargs)
+
+        {}
+
+.. method:: relationships(*args, **kwargs)
+
+        {}
+
+.. method:: related(*args, **kwargs)
+
+        {}
+
+""".format(_environ.creator_of.__doc__,
+           _environ.relationships.__doc__,
+           _environ.related_to.__doc__)
+
 
 def _created_by_wrapper(self, *args, **kwargs):
     return _environ.creator_of(self, *args, **kwargs)
@@ -76,58 +127,146 @@ def _constructor_wrapper(obj_type):
 
 # Create wrapper classes whose constructors call the implicit environment's create()
 for obj_type in STIX_OBJS:
-    new_class = type(obj_type.__name__, (), {})
-    new_class.__new__ = _constructor_wrapper(obj_type)
-    new_class.__doc__ = ':autodoc-skip:'
+    new_class_dict = {
+        '__new__': _constructor_wrapper(obj_type),
+        '__doc__': 'Workbench wrapper around the `{0} `__. object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
+    }
+    new_class = type(obj_type.__name__, (), new_class_dict)
+
     globals()[obj_type.__name__] = new_class
+    new_class = None
 
 
 # Functions to get all objects of a specific type
 
 
 def attack_patterns(filters=None):
+    """Retrieve all Attack Pattern objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('attack-pattern', filters)
 
 
 def campaigns(filters=None):
+    """Retrieve all Campaign objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('campaign', filters)
 
 
 def courses_of_action(filters=None):
+    """Retrieve all Course of Action objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('course-of-action', filters)
 
 
 def identities(filters=None):
+    """Retrieve all Identity objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('identity', filters)
 
 
 def indicators(filters=None):
+    """Retrieve all Indicator objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('indicator', filters)
 
 
 def intrusion_sets(filters=None):
+    """Retrieve all Intrusion Set objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('intrusion-set', filters)
 
 
 def malware(filters=None):
+    """Retrieve all Malware objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('malware', filters)
 
 
 def observed_data(filters=None):
+    """Retrieve all Observed Data objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('observed-data', filters)
 
 
 def reports(filters=None):
+    """Retrieve all Report objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('report', filters)
 
 
 def threat_actors(filters=None):
+    """Retrieve all Threat Actor objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('threat-actor', filters)
 
 
 def tools(filters=None):
+    """Retrieve all Tool objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('tool', filters)
 
 
 def vulnerabilities(filters=None):
+    """Retrieve all Vulnerability objects.
+
+    Args:
+        filters (list, optional): A list of additional filters to apply to
+            the query.
+
+    """
     return query_by_type('vulnerability', filters)

From 98cc86eef6e341b149494779c0318f963ed20af2 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 23 Mar 2018 13:13:41 -0400
Subject: [PATCH 061/150] Fix workbench wrapped classes for `parse()`.

The wrapped classes need to be in the OBJ_MAP mapping, not just the
workbench.py globals.
---
 stix2/workbench.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/stix2/workbench.py b/stix2/workbench.py
index 91d626d..f57b5f4 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -21,6 +21,7 @@
 
 """
 
+import stix2
 from . import AttackPattern as _AttackPattern
 from . import Campaign as _Campaign
 from . import CourseOfAction as _CourseOfAction
@@ -121,20 +122,28 @@ def _constructor_wrapper(obj_type):
 
     @staticmethod
     def new_constructor(cls, *args, **kwargs):
-        return _environ.create(wrapped_type, *args, **kwargs)
+        x = _environ.create(wrapped_type, *args, **kwargs)
+        return x
     return new_constructor
 
 
-# Create wrapper classes whose constructors call the implicit environment's create()
-for obj_type in STIX_OBJS:
-    new_class_dict = {
-        '__new__': _constructor_wrapper(obj_type),
-        '__doc__': 'Workbench wrapper around the `{0} `__. object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
-    }
-    new_class = type(obj_type.__name__, (), new_class_dict)
+def _setup_workbench():
+    # Create wrapper classes whose constructors call the implicit environment's create()
+    for obj_type in STIX_OBJS:
+        new_class_dict = {
+            '__new__': _constructor_wrapper(obj_type),
+            '__doc__': 'Workbench wrapper around the `{0} `__. object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
+        }
+        new_class = type(obj_type.__name__, (), new_class_dict)
 
-    globals()[obj_type.__name__] = new_class
-    new_class = None
+        # Add our new class to this module's globals and to the library-wide mapping.
+        # This allows parse() to use the wrapped classes.
+        globals()[obj_type.__name__] = new_class
+        stix2.OBJ_MAP[obj_type._type] = new_class
+        new_class = None
+
+
+_setup_workbench()
 
 
 # Functions to get all objects of a specific type

From 4a2ac6df3a8528eaac738e6d74b8ad0822d46aaa Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 30 Mar 2018 11:53:15 -0400
Subject: [PATCH 062/150] Add/fix workbench docs, rename add() -> save()

---
 docs/guide/workbench.ipynb   | 472 +++++++++++++++++++++++++++++++++++
 stix2/datastore/__init__.py  |  10 +-
 stix2/test/test_workbench.py |  38 +--
 stix2/workbench.py           |   6 +-
 4 files changed, 501 insertions(+), 25 deletions(-)
 create mode 100644 docs/guide/workbench.ipynb

diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb
new file mode 100644
index 0000000..bc7942a
--- /dev/null
+++ b/docs/guide/workbench.ipynb
@@ -0,0 +1,472 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "nbsphinx": "hidden"
+   },
+   "outputs": [],
+   "source": [
+    "# Delete this cell to re-enable tracebacks\n",
+    "import sys\n",
+    "ipython = get_ipython()\n",
+    "\n",
+    "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n",
+    "                   exception_only=False, running_compiled_code=False):\n",
+    "    etype, value, tb = sys.exc_info()\n",
+    "    return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n",
+    "\n",
+    "ipython.showtraceback = hide_traceback"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "nbsphinx": "hidden"
+   },
+   "outputs": [],
+   "source": [
+    "# JSON output syntax highlighting\n",
+    "from __future__ import print_function\n",
+    "from pygments import highlight\n",
+    "from pygments.lexers import JsonLexer\n",
+    "from pygments.formatters import HtmlFormatter\n",
+    "from six.moves import builtins\n",
+    "from IPython.display import display, HTML\n",
+    "\n",
+    "def json_print(inpt):\n",
+    "    string = str(inpt)\n",
+    "    if string[0] == '{':\n",
+    "        formatter = HtmlFormatter()\n",
+    "        display(HTML('{}'.format(\n",
+    "                     formatter.get_style_defs('.highlight'),\n",
+    "                     highlight(string, JsonLexer(), formatter))))\n",
+    "    else:\n",
+    "        builtins.print(inpt)\n",
+    "\n",
+    "globals()['print'] = json_print"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Using A Workbench"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The [Workbench API](../api/stix2.workbench.rst) hides most of the complexity of the rest of the library to make it easy to interact with STIX data. To use it, just import everything from ``stix2.workbench``:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from stix2.workbench import *"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Retrieving STIX Data\n",
+    "\n",
+    "To get some STIX data to work with, let's set up a DataSource and add it to our workbench."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from taxii2client import Collection\n",
+    "\n",
+    "collection = Collection(\"http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/\", user=\"admin\", password=\"Password0\")\n",
+    "tc_source = TAXIICollectionSource(collection)\n",
+    "add_data_source(tc_source)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true
+   },
+   "source": [
+    "Now we can get all of the indicators from the data source."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = indicators()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Similar functions are available for the other STIX Object types. See the full list [here](../api/stix2.workbench.rst#stix2.workbench.attack_patterns).\n",
+    "\n",
+    "If you want to only retrieve *some* indicators, you can pass in one or more [Filters](../api/datastore/stix2.datastore.filters.rst). This example finds all the indicators created by a specific identity:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = indicators(filters=Filter('created_by_ref', '=', 'identity--adede3e8-bf44-4e6f-b3c9-1958cbc3b188'))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The objects returned let you easily traverse their relationships. Get all Relationship objects involving that object with ``.relationships()``, all other objects related to this object with ``.related()``, and the Identity object for the creator of the object (if one exists) with ``.created_by()``. For full details on these methods and their arguments, see the [Workbench API](../api/stix2.workbench.rst) documentation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\n",
+      "indicates\n",
+      "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in indicators():\n",
+    "    for rel in i.relationships():\n",
+    "        print(rel.source_ref)\n",
+    "        print(rel.relationship_type)\n",
+    "        print(rel.target_ref)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "malware",\n",
+       "    "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",\n",
+       "    "created": "2017-01-27T13:49:53.997Z",\n",
+       "    "modified": "2017-01-27T13:49:53.997Z",\n",
+       "    "name": "Poison Ivy",\n",
+       "    "description": "Poison Ivy",\n",
+       "    "labels": [\n",
+       "        "remote-access-trojan"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "for i in indicators():\n",
+    "    for obj in i.related():\n",
+    "        print(obj)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If there are a lot of related objects, you can narrow it down by passing in one or more [Filters](../api/datastore/stix2.datastore.filters.rst) just as before. For example, if we want to get only the indicators related to a specific piece of malware (and not any entities that use it or are targeted by it):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",\n",
+       "    "created": "2014-05-08T09:00:00.000Z",\n",
+       "    "modified": "2014-05-08T09:00:00.000Z",\n",
+       "    "name": "File hash for Poison Ivy variant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",\n",
+       "    "valid_from": "2014-05-08T09:00:00Z",\n",
+       "    "labels": [\n",
+       "        "file-hash-watchlist"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "malware = get('malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111')\n",
+    "indicator = malware.related(filters=Filter('type', '=', 'indicator'))\n",
+    "print(indicator[0])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Creating STIX Data\n",
+    "\n",
+    "To create a STIX object, just use that object's class constructor. Once it's created, add it to the workbench with [save()](../api/datastore/stix2.workbench.rst#stix2.workbench.save)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "identity = Identity(name=\"ACME Threat Intel Co.\", identity_class=\"organization\")\n",
+    "save(identity)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also set defaults for certain properties when creating objects. For example, let's set the default creator to be the identity object we just created:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "set_default_creator(identity)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now when we create an indicator (or any other STIX Domain Object), it will automatically have the right ``create_by_ref`` value."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ACME Threat Intel Co.\n"
+     ]
+    }
+   ],
+   "source": [
+    "indicator = Indicator(labels=[\"malicious-activity\"], pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
+    "save(indicator)\n",
+    "\n",
+    "indicator_creator = get(indicator.created_by_ref)\n",
+    "print(indicator_creator.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Defaults can also be set for the [created timestamp](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_created), [external references](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_external_refs) and [object marking references](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_object_marking_refs)."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index f02d773..9bf23a2 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -181,6 +181,8 @@ class DataStoreMixin(object):
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this
                 object is the target_ref. Default: False.
+            filters (list): list of additional filters the related objects must
+                match.
 
         Returns:
             list: The STIX objects related to the given STIX object.
@@ -194,8 +196,8 @@ class DataStoreMixin(object):
     def add(self, *args, **kwargs):
         """Method for storing STIX objects.
 
-        Define custom behavior before storing STIX objects using the associated
-        DataSink. Translates add() to the appropriate DataSink call.
+        Defines custom behavior before storing STIX objects using the
+        appropriate method call on the associated DataSink.
 
         Args:
             stix_objs (list): a list of STIX objects
@@ -416,7 +418,7 @@ class DataSource(with_metaclass(ABCMeta)):
         ids = set()
         for r in rels:
             ids.update((r.source_ref, r.target_ref))
-        ids.remove(obj_id)
+        ids.discard(obj_id)
 
         # Assemble filters
         filter_list = []
@@ -678,6 +680,8 @@ class CompositeDataSource(DataSource):
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this
                 object is the target_ref. Default: False.
+            filters (list): list of additional filters the related objects must
+                match.
 
         Returns:
             list: The STIX objects related to the given STIX object.
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index a2016cc..7857eb2 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -6,12 +6,12 @@ from stix2.workbench import (AttackPattern, Campaign, CourseOfAction,
                              Identity, Indicator, IntrusionSet, Malware,
                              MarkingDefinition, ObservedData, Relationship,
                              Report, StatementMarking, ThreatActor, Tool,
-                             Vulnerability, add, add_data_source, all_versions,
+                             Vulnerability, add_data_source, all_versions,
                              attack_patterns, campaigns, courses_of_action,
                              create, get, identities, indicators,
                              intrusion_sets, malware, observed_data, query,
-                             reports, set_default_created, set_default_creator,
-                             set_default_external_refs,
+                             reports, save, set_default_created,
+                             set_default_creator, set_default_external_refs,
                              set_default_object_marking_refs, threat_actors,
                              tools, vulnerabilities)
 
@@ -30,7 +30,7 @@ def test_workbench_environment():
 
     # Create a STIX object
     ind = create(Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS)
-    add(ind)
+    save(ind)
 
     resp = get(INDICATOR_ID)
     assert resp['labels'][0] == 'malicious-activity'
@@ -46,7 +46,7 @@ def test_workbench_environment():
 
 def test_workbench_get_all_attack_patterns():
     mal = AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS)
-    add(mal)
+    save(mal)
 
     resp = attack_patterns()
     assert len(resp) == 1
@@ -55,7 +55,7 @@ def test_workbench_get_all_attack_patterns():
 
 def test_workbench_get_all_campaigns():
     cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
-    add(cam)
+    save(cam)
 
     resp = campaigns()
     assert len(resp) == 1
@@ -64,7 +64,7 @@ def test_workbench_get_all_campaigns():
 
 def test_workbench_get_all_courses_of_action():
     coa = CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS)
-    add(coa)
+    save(coa)
 
     resp = courses_of_action()
     assert len(resp) == 1
@@ -73,7 +73,7 @@ def test_workbench_get_all_courses_of_action():
 
 def test_workbench_get_all_identities():
     idty = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
-    add(idty)
+    save(idty)
 
     resp = identities()
     assert len(resp) == 1
@@ -88,7 +88,7 @@ def test_workbench_get_all_indicators():
 
 def test_workbench_get_all_intrusion_sets():
     ins = IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS)
-    add(ins)
+    save(ins)
 
     resp = intrusion_sets()
     assert len(resp) == 1
@@ -97,7 +97,7 @@ def test_workbench_get_all_intrusion_sets():
 
 def test_workbench_get_all_malware():
     mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
-    add(mal)
+    save(mal)
 
     resp = malware()
     assert len(resp) == 1
@@ -106,7 +106,7 @@ def test_workbench_get_all_malware():
 
 def test_workbench_get_all_observed_data():
     od = ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS)
-    add(od)
+    save(od)
 
     resp = observed_data()
     assert len(resp) == 1
@@ -115,7 +115,7 @@ def test_workbench_get_all_observed_data():
 
 def test_workbench_get_all_reports():
     rep = Report(id=REPORT_ID, **REPORT_KWARGS)
-    add(rep)
+    save(rep)
 
     resp = reports()
     assert len(resp) == 1
@@ -124,7 +124,7 @@ def test_workbench_get_all_reports():
 
 def test_workbench_get_all_threat_actors():
     thr = ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS)
-    add(thr)
+    save(thr)
 
     resp = threat_actors()
     assert len(resp) == 1
@@ -133,7 +133,7 @@ def test_workbench_get_all_threat_actors():
 
 def test_workbench_get_all_tools():
     tool = Tool(id=TOOL_ID, **TOOL_KWARGS)
-    add(tool)
+    save(tool)
 
     resp = tools()
     assert len(resp) == 1
@@ -142,7 +142,7 @@ def test_workbench_get_all_tools():
 
 def test_workbench_get_all_vulnerabilities():
     vuln = Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS)
-    add(vuln)
+    save(vuln)
 
     resp = vulnerabilities()
     assert len(resp) == 1
@@ -151,7 +151,7 @@ def test_workbench_get_all_vulnerabilities():
 
 def test_workbench_relationships():
     rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID)
-    add(rel)
+    save(rel)
 
     ind = get(INDICATOR_ID)
     resp = ind.relationships()
@@ -163,7 +163,7 @@ def test_workbench_relationships():
 
 def test_workbench_created_by():
     intset = IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID)
-    add(intset)
+    save(intset)
     creator = intset.created_by()
     assert creator.id == IDENTITY_ID
 
@@ -171,7 +171,7 @@ def test_workbench_created_by():
 def test_workbench_related():
     rel1 = Relationship(MALWARE_ID, 'targets', IDENTITY_ID)
     rel2 = Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID)
-    add([rel1, rel2])
+    save([rel1, rel2])
 
     resp = get(MALWARE_ID).related()
     assert len(resp) == 3
@@ -186,7 +186,7 @@ def test_workbench_related():
 def test_workbench_related_with_filters():
     malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID)
     rel = Relationship(malware.id, 'variant-of', MALWARE_ID)
-    add([malware, rel])
+    save([malware, rel])
 
     filters = [Filter('created_by_ref', '=', IDENTITY_ID)]
     resp = get(MALWARE_ID).related(filters=filters)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index f57b5f4..0338353 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -12,7 +12,7 @@
 .. autofunction:: creator_of
 .. autofunction:: relationships
 .. autofunction:: related_to
-.. autofunction:: add
+.. autofunction:: save
 .. autofunction:: add_filters
 .. autofunction:: add_filter
 .. autofunction:: parse
@@ -65,7 +65,7 @@ query_by_type = _environ.query_by_type
 creator_of = _environ.creator_of
 relationships = _environ.relationships
 related_to = _environ.related_to
-add = _environ.add
+save = _environ.add
 add_filters = _environ.add_filters
 add_filter = _environ.add_filter
 parse = _environ.parse
@@ -132,7 +132,7 @@ def _setup_workbench():
     for obj_type in STIX_OBJS:
         new_class_dict = {
             '__new__': _constructor_wrapper(obj_type),
-            '__doc__': 'Workbench wrapper around the `{0} `__. object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
+            '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
         }
         new_class = type(obj_type.__name__, (), new_class_dict)
 

From d453bf6f1a2c9ef2ac5014faa48ad941bbac26df Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 30 Mar 2018 13:12:51 -0400
Subject: [PATCH 063/150] Add a couple granular markings tests

---
 stix2/test/test_granular_markings.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py
index f8fc803..9e024a1 100644
--- a/stix2/test/test_granular_markings.py
+++ b/stix2/test/test_granular_markings.py
@@ -2,6 +2,7 @@
 import pytest
 
 from stix2 import TLP_RED, Malware, markings
+from stix2.exceptions import MarkingNotFoundError
 
 from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST
 from .constants import MARKING_IDS
@@ -546,6 +547,20 @@ def test_remove_marking_bad_selector():
         markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"])
 
 
+def test_remove_marking_not_present():
+    before = Malware(
+        granular_markings=[
+            {
+                "selectors": ["description"],
+                "marking_ref": MARKING_IDS[0]
+            }
+        ],
+        **MALWARE_KWARGS
+    )
+    with pytest.raises(MarkingNotFoundError):
+        markings.remove_markings(before, [MARKING_IDS[1]], ["description"])
+
+
 IS_MARKED_TEST_DATA = [
     Malware(
         granular_markings=[
@@ -1044,3 +1059,10 @@ def test_clear_marking_bad_selector(data, selector):
     """Test bad selector raises exception."""
     with pytest.raises(AssertionError):
         markings.clear_markings(data, selector)
+
+
+@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA)
+def test_clear_marking_not_present(data):
+    """Test clearing markings for a selector that has no associated markings."""
+    with pytest.raises(MarkingNotFoundError):
+        data = markings.clear_markings(data, ["labels"])

From 90834c5b953c060a08bf866e437114b3bca3c4d4 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 30 Mar 2018 13:21:07 -0400
Subject: [PATCH 064/150] docs and tests for parse() mod

---
 docs/guide/parsing.ipynb  | 395 +++++++++++++++++++++++++++++++++++++-
 stix2/core.py             |  19 +-
 stix2/test/test_custom.py |  14 ++
 3 files changed, 421 insertions(+), 7 deletions(-)

diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb
index d24f994..b3460b3 100644
--- a/docs/guide/parsing.ipynb
+++ b/docs/guide/parsing.ipynb
@@ -63,21 +63,120 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well."
+    "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string, dictionary, or file-like object. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well.\n",
+    "\n",
+    "**Parsing a string**"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "observed-data\n",
-      "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038\n"
+      "\n"
      ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "observed-data",\n",
+       "    "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",\n",
+       "    "created": "2016-04-06T19:58:16.000Z",\n",
+       "    "modified": "2016-04-06T19:58:16.000Z",\n",
+       "    "first_observed": "2015-12-21T19:00:00Z",\n",
+       "    "last_observed": "2015-12-21T19:00:00Z",\n",
+       "    "number_observed": 50,\n",
+       "    "objects": {\n",
+       "        "0": {\n",
+       "            "type": "file",\n",
+       "            "hashes": {\n",
+       "                "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038"\n",
+       "            }\n",
+       "        }\n",
+       "    }\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -102,8 +201,292 @@
     "}\"\"\"\n",
     "\n",
     "obj = parse(input_string)\n",
-    "print(obj.type)\n",
-    "print(obj.objects[\"0\"].hashes['SHA-256'])"
+    "print(type(obj))\n",
+    "print(obj)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Parsing a dictionary**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "identity",\n",
+       "    "id": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",\n",
+       "    "created": "2015-12-21T19:59:11.000Z",\n",
+       "    "modified": "2015-12-21T19:59:11.000Z",\n",
+       "    "name": "Cole Powers",\n",
+       "    "identity_class": "individual"\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "input_dict = {\n",
+    "    \"type\": \"identity\",\n",
+    "    \"id\": \"identity--311b2d2d-f010-5473-83ec-1edf84858f4c\",\n",
+    "    \"created\": \"2015-12-21T19:59:11Z\",\n",
+    "    \"modified\": \"2015-12-21T19:59:11Z\",\n",
+    "    \"name\": \"Cole Powers\",\n",
+    "    \"identity_class\": \"individual\"\n",
+    "}\n",
+    "\n",
+    "obj = parse(input_dict)\n",
+    "print(type(obj))\n",
+    "print(obj)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Parsing a file-like object**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "course-of-action",\n",
+       "    "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:30:41.022Z",\n",
+       "    "modified": "2017-05-31T21:30:41.022Z",\n",
+       "    "name": "Data from Network Shared Drive Mitigation",\n",
+       "    "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]"\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "file_handle = open(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json\")\n",
+    "\n",
+    "obj = parse(file_handle)\n",
+    "print(type(obj))\n",
+    "print(obj)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Parsing Custom STIX Content"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument *allow_custom=True*. When *allow_custom* is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX2 domain objects and/or previously defined custom defined STIX2 objects. If the conversion cannot be completed (and *allow_custom* is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX2 content as valid STIX2 objects and return them. **Warning: Specifying *allow_custom* may lead to critical errors if further processing (searching, filtering, modifying etc...) of the custom STIX2 content occurs where the custom STIX2 content supplied is not valid STIX2**. This is an axiomatic possibility as the STIX2 library cannot guarantee proper processing of unknown custom STIX2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n",
+    "\n",
+    "For examples on parsing STIX2 objects with custom STIX properties, see [Custom STIX Content:Custom Properties](custom.ipynb#Custom-Properties)\n",
+    "\n",
+    "For examples on parsing defined custom STIX2 objects, see [Custom STIX Content: Custom STIX Object Types](custom.ipynb#Custom-STIX-Object-Types)\n",
+    "\n",
+    "For the case where it is desired to retrieve STIX2 content from a source (e.g. file system, TAXII) that may possibly have custom STIX2 content unknown to the user, the user can create a STIX2 DataStore/Source with the flag *allow_custom=True*. As aforementioned this will configure the DataStore/Source to allow for unknown STIX2 content to be returned (albeit not converted to full STIX2 domain objects and properties); notable processing capabilites of the STIX2 library may be precluded by the unknown STIX2 content, if the content is not valid or actual STIX2 domain objects and properties."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from taxii2client import Collection\n",
+    "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n",
+    "\n",
+    "# to allow for the retrieval of unknown custom STIX2 content,\n",
+    "# just create *Stores/*Sources with the 'allow_custom' flag\n",
+    "\n",
+    "# create FileSystemStore\n",
+    "fs = FileSystemSource(\"/path/to/stix2_data/\", allow_custom=True)\n",
+    "\n",
+    "# create TAXIICollectionSource\n",
+    "colxn = Collection('http://taxii_url')\n",
+    "ts = TAXIICollectionSource(colxn, allow_custom=True)\n"
    ]
   }
  ],
diff --git a/stix2/core.py b/stix2/core.py
index 64307ff..7de7984 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -73,7 +73,7 @@ STIX2_OBJ_MAPS = {}
 
 
 def parse(data, allow_custom=False, version=None):
-    """Deserialize a string or file-like object into a STIX object.
+    """Convert a string, dict or file-like object into a STIX object.
 
     Args:
         data (str, dict, file-like object): The STIX 2 content to be parsed.
@@ -86,6 +86,13 @@ def parse(data, allow_custom=False, version=None):
     Returns:
         An instantiated Python STIX object.
 
+    WARNING: 'allow_custom=True' will allow for the return of any supplied STIX
+        dict(s) that cannot be found to map to any known STIX object types (both STIX2
+        domain objects or defined custom STIX2 objects); NO validation is done. This is
+        done to allow the processing of possibly unknown custom STIX objects (example
+        scenario: I need to query a third-party TAXII endpoint that could provide custom
+        STIX objects that I dont know about ahead of time)
+
     """
     # convert STIX object to dict, if not already
     obj = get_dict(data)
@@ -107,6 +114,16 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None):
                 custom objects. Note that unknown custom objects cannot be parsed
                 into STIX objects, and will be returned as is. Default: False.
 
+        Returns:
+            An instantiated Python STIX object
+
+        WARNING: 'allow_custom=True' will allow for the return of any supplied STIX
+        dict(s) that cannot be found to map to any known STIX object types (both STIX2
+        domain objects or defined custom STIX2 objects); NO validation is done. This is
+        done to allow the processing of possibly unknown custom STIX objects (example
+        scenario: I need to query a third-party TAXII endpoint that could provide custom
+        STIX objects that I dont know about ahead of time)
+
     """
     if not version:
         # Use latest version
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 76ad61b..cc8b32b 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -221,6 +221,20 @@ def test_parse_unregistered_custom_object_type():
     assert "use the CustomObject decorator." in str(excinfo.value)
 
 
+def test_parse_unregistered_custom_object_type_w_allow_custom():
+    """parse an unknown custom object, allowed by passing
+    'allow_custom' flag
+    """
+    nt_string = """{
+        "type": "x-foobar-observable",
+        "created": "2015-12-21T19:59:11Z",
+        "property1": "something"
+    }"""
+
+    custom_obj = stix2.parse(nt_string, allow_custom=True)
+    assert custom_obj["type"] == "x-foobar-observable"
+
+
 @stix2.observables.CustomObservable('x-new-observable', [
     ('property1', stix2.properties.StringProperty(required=True)),
     ('property2', stix2.properties.IntegerProperty()),

From 0bb1bb37f1bf3a13ccec7a8910a85dc062c693f5 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 2 Apr 2018 07:40:25 -0400
Subject: [PATCH 065/150] Call parse() on stix_data

# Fix on MemorySource.load_from_file()
---
 stix2/datastore/memory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index e057271..1c4ff6c 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -295,5 +295,5 @@ class MemorySource(DataSource):
             for stix_obj in stix_data["objects"]:
                 _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"]))
         else:
-            _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=version))
+            _add(self, stix_data=parse(stix_data, allow_custom=self.allow_custom, version=version))
     load_from_file.__doc__ = MemoryStore.load_from_file.__doc__

From c5185332754b6b6e23c8bc0d28b5a03be7c93a8b Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 2 Apr 2018 07:51:51 -0400
Subject: [PATCH 066/150] Update stix2 setup configuration

# Remove taxii2-client as a requirement to install stix2
# Add taxii2-client to the Tox configuration instead
# Re-factor the version and description loading
---
 setup.py | 8 ++++----
 tox.ini  | 1 +
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index fa68616..5edf20f 100644
--- a/setup.py
+++ b/setup.py
@@ -4,11 +4,12 @@ import os.path
 
 from setuptools import find_packages, setup
 
-here = os.path.abspath(os.path.dirname(__file__))
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_FILE = os.path.join(BASE_DIR, 'stix2', 'version.py')
 
 
 def get_version():
-    with open('stix2/version.py', encoding="utf-8") as f:
+    with open(VERSION_FILE) as f:
         for line in f.readlines():
             if line.startswith("__version__"):
                 version = line.split()[-1].strip('"')
@@ -16,7 +17,7 @@ def get_version():
         raise AttributeError("Package does not have a __version__")
 
 
-with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
+with open('README.rst') as f:
     long_description = f.read()
 
 
@@ -52,6 +53,5 @@ setup(
         'simplejson',
         'six',
         'stix2-patterns',
-        'taxii2-client',
     ],
 )
diff --git a/tox.ini b/tox.ini
index bfc8c1b..ed26bc0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,7 @@ deps =
     pytest
     pytest-cov
     coverage
+    taxii2-client
 commands =
     py.test --cov=stix2 stix2/test/ --cov-report term-missing
 

From 470b3ec092039dcd32f5a1e30acb76d1fbcdf75e Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 2 Apr 2018 09:25:26 -0400
Subject: [PATCH 067/150] Update setup.py to include taxii2-client as an extra
 dependency

---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index 5edf20f..9700ec8 100644
--- a/setup.py
+++ b/setup.py
@@ -54,4 +54,7 @@ setup(
         'six',
         'stix2-patterns',
     ],
+    extras_require={
+        'taxii': ['taxii2-client']
+    }
 )

From 940afb0012f67b2cbe3e8ee2660d027968799112 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 2 Apr 2018 09:54:22 -0400
Subject: [PATCH 068/150] Require a type when querying by type

---
 stix2/datastore/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 9bf23a2..890ac45 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -298,7 +298,7 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
-    def query_by_type(self, obj_type='indicator', filters=None):
+    def query_by_type(self, obj_type, filters=None):
         """Retrieve all objects of the given STIX object type.
 
         This helper function is a shortcut that calls query() under the hood.

From 3abfe7868a1393e7c60529a6ceca5aba306978f1 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 2 Apr 2018 16:38:04 -0400
Subject: [PATCH 069/150] Add more patterning tests

... and fix bugs detected in doing so
---
 stix2/__init__.py                      |   5 +-
 stix2/patterns.py                      |  66 ++++----
 stix2/test/test_pattern_expressions.py | 203 ++++++++++++++++++++++++-
 3 files changed, 239 insertions(+), 35 deletions(-)

diff --git a/stix2/__init__.py b/stix2/__init__.py
index 401d44b..9ab99d8 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -31,11 +31,12 @@ from .environment import Environment, ObjectFactory
 from .markings import (add_markings, clear_markings, get_markings, is_marked,
                        remove_markings, set_markings)
 from .patterns import (AndBooleanExpression, AndObservationExpression,
-                       BasicObjectPathComponent, EqualityComparisonExpression,
+                       BasicObjectPathComponent, BinaryConstant,
+                       BooleanConstant, EqualityComparisonExpression,
                        FloatConstant, FollowedByObservationExpression,
                        GreaterThanComparisonExpression,
                        GreaterThanEqualComparisonExpression, HashConstant,
-                       HexConstant, IntegerConstant,
+                       HexConstant, InComparisonExpression, IntegerConstant,
                        IsSubsetComparisonExpression,
                        IsSupersetComparisonExpression,
                        LessThanComparisonExpression,
diff --git a/stix2/patterns.py b/stix2/patterns.py
index 94ae7d2..23ce71b 100644
--- a/stix2/patterns.py
+++ b/stix2/patterns.py
@@ -3,8 +3,11 @@
 
 import base64
 import binascii
+import datetime
 import re
 
+from .utils import parse_into_datetime
+
 
 def escape_quotes_and_backslashes(s):
     return s.replace(u'\\', u'\\\\').replace(u"'", u"\\'")
@@ -24,10 +27,13 @@ class StringConstant(_Constant):
 
 class TimestampConstant(_Constant):
     def __init__(self, value):
-        self.value = value
+        try:
+            self.value = parse_into_datetime(value)
+        except Exception:
+            raise ValueError("must be a datetime object or timestamp string.")
 
     def __str__(self):
-        return "t'%s'" % escape_quotes_and_backslashes(self.value)
+        return "t%s" % repr(self.value)
 
 
 class IntegerConstant(_Constant):
@@ -46,7 +52,7 @@ class FloatConstant(_Constant):
         try:
             self.value = float(value)
         except Exception:
-            raise ValueError("must be an float.")
+            raise ValueError("must be a float.")
 
     def __str__(self):
         return "%s" % self.value
@@ -56,24 +62,29 @@ class BooleanConstant(_Constant):
     def __init__(self, value):
         if isinstance(value, bool):
             self.value = value
+            return
 
         trues = ['true', 't']
         falses = ['false', 'f']
         try:
             if value.lower() in trues:
                 self.value = True
-            if value.lower() in falses:
+                return
+            elif value.lower() in falses:
                 self.value = False
+                return
         except AttributeError:
             if value == 1:
                 self.value = True
-            if value == 0:
+                return
+            elif value == 0:
                 self.value = False
+                return
 
         raise ValueError("must be a boolean value.")
 
     def __str__(self):
-        return "%s" % self.value
+        return str(self.value).lower()
 
 
 _HASH_REGEX = {
@@ -132,20 +143,25 @@ class ListConstant(_Constant):
         self.value = values
 
     def __str__(self):
-        return "(" + ", ".join([("%s" % x) for x in self.value]) + ")"
+        return "(" + ", ".join([("%s" % make_constant(x)) for x in self.value]) + ")"
 
 
 def make_constant(value):
+    try:
+        return parse_into_datetime(value)
+    except ValueError:
+        pass
+
     if isinstance(value, str):
         return StringConstant(value)
+    elif isinstance(value, bool):
+        return BooleanConstant(value)
     elif isinstance(value, int):
         return IntegerConstant(value)
     elif isinstance(value, float):
         return FloatConstant(value)
     elif isinstance(value, list):
         return ListConstant(value)
-    elif isinstance(value, bool):
-        return BooleanConstant(value)
     else:
         raise ValueError("Unable to create a constant from %s" % value)
 
@@ -210,15 +226,12 @@ class ObjectPath(object):
 
 
 class _PatternExpression(object):
-
-    @staticmethod
-    def escape_quotes_and_backslashes(s):
-        return s.replace(u'\\', u'\\\\').replace(u"'", u"\\'")
+    pass
 
 
 class _ComparisonExpression(_PatternExpression):
     def __init__(self, operator, lhs, rhs, negated=False):
-        if operator == "=" and isinstance(rhs, ListConstant):
+        if operator == "=" and isinstance(rhs, (ListConstant, list)):
             self.operator = "IN"
         else:
             self.operator = operator
@@ -234,13 +247,6 @@ class _ComparisonExpression(_PatternExpression):
         self.root_type = self.lhs.object_type_name
 
     def __str__(self):
-        # if isinstance(self.rhs, list):
-        #     final_rhs = []
-        #     for r in self.rhs:
-        #         final_rhs.append("'" + self.escape_quotes_and_backslashes("%s" % r) + "'")
-        #     rhs_string = "(" + ", ".join(final_rhs) + ")"
-        # else:
-        #     rhs_string = self.rhs
         if self.negated:
             return "%s NOT %s %s" % (self.lhs, self.operator, self.rhs)
         else:
@@ -383,7 +389,7 @@ class RepeatQualifier(_ExpressionQualifier):
         elif isinstance(times_to_repeat, int):
             self.times_to_repeat = IntegerConstant(times_to_repeat)
         else:
-            raise ValueError("%s is not a valid argument for a Within Qualifier" % times_to_repeat)
+            raise ValueError("%s is not a valid argument for a Repeat Qualifier" % times_to_repeat)
 
     def __str__(self):
         return "REPEATS %s TIMES" % self.times_to_repeat
@@ -404,18 +410,18 @@ class WithinQualifier(_ExpressionQualifier):
 
 class StartStopQualifier(_ExpressionQualifier):
     def __init__(self, start_time, stop_time):
-        if isinstance(start_time, IntegerConstant):
+        if isinstance(start_time, TimestampConstant):
             self.start_time = start_time
-        elif isinstance(start_time, int):
-            self.start_time = IntegerConstant(start_time)
+        elif isinstance(start_time, datetime.date):
+            self.start_time = TimestampConstant(start_time)
         else:
-            raise ValueError("%s is not a valid argument for a Within Qualifier" % start_time)
-        if isinstance(stop_time, IntegerConstant):
+            raise ValueError("%s is not a valid argument for a Start/Stop Qualifier" % start_time)
+        if isinstance(stop_time, TimestampConstant):
             self.stop_time = stop_time
-        elif isinstance(stop_time, int):
-            self.stop_time = IntegerConstant(stop_time)
+        elif isinstance(stop_time, datetime.date):
+            self.stop_time = TimestampConstant(stop_time)
         else:
-            raise ValueError("%s is not a valid argument for a Within Qualifier" % stop_time)
+            raise ValueError("%s is not a valid argument for a Start/Stop Qualifier" % stop_time)
 
     def __str__(self):
         return "START %s STOP %s" % (self.start_time, self.stop_time)
diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py
index 0db1083..363458a 100644
--- a/stix2/test/test_pattern_expressions.py
+++ b/stix2/test/test_pattern_expressions.py
@@ -1,3 +1,7 @@
+import datetime
+
+import pytest
+
 import stix2
 
 
@@ -67,7 +71,11 @@ def test_file_observable_expression():
     assert str(exp) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']"  # noqa
 
 
-def test_multiple_file_observable_expression():
+@pytest.mark.parametrize("observation_class, op", [
+    (stix2.AndObservationExpression, 'AND'),
+    (stix2.OrObservationExpression, 'OR'),
+])
+def test_multiple_file_observable_expression(observation_class, op):
     exp1 = stix2.EqualityComparisonExpression("file:hashes.'SHA-256'",
                                               stix2.HashConstant(
                                                   "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c",
@@ -81,8 +89,8 @@ def test_multiple_file_observable_expression():
                                                   'SHA-256'))
     op1_exp = stix2.ObservationExpression(bool1_exp)
     op2_exp = stix2.ObservationExpression(exp3)
-    exp = stix2.AndObservationExpression([op1_exp, op2_exp])
-    assert str(exp) == "[file:hashes.'SHA-256' = 'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c' OR file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a6879f'] AND [file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']"  # noqa
+    exp = observation_class([op1_exp, op2_exp])
+    assert str(exp) == "[file:hashes.'SHA-256' = 'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c' OR file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a6879f'] {} [file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']".format(op)  # noqa
 
 
 def test_root_types():
@@ -120,6 +128,31 @@ def test_greater_than():
     assert str(exp) == "[file:extensions.windows-pebinary-ext.sections[*].entropy > 7.0]"
 
 
+def test_less_than():
+    exp = stix2.LessThanComparisonExpression("file:size",
+                                             1024)
+    assert str(exp) == "file:size < 1024"
+
+
+def test_greater_than_or_equal():
+    exp = stix2.GreaterThanEqualComparisonExpression("file:size",
+                                                     1024)
+    assert str(exp) == "file:size >= 1024"
+
+
+def test_less_than_or_equal():
+    exp = stix2.LessThanEqualComparisonExpression("file:size",
+                                                  1024)
+    assert str(exp) == "file:size <= 1024"
+
+
+def test_not():
+    exp = stix2.LessThanComparisonExpression("file:size",
+                                             1024,
+                                             negated=True)
+    assert str(exp) == "file:size NOT < 1024"
+
+
 def test_and_observable_expression():
     exp1 = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:account_type",
                                                                           "unix"),
@@ -145,6 +178,15 @@ def test_and_observable_expression():
     assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']"  # noqa
 
 
+def test_invalid_and_observable_expression():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("user-account:display_name",
+                                                                       "admin"),
+                                    stix2.EqualityComparisonExpression("email-addr:display_name",
+                                                                       stix2.StringConstant("admin"))])
+    assert "All operands to an 'AND' expression must have the same object type" in str(excinfo)
+
+
 def test_hex():
     exp_and = stix2.AndBooleanExpression([stix2.EqualityComparisonExpression("file:mime_type",
                                                                              "image/bmp"),
@@ -175,3 +217,158 @@ def test_set_op():
 def test_timestamp():
     ts = stix2.TimestampConstant('2014-01-13T07:03:17Z')
     assert str(ts) == "t'2014-01-13T07:03:17Z'"
+
+
+def test_boolean():
+    exp = stix2.EqualityComparisonExpression("email-message:is_multipart",
+                                             True)
+    assert str(exp) == "email-message:is_multipart = true"
+
+
+def test_binary():
+    const = stix2.BinaryConstant("dGhpcyBpcyBhIHRlc3Q=")
+    exp = stix2.EqualityComparisonExpression("artifact:payload_bin",
+                                             const)
+    assert str(exp) == "artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q='"
+
+
+def test_list():
+    exp = stix2.InComparisonExpression("process:name",
+                                       ['proccy', 'proximus', 'badproc'])
+    assert str(exp) == "process:name IN ('proccy', 'proximus', 'badproc')"
+
+
+def test_list2():
+    # alternate way to construct an "IN" Comparison Expression
+    exp = stix2.EqualityComparisonExpression("process:name",
+                                             ['proccy', 'proximus', 'badproc'])
+    assert str(exp) == "process:name IN ('proccy', 'proximus', 'badproc')"
+
+
+def test_invalid_constant_type():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.EqualityComparisonExpression("artifact:payload_bin",
+                                           {'foo': 'bar'})
+    assert 'Unable to create a constant' in str(excinfo)
+
+
+def test_invalid_integer_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.IntegerConstant('foo')
+    assert 'must be an integer' in str(excinfo)
+
+
+def test_invalid_timestamp_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.TimestampConstant('foo')
+    assert 'must be a datetime object or timestamp string' in str(excinfo)
+
+
+def test_invalid_float_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.FloatConstant('foo')
+    assert 'must be a float' in str(excinfo)
+
+
+@pytest.mark.parametrize("data, result", [
+    (True, True),
+    (False, False),
+    ('True', True),
+    ('False', False),
+    ('true', True),
+    ('false', False),
+    ('t', True),
+    ('f', False),
+    ('T', True),
+    ('F', False),
+    (1, True),
+    (0, False),
+])
+def test_boolean_constant(data, result):
+    boolean = stix2.BooleanConstant(data)
+    assert boolean.value == result
+
+
+def test_invalid_boolean_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.BooleanConstant('foo')
+    assert 'must be a boolean' in str(excinfo)
+
+
+@pytest.mark.parametrize("hashtype, data", [
+    ('MD5', 'zzz'),
+    ('ssdeep', 'zzz=='),
+])
+def test_invalid_hash_constant(hashtype, data):
+    with pytest.raises(ValueError) as excinfo:
+        stix2.HashConstant(data, hashtype)
+    assert 'is not a valid {} hash'.format(hashtype) in str(excinfo)
+
+
+def test_invalid_hex_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.HexConstant('mm')
+    assert "must contain an even number of hexadecimal characters" in str(excinfo)
+
+
+def test_invalid_binary_constant():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.BinaryConstant('foo')
+    assert 'must contain a base64' in str(excinfo)
+
+
+def test_escape_quotes_and_backslashes():
+    exp = stix2.MatchesComparisonExpression("file:name",
+                                            "^Final Report.+\.exe$")
+    assert str(exp) == "file:name MATCHES '^Final Report.+\\\\.exe$'"
+
+
+def test_like():
+    exp = stix2.LikeComparisonExpression("directory:path",
+                                         "C:\Windows\%\\foo")
+    assert str(exp) == "directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo'"
+
+
+def test_issuperset():
+    exp = stix2.IsSupersetComparisonExpression("ipv4-addr:value",
+                                               "198.51.100.0/24")
+    assert str(exp) == "ipv4-addr:value ISSUPERSET '198.51.100.0/24'"
+
+
+def test_repeat_qualifier():
+    qual = stix2.RepeatQualifier(stix2.IntegerConstant(5))
+    assert str(qual) == 'REPEATS 5 TIMES'
+
+
+def test_invalid_repeat_qualifier():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.RepeatQualifier('foo')
+    assert 'is not a valid argument for a Repeat Qualifier' in str(excinfo)
+
+
+def test_invalid_within_qualifier():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.WithinQualifier('foo')
+    assert 'is not a valid argument for a Within Qualifier' in str(excinfo)
+
+
+def test_startstop_qualifier():
+    qual = stix2.StartStopQualifier(stix2.TimestampConstant('2016-06-01T00:00:00Z'),
+                                    datetime.datetime(2017, 3, 12, 8, 30, 0))
+    assert str(qual) == "START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'"
+
+    qual2 = stix2.StartStopQualifier(datetime.date(2016, 6, 1),
+                                     stix2.TimestampConstant('2016-07-01T00:00:00Z'))
+    assert str(qual2) == "START t'2016-06-01T00:00:00Z' STOP t'2016-07-01T00:00:00Z'"
+
+
+def test_invalid_startstop_qualifier():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.StartStopQualifier('foo',
+                                 stix2.TimestampConstant('2016-06-01T00:00:00Z'))
+    assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo)
+
+    with pytest.raises(ValueError) as excinfo:
+        stix2.StartStopQualifier(datetime.date(2016, 6, 1),
+                                 'foo')
+    assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo)

From dd8f0f5c7287f5d79a409cfc045c7c3c97470c85 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 2 Apr 2018 16:44:57 -0400
Subject: [PATCH 070/150] Increase code coverage slightly

An Environment will always have a CompositeDataSource, so there was no way
those exceptions could get raised.
---
 stix2/environment.py          | 10 ++--------
 stix2/test/test_properties.py |  2 ++
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index eb5583e..dcf2449 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -114,16 +114,10 @@ class Environment(DataStoreMixin):
     create.__doc__ = ObjectFactory.create.__doc__
 
     def add_filters(self, *args, **kwargs):
-        try:
-            return self.source.filters.update(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
+        return self.source.filters.update(*args, **kwargs)
 
     def add_filter(self, *args, **kwargs):
-        try:
-            return self.source.filters.add(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
+        return self.source.filters.add(*args, **kwargs)
 
     def parse(self, *args, **kwargs):
         return _parse(*args, **kwargs)
diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py
index 6bd1888..34edc96 100644
--- a/stix2/test/test_properties.py
+++ b/stix2/test/test_properties.py
@@ -16,6 +16,8 @@ def test_property():
     p = Property()
 
     assert p.required is False
+    assert p.clean('foo') == 'foo'
+    assert p.clean(3) == 3
 
 
 def test_basic_clean():

From d5bcc902cc8980024eb6d3215bfb7e073a427b9c Mon Sep 17 00:00:00 2001
From: Robin Cover 
Date: Tue, 3 Apr 2018 11:20:43 -0500
Subject: [PATCH 071/150] fixup program name w/ 'TC' to OASIS TC Open Repo...

---
 CONTRIBUTING.md |  14 ++---
 README.rst      | 150 +++++++++++++++++++++++++++++++-----------------
 2 files changed, 105 insertions(+), 59 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4238e94..5b1f699 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,25 +1,25 @@
 

 
Public Participation Invited

 
-
This OASIS Open Repository ( github.com/oasis-open/cti-python-stix2 ) is a community public repository that supports participation by anyone, whether affiliated with OASIS or not.  Substantive contributions (repository "code") and related feedback is invited from all parties, following the common conventions for participation in GitHub public repository projects.  Participation is expected to be consistent with the OASIS Open Repository Guidelines and Procedures, the LICENSE designated for this particular repository (BSD-3-Clause License), and the requirement for an Individual Contributor License Agreement.  Please see the repository README document for other details.

+
This OASIS TC Open Repository ( github.com/oasis-open/cti-python-stix2 ) is a community public repository that supports participation by anyone, whether affiliated with OASIS or not.  Substantive contributions (repository "code") and related feedback is invited from all parties, following the common conventions for participation in GitHub public repository projects.  Participation is expected to be consistent with the OASIS TC Open Repository Guidelines and Procedures, the LICENSE designated for this particular repository (BSD-3-Clause License), and the requirement for an Individual Contributor License Agreement.  Please see the repository README document for other details.

 

 
 

 
Governance Distinct from OASIS TC Process

-
Content accepted as "contributions" to this Open Repository, as defined below, are distinct from any Contributions made to the associated OASIS Cyber Threat Intelligence (CTI) TC itself.  Participation in the associated Technical Committee is governed by the OASIS Bylaws, OASIS TC Process, IPR Policy, and related policies.  This Open Repository is not subject to the OASIS TC-related policies.  Open Repository governance is defined by separate participation and contribution guidelines as referenced in the OASIS Open Repositories Overview.

+
Content accepted as "contributions" to this TC Open Repository, as defined below, are distinct from any Contributions made to the associated OASIS Cyber Threat Intelligence (CTI) TC itself.  Participation in the associated Technical Committee is governed by the OASIS Bylaws, OASIS TC Process, IPR Policy, and related policies.  This TC Open Repository is not subject to the OASIS TC-related policies.  TC Open Repository governance is defined by separate participation and contribution guidelines as referenced in the OASIS TC Open Repositories Overview.

 

 
 

 
Licensing Distinct from OASIS IPR Policy

-
Because different licenses apply to the OASIS TC's specification work, and this Open Repository, there is no guarantee that the licensure of specific repository material will be compatible with licensing requirements of an implementation of a TC's specification.  Please refer to the LICENSE file for the terms of this material, and to the OASIS IPR Policy for the terms applicable to the TC's specifications, including any applicable declarations.

+
Because different licenses apply to the OASIS TC's specification work, and this TC Open Repository, there is no guarantee that the licensure of specific repository material will be compatible with licensing requirements of an implementation of a TC's specification.  Please refer to the LICENSE file for the terms of this material, and to the OASIS IPR Policy for the terms applicable to the TC's specifications, including any applicable declarations.

 

 
 

 
Contributions Subject to Individual CLA

 
-
Formally, "contribution" to this Open Repository refers to content merged into the "Code" repository (repository changes represented by code commits), following the GitHub definition of contributor: "someone who has contributed to a project by having a pull request merged but does not have collaborator [i.e., direct write] access."  Anyone who signs the Open Repository Individual Contributor License Agreement (CLA), signifying agreement with the licensing requirement, may contribute substantive content — subject to evaluation of a GitHub pull request.  The main web page for this repository, as with any GitHub public repository, displays a link to a document listing contributions to the repository's default branch (filtered by Commits, Additions, and Deletions).

+
Formally, "contribution" to this TC Open Repository refers to content merged into the "Code" repository (repository changes represented by code commits), following the GitHub definition of contributor: "someone who has contributed to a project by having a pull request merged but does not have collaborator [i.e., direct write] access."  Anyone who signs the TC Open Repository Individual Contributor License Agreement (CLA), signifying agreement with the licensing requirement, may contribute substantive content — subject to evaluation of a GitHub pull request.  The main web page for this repository, as with any GitHub public repository, displays a link to a document listing contributions to the repository's default branch (filtered by Commits, Additions, and Deletions).

 
-
This Open Repository, as with GitHub public repositories generally, also accepts public feedback from any GitHub user.  Public feedback includes opening issues, authoring and editing comments, participating in conversations, making wiki edits, creating repository stars, and making suggestions via pull requests.  Such feedback does not constitute an OASIS Open Repository contribution.   Some details are presented under "Read permissions" in the table of permission levels for a GitHub organization.  Technical content intended as a substantive contribution (repository "Code") to an Open Repository is subject to evaluation, and requires a signed Individual CLA.

+
This TC Open Repository, as with GitHub public repositories generally, also accepts public feedback from any GitHub user.  Public feedback includes opening issues, authoring and editing comments, participating in conversations, making wiki edits, creating repository stars, and making suggestions via pull requests.  Such feedback does not constitute an OASIS TC Open Repository contribution.   Some details are presented under "Read permissions" in the table of permission levels for a GitHub organization.  Technical content intended as a substantive contribution (repository "Code") to an TC Open Repository is subject to evaluation, and requires a signed Individual CLA.

 
 
 

@@ -27,12 +27,12 @@
 

 
Fork-and-Pull Collaboration Model

 
-
OASIS Open Repositories use the familiar fork-and-pull collaboration model supported by GitHub and other distributed version-control systems.  Any GitHub user wishing to contribute should fork the repository, make additions or other modifications, and then submit a pull request.  GitHub pull requests should be accompanied by supporting comments and/or issues. Community conversations about pull requests, supported by GitHub notifications, will provide the basis for a consensus determination to merge, modify, close, or take other action, as communicated by the repository Maintainers.

+
OASIS TC Open Repositories use the familiar fork-and-pull collaboration model supported by GitHub and other distributed version-control systems.  Any GitHub user wishing to contribute should fork the repository, make additions or other modifications, and then submit a pull request.  GitHub pull requests should be accompanied by supporting comments and/or issues. Community conversations about pull requests, supported by GitHub notifications, will provide the basis for a consensus determination to merge, modify, close, or take other action, as communicated by the repository Maintainers.

 

 
 

 
Feedback

 
-
Questions or comments about this Open Repository's activities should be composed as GitHub issues or comments. If use of an issue/comment is not possible or appropriate, questions may be directed by email to the repository Maintainer(s).  Please send general questions about Open Repository participation to OASIS Staff at repository-admin@oasis-open.org and any specific CLA-related questions to repository-cla@oasis-open.org.

+
Questions or comments about this TC Open Repository's activities should be composed as GitHub issues or comments. If use of an issue/comment is not possible or appropriate, questions may be directed by email to the repository Maintainer(s).  Please send general questions about TC Open Repository participation to OASIS Staff at repository-admin@oasis-open.org and any specific CLA-related questions to repository-cla@oasis-open.org.

 
 

diff --git a/README.rst b/README.rst
index faacc53..ffd7b40 100644
--- a/README.rst
+++ b/README.rst
@@ -3,11 +3,13 @@
 cti-python-stix2
 ================
 
-This is an `OASIS Open
-Repository `__.
+This is an `OASIS TC Open
+Repository `__.
 See the `Governance <#governance>`__ section for more information.
 
-This repository provides Python APIs for serializing and de-serializing
+This repository provides Python APIs for serializing and de-
+serializing
 STIX 2 JSON content, along with higher-level APIs for common tasks,
 including data markings, versioning, and for resolving STIX IDs across
 multiple data sources.
@@ -29,8 +31,10 @@ Usage
 -----
 
 To create a STIX object, provide keyword arguments to the type's
-constructor. Certain required attributes of all objects, such as ``type`` or
-``id``,  will be set automatically if not provided as keyword arguments.
+constructor. Certain required attributes of all objects, such as
+``type`` or
+``id``,  will be set automatically if not provided as keyword
+arguments.
 
 .. code:: python
 
@@ -38,9 +42,11 @@ constructor. Certain required attributes of all objects, such as ``type`` or
 
     indicator = Indicator(name="File hash for malware variant",
                           labels=["malicious-activity"],
-                          pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']")
+                          pattern="[file:hashes.md5 =
+                          'd41d8cd98f00b204e9800998ecf8427e']")
 
-To parse a STIX JSON string into a Python STIX object, use ``parse()``:
+To parse a STIX JSON string into a Python STIX object, use
+``parse()``:
 
 .. code:: python
 
@@ -55,21 +61,28 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``:
             "malicious-activity"
         ],
         "name": "File hash for malware variant",
-        "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",
+        "pattern": "[file:hashes.md5 =
+        'd41d8cd98f00b204e9800998ecf8427e']",
         "valid_from": "2017-09-26T23:33:39.829952Z"
     }""")
     print(indicator)
 
-For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__.
+For more in-depth documentation, please see
+`https://stix2.readthedocs.io/ `__.
 
 STIX 2.X Technical Specification Support
 ----------------------------------------
 
-This version of python-stix2 supports STIX 2.0 by default. Although, the
-`stix2` Python library is built to support multiple versions of the STIX
-Technical Specification. With every major release of stix2 the ``import stix2``
-statement will automatically load the SDO/SROs equivalent to the most recent
-supported 2.X Technical Specification. Please see the library documentation
+This version of python-stix2 supports STIX 2.0 by default. Although,
+the
+`stix2` Python library is built to support multiple versions of the
+STIX
+Technical Specification. With every major release of stix2 the
+``import stix2``
+statement will automatically load the SDO/SROs equivalent to the most
+recent
+supported 2.X Technical Specification. Please see the library
+documentation
 for more details.
 
 Governance
@@ -77,66 +90,87 @@ Governance
 
 This GitHub public repository (
 **https://github.com/oasis-open/cti-python-stix2** ) was
-`proposed `__
+`proposed `__
 and
-`approved `__
+`approved `__
 [`bis `__] by the
 `OASIS Cyber Threat Intelligence (CTI)
-TC `__ as an `OASIS Open
-Repository `__
+TC `__ as an `OASIS TC
+Open
+Repository `__
 to support development of open source resources related to Technical
 Committee work.
 
-While this Open Repository remains associated with the sponsor TC, its
+While this TC Open Repository remains associated with the sponsor TC,
+its
 development priorities, leadership, intellectual property terms,
 participation rules, and other matters of governance are `separate and
-distinct `__
+distinct `__
 from the OASIS TC Process and related policies.
 
-All contributions made to this Open Repository are subject to open
+All contributions made to this TC Open Repository are subject to open
 source license terms expressed in the `BSD-3-Clause
-License `__.
+License `__.
 That license was selected as the declared `"Applicable
-License" `__
-when the Open Repository was created.
+License" `__
+when the TC Open Repository was created.
 
 As documented in `"Public Participation
-Invited `__",
-contributions to this OASIS Open Repository are invited from all
-parties, whether affiliated with OASIS or not. Participants must have a
+Invited `__",
+contributions to this OASIS TC Open Repository are invited from all
+parties, whether affiliated with OASIS or not. Participants must have
+a
 GitHub account, but no fees or OASIS membership obligations are
 required. Participation is expected to be consistent with the `OASIS
-Open Repository Guidelines and
-Procedures `__,
+TC Open Repository Guidelines and
+Procedures `__,
 the open source
-`LICENSE `__
+`LICENSE `__
 designated for this particular repository, and the requirement for an
 `Individual Contributor License
-Agreement `__
+Agreement `__
 that governs intellectual property.
 
 Maintainers
 ~~~~~~~~~~~
 
-Open Repository
-`Maintainers `__
+TC Open Repository
+`Maintainers `__
 are responsible for oversight of this project's community development
 activities, including evaluation of GitHub `pull
-requests `__
+requests `__
 and
-`preserving `__
+`preserving `__
 open source principles of openness and fairness. Maintainers are
 recognized and trusted experts who serve to implement community goals
 and consensus design preferences.
 
-Initially, the associated TC members have designated one or more persons
-to serve as Maintainer(s); subsequently, participating community members
+Initially, the associated TC members have designated one or more
+persons
+to serve as Maintainer(s); subsequently, participating community
+members
 may select additional or substitute Maintainers, per `consensus
-agreements `__.
+agreements `__.
 
 .. _currentMaintainers:
 
-**Current Maintainers of this Open Repository**
+**Current Maintainers of this TC Open Repository**
 
 -  `Greg Back `__; GitHub ID:
    https://github.com/gtback/; WWW: `MITRE
@@ -145,34 +179,46 @@ agreements `__
 
-About OASIS Open Repositories
+About OASIS TC Open Repositories
 -----------------------------
 
--  `Open Repositories: Overview and
-   Resources `__
+-  `TC Open Repositories: Overview and
+   Resources `__
 -  `Frequently Asked
-   Questions `__
+   Questions `__
 -  `Open Source
-   Licenses `__
+   Licenses `__
 -  `Contributor License Agreements
-   (CLAs) `__
+   (CLAs) `__
 -  `Maintainers' Guidelines and
-   Agreement `__
+   Agreement `__
 
 Feedback
 --------
 
-Questions or comments about this Open Repository's activities should be
-composed as GitHub issues or comments. If use of an issue/comment is not
+Questions or comments about this TC Open Repository's activities
+should be
+composed as GitHub issues or comments. If use of an issue/comment is
+not
 possible or appropriate, questions may be directed by email to the
 Maintainer(s) `listed above <#currentmaintainers>`__. Please send
-general questions about Open Repository participation to OASIS Staff at
+general questions about TC Open Repository participation to OASIS
+Staff at
 repository-admin@oasis-open.org and any specific CLA-related questions
 to repository-cla@oasis-open.org.
 
-.. |Build_Status| image:: https://travis-ci.org/oasis-open/cti-python-stix2.svg?branch=master
+.. |Build_Status| image:: https://travis-ci.org/oasis-open/cti-python-
+stix2.svg?branch=master
    :target: https://travis-ci.org/oasis-open/cti-python-stix2
-.. |Coverage| image:: https://codecov.io/gh/oasis-open/cti-python-stix2/branch/master/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/oasis-open/cti-python-
+stix2/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/oasis-open/cti-python-stix2
-.. |Version| image:: https://img.shields.io/pypi/v/stix2.svg?maxAge=3600
+.. |Version| image:: https://img.shields.io/pypi/v/stix2.svg?maxAge=
+3600
    :target: https://pypi.python.org/pypi/stix2/
+

From fe1852d41e090f9e0bf33998b1905d658ffd33fb Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Wed, 4 Apr 2018 05:48:32 -0400
Subject: [PATCH 072/150] Update README.rst

Update broken banners for Build_Status, Coverage and Version
---
 README.rst | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/README.rst b/README.rst
index ffd7b40..6b9f407 100644
--- a/README.rst
+++ b/README.rst
@@ -212,13 +212,9 @@ Staff at
 repository-admin@oasis-open.org and any specific CLA-related questions
 to repository-cla@oasis-open.org.
 
-.. |Build_Status| image:: https://travis-ci.org/oasis-open/cti-python-
-stix2.svg?branch=master
+.. |Build_Status| image:: https://travis-ci.org/oasis-open/cti-python-stix2.svg?branch=master
    :target: https://travis-ci.org/oasis-open/cti-python-stix2
-.. |Coverage| image:: https://codecov.io/gh/oasis-open/cti-python-
-stix2/branch/master/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/oasis-open/cti-python-stix2/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/oasis-open/cti-python-stix2
-.. |Version| image:: https://img.shields.io/pypi/v/stix2.svg?maxAge=
-3600
+.. |Version| image:: https://img.shields.io/pypi/v/stix2.svg?maxAge=3600
    :target: https://pypi.python.org/pypi/stix2/
-

From a1af05a14eb4e5863afb43cd6aef65bf6a7ae9b1 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 4 Apr 2018 09:47:48 -0400
Subject: [PATCH 073/150] Warn against using workbench with the rest of lib

Doing so may cause issues because the workbench overwrites the STIX
Object mapping.
---
 docs/guide/workbench.ipynb | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb
index bc7942a..9cb099a 100644
--- a/docs/guide/workbench.ipynb
+++ b/docs/guide/workbench.ipynb
@@ -53,7 +53,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Using A Workbench"
+    "## Using The Workbench"
    ]
   },
   {
@@ -446,6 +446,19 @@
    "source": [
     "Defaults can also be set for the [created timestamp](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_created), [external references](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_external_refs) and [object marking references](../api/datastore/stix2.workbench.rst#stix2.workbench.set_default_object_marking_refs)."
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "
\n",
+    "\n",
+    "**Warning:**\n",
+    "\n",
+    "The workbench layer replaces STIX Object classes with special versions of them that use \"wrappers\" to provide extra functionality. Because of this, we recommend that you **either use the workbench layer or the rest of the library, but not both**. In other words, don't import from both ``stix2.workbench`` and any other submodules of ``stix2``.\n",
+    "\n",
+    "
"
+   ]
   }
  ],
  "metadata": {

From 9a50f54ec4882438c195302f185997dd6d3b01bf Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 4 Apr 2018 12:12:09 -0400
Subject: [PATCH 074/150] Update README.rst

Fix line break in usage example
---
 README.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 6b9f407..46e8164 100644
--- a/README.rst
+++ b/README.rst
@@ -42,8 +42,7 @@ arguments.
 
     indicator = Indicator(name="File hash for malware variant",
                           labels=["malicious-activity"],
-                          pattern="[file:hashes.md5 =
-                          'd41d8cd98f00b204e9800998ecf8427e']")
+                          pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']")
 
 To parse a STIX JSON string into a Python STIX object, use
 ``parse()``:

From f951b9b09e23f68edf9b06b6115cafd4e12ad4fb Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 4 Apr 2018 13:21:25 -0400
Subject: [PATCH 075/150] Factor out dupl. code for creating list of filters

---
 stix2/datastore/__init__.py | 17 +++--------------
 stix2/datastore/filters.py  | 23 +++++++++++++++++++++++
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 890ac45..16a0a9e 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -16,7 +16,7 @@ import uuid
 
 from six import with_metaclass
 
-from stix2.datastore.filters import Filter
+from stix2.datastore.filters import Filter, _assemble_filters
 from stix2.utils import deduplicate
 
 
@@ -312,13 +312,7 @@ class DataSource(with_metaclass(ABCMeta)):
             list: The STIX objects that matched the query.
 
         """
-        filter_list = [Filter('type', '=', obj_type)]
-        if filters:
-            if isinstance(filters, list):
-                filter_list.extend(filters)
-            else:
-                filter_list.append(filters)
-
+        filter_list = _assemble_filters(filters, [Filter('type', '=', obj_type)])
         return self.query(filter_list)
 
     def creator_of(self, obj):
@@ -421,12 +415,7 @@ class DataSource(with_metaclass(ABCMeta)):
         ids.discard(obj_id)
 
         # Assemble filters
-        filter_list = []
-        if filters:
-            if isinstance(filters, list):
-                filter_list.extend(filters)
-            else:
-                filter_list.append(filters)
+        filter_list = _assemble_filters(filters)
 
         for i in ids:
             results.extend(self.query(filter_list + [Filter('id', '=', i)]))
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 9065b61..a116507 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -44,6 +44,29 @@ def _check_filter_components(prop, op, value):
     return True
 
 
+def _assemble_filters(filter_arg, filters=[]):
+    """Assemble a list of filters.
+
+    This can be used to allow certain functions to work correctly no matter if
+    the user provides a single filter or a list of them.
+
+    Args:
+        filter_arg (Filter or list): The single Filter or list of Filters to be
+            coerced into a list of Filters.
+        filters (list, optional): A list of Filters to be automatically appended.
+
+    Returns:
+        List of Filters.
+
+    """
+    if isinstance(filter_arg, list):
+        filters.extend(filter_arg)
+    else:
+        filters.append(filter_arg)
+
+    return filters
+
+
 class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
     """STIX 2 filters that support the querying functionality of STIX 2
     DataStores and DataSources.

From 589c00064bc0079f490908b595a577d4cf4479d3 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 4 Apr 2018 14:02:39 -0400
Subject: [PATCH 076/150] Remove query_by_type

It's not that much of a shortcut and we can add it back in later if it
makes sense.
---
 stix2/datastore/__init__.py    | 19 +---------
 stix2/environment.py           |  1 -
 stix2/test/test_environment.py |  4 ---
 stix2/workbench.py             | 63 ++++++++++++++++++++--------------
 4 files changed, 38 insertions(+), 49 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 16a0a9e..4e53b17 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -298,23 +298,6 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
-    def query_by_type(self, obj_type, filters=None):
-        """Retrieve all objects of the given STIX object type.
-
-        This helper function is a shortcut that calls query() under the hood.
-
-        Args:
-            obj_type (str): The STIX object type to retrieve.
-            filters (list, optional): A list of additional filters to apply to
-                the query.
-
-        Returns:
-            list: The STIX objects that matched the query.
-
-        """
-        filter_list = _assemble_filters(filters, [Filter('type', '=', obj_type)])
-        return self.query(filter_list)
-
     def creator_of(self, obj):
         """Retrieve the Identity refered to by the object's `created_by_ref`.
 
@@ -376,7 +359,7 @@ class DataSource(with_metaclass(ABCMeta)):
 
         return results
 
-    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=None):
+    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=[]):
         """Retrieve STIX Objects that have a Relationship involving the given
         STIX object.
 
diff --git a/stix2/environment.py b/stix2/environment.py
index 9f55a7e..e40e991 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -114,7 +114,6 @@ class Environment(DataStoreMixin):
     .. automethod:: get
     .. automethod:: all_versions
     .. automethod:: query
-    .. automethod:: query_by_type
     .. automethod:: creator_of
     .. automethod:: relationships
     .. automethod:: related_to
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 8764e25..176d3f0 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -164,10 +164,6 @@ def test_environment_no_datastore():
         env.query(INDICATOR_ID)
     assert 'Environment has no data source' in str(excinfo.value)
 
-    with pytest.raises(AttributeError) as excinfo:
-        env.query_by_type('indicator')
-    assert 'Environment has no data source' in str(excinfo.value)
-
     with pytest.raises(AttributeError) as excinfo:
         env.relationships(INDICATOR_ID)
     assert 'Environment has no data source' in str(excinfo.value)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 0338353..61e99af 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -8,7 +8,6 @@
 .. autofunction:: get
 .. autofunction:: all_versions
 .. autofunction:: query
-.. autofunction:: query_by_type
 .. autofunction:: creator_of
 .. autofunction:: relationships
 .. autofunction:: related_to
@@ -49,6 +48,7 @@ from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # n
                WindowsPEOptionalHeaderType, WindowsPESection,
                WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
                WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
+from .datastore.filters import _assemble_filters
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -61,7 +61,6 @@ set_default_object_marking_refs = _environ.set_default_object_marking_refs
 get = _environ.get
 all_versions = _environ.all_versions
 query = _environ.query
-query_by_type = _environ.query_by_type
 creator_of = _environ.creator_of
 relationships = _environ.relationships
 related_to = _environ.related_to
@@ -149,7 +148,7 @@ _setup_workbench()
 # Functions to get all objects of a specific type
 
 
-def attack_patterns(filters=None):
+def attack_patterns(filters=[]):
     """Retrieve all Attack Pattern objects.
 
     Args:
@@ -157,10 +156,11 @@ def attack_patterns(filters=None):
             the query.
 
     """
-    return query_by_type('attack-pattern', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'attack-pattern')])
+    return query(filter_list)
 
 
-def campaigns(filters=None):
+def campaigns(filters=[]):
     """Retrieve all Campaign objects.
 
     Args:
@@ -168,10 +168,11 @@ def campaigns(filters=None):
             the query.
 
     """
-    return query_by_type('campaign', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'campaign')])
+    return query(filter_list)
 
 
-def courses_of_action(filters=None):
+def courses_of_action(filters=[]):
     """Retrieve all Course of Action objects.
 
     Args:
@@ -179,10 +180,11 @@ def courses_of_action(filters=None):
             the query.
 
     """
-    return query_by_type('course-of-action', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'course-of-action')])
+    return query(filter_list)
 
 
-def identities(filters=None):
+def identities(filters=[]):
     """Retrieve all Identity objects.
 
     Args:
@@ -190,10 +192,11 @@ def identities(filters=None):
             the query.
 
     """
-    return query_by_type('identity', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'identity')])
+    return query(filter_list)
 
 
-def indicators(filters=None):
+def indicators(filters=[]):
     """Retrieve all Indicator objects.
 
     Args:
@@ -201,10 +204,11 @@ def indicators(filters=None):
             the query.
 
     """
-    return query_by_type('indicator', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'indicator')])
+    return query(filter_list)
 
 
-def intrusion_sets(filters=None):
+def intrusion_sets(filters=[]):
     """Retrieve all Intrusion Set objects.
 
     Args:
@@ -212,10 +216,11 @@ def intrusion_sets(filters=None):
             the query.
 
     """
-    return query_by_type('intrusion-set', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'intrusion-set')])
+    return query(filter_list)
 
 
-def malware(filters=None):
+def malware(filters=[]):
     """Retrieve all Malware objects.
 
     Args:
@@ -223,10 +228,11 @@ def malware(filters=None):
             the query.
 
     """
-    return query_by_type('malware', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'malware')])
+    return query(filter_list)
 
 
-def observed_data(filters=None):
+def observed_data(filters=[]):
     """Retrieve all Observed Data objects.
 
     Args:
@@ -234,10 +240,11 @@ def observed_data(filters=None):
             the query.
 
     """
-    return query_by_type('observed-data', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'observed-data')])
+    return query(filter_list)
 
 
-def reports(filters=None):
+def reports(filters=[]):
     """Retrieve all Report objects.
 
     Args:
@@ -245,10 +252,11 @@ def reports(filters=None):
             the query.
 
     """
-    return query_by_type('report', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'report')])
+    return query(filter_list)
 
 
-def threat_actors(filters=None):
+def threat_actors(filters=[]):
     """Retrieve all Threat Actor objects.
 
     Args:
@@ -256,10 +264,11 @@ def threat_actors(filters=None):
             the query.
 
     """
-    return query_by_type('threat-actor', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'threat-actor')])
+    return query(filter_list)
 
 
-def tools(filters=None):
+def tools(filters=[]):
     """Retrieve all Tool objects.
 
     Args:
@@ -267,10 +276,11 @@ def tools(filters=None):
             the query.
 
     """
-    return query_by_type('tool', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'tool')])
+    return query(filter_list)
 
 
-def vulnerabilities(filters=None):
+def vulnerabilities(filters=[]):
     """Retrieve all Vulnerability objects.
 
     Args:
@@ -278,4 +288,5 @@ def vulnerabilities(filters=None):
             the query.
 
     """
-    return query_by_type('vulnerability', filters)
+    filter_list = _assemble_filters(filters, [Filter('type', '=', 'vulnerability')])
+    return query(filter_list)

From e3bbc39353fb4e536035f0973dbd8c2c9b12bd24 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 5 Apr 2018 10:07:35 -0400
Subject: [PATCH 077/150] Fix bug with mutable default parameter

---
 stix2/datastore/__init__.py  |  2 +-
 stix2/datastore/filters.py   | 24 ++++++++++++++++--------
 stix2/test/test_datastore.py | 11 ++++++++++-
 stix2/workbench.py           | 24 ++++++++++++------------
 4 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 4e53b17..e0de6fe 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -359,7 +359,7 @@ class DataSource(with_metaclass(ABCMeta)):
 
         return results
 
-    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=[]):
+    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=None):
         """Retrieve STIX Objects that have a Relationship involving the given
         STIX object.
 
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index a116507..10bbeee 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -44,27 +44,35 @@ def _check_filter_components(prop, op, value):
     return True
 
 
-def _assemble_filters(filter_arg, filters=[]):
+def _assemble_filters(filters1=None, filters2=None):
     """Assemble a list of filters.
 
     This can be used to allow certain functions to work correctly no matter if
     the user provides a single filter or a list of them.
 
     Args:
-        filter_arg (Filter or list): The single Filter or list of Filters to be
-            coerced into a list of Filters.
-        filters (list, optional): A list of Filters to be automatically appended.
+        filters1 (Filter or list, optional): The single Filter or list of Filters to
+            coerce into a list of Filters.
+        filters2 (Filter or list, optional): The single Filter or list of Filters to
+            append to the list of Filters.
 
     Returns:
         List of Filters.
 
     """
-    if isinstance(filter_arg, list):
-        filters.extend(filter_arg)
+    if filters1 is None:
+        filter_list = []
+    elif not isinstance(filters1, list):
+        filter_list = [filters1]
     else:
-        filters.append(filter_arg)
+        filter_list = filters1
 
-    return filters
+    if isinstance(filters2, list):
+        filter_list.extend(filters2)
+    elif filters2 is not None:
+        filter_list.append(filters2)
+
+    return filter_list
 
 
 class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index e80e8d8..8f40401 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -4,7 +4,7 @@ from taxii2client import Collection
 from stix2 import Filter, MemorySink, MemorySource
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
                              make_id, taxii)
-from stix2.datastore.filters import apply_common_filters
+from stix2.datastore.filters import _assemble_filters, apply_common_filters
 from stix2.utils import deduplicate
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
@@ -473,6 +473,15 @@ def test_filters7():
     assert len(resp) == 1
 
 
+def test_assemble_filters():
+    filter1 = Filter("name", "=", "Malicious site hosting downloader")
+    filter2 = Filter("modified", ">", "2017-01-28T13:49:53.935Z")
+    result = _assemble_filters(filter1, filter2)
+    assert len(result) == 2
+    assert result[0].property == 'name'
+    assert result[1].property == 'modified'
+
+
 def test_deduplicate():
     unique = deduplicate(STIX_OBJS1)
 
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 61e99af..9e31b50 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -148,7 +148,7 @@ _setup_workbench()
 # Functions to get all objects of a specific type
 
 
-def attack_patterns(filters=[]):
+def attack_patterns(filters=None):
     """Retrieve all Attack Pattern objects.
 
     Args:
@@ -160,7 +160,7 @@ def attack_patterns(filters=[]):
     return query(filter_list)
 
 
-def campaigns(filters=[]):
+def campaigns(filters=None):
     """Retrieve all Campaign objects.
 
     Args:
@@ -172,7 +172,7 @@ def campaigns(filters=[]):
     return query(filter_list)
 
 
-def courses_of_action(filters=[]):
+def courses_of_action(filters=None):
     """Retrieve all Course of Action objects.
 
     Args:
@@ -184,7 +184,7 @@ def courses_of_action(filters=[]):
     return query(filter_list)
 
 
-def identities(filters=[]):
+def identities(filters=None):
     """Retrieve all Identity objects.
 
     Args:
@@ -196,7 +196,7 @@ def identities(filters=[]):
     return query(filter_list)
 
 
-def indicators(filters=[]):
+def indicators(filters=None):
     """Retrieve all Indicator objects.
 
     Args:
@@ -208,7 +208,7 @@ def indicators(filters=[]):
     return query(filter_list)
 
 
-def intrusion_sets(filters=[]):
+def intrusion_sets(filters=None):
     """Retrieve all Intrusion Set objects.
 
     Args:
@@ -220,7 +220,7 @@ def intrusion_sets(filters=[]):
     return query(filter_list)
 
 
-def malware(filters=[]):
+def malware(filters=None):
     """Retrieve all Malware objects.
 
     Args:
@@ -232,7 +232,7 @@ def malware(filters=[]):
     return query(filter_list)
 
 
-def observed_data(filters=[]):
+def observed_data(filters=None):
     """Retrieve all Observed Data objects.
 
     Args:
@@ -244,7 +244,7 @@ def observed_data(filters=[]):
     return query(filter_list)
 
 
-def reports(filters=[]):
+def reports(filters=None):
     """Retrieve all Report objects.
 
     Args:
@@ -256,7 +256,7 @@ def reports(filters=[]):
     return query(filter_list)
 
 
-def threat_actors(filters=[]):
+def threat_actors(filters=None):
     """Retrieve all Threat Actor objects.
 
     Args:
@@ -268,7 +268,7 @@ def threat_actors(filters=[]):
     return query(filter_list)
 
 
-def tools(filters=[]):
+def tools(filters=None):
     """Retrieve all Tool objects.
 
     Args:
@@ -280,7 +280,7 @@ def tools(filters=[]):
     return query(filter_list)
 
 
-def vulnerabilities(filters=[]):
+def vulnerabilities(filters=None):
     """Retrieve all Vulnerability objects.
 
     Args:

From e6e72856b3e9b7adf93ce07a1d4218b59456044d Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 5 Apr 2018 11:22:50 -0400
Subject: [PATCH 078/150] Remove query_by_type()

Missed these earlier.
---
 stix2/datastore/__init__.py | 48 -------------------------------------
 1 file changed, 48 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index e0de6fe..442a242 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -99,25 +99,6 @@ class DataStoreMixin(object):
         except AttributeError:
             raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
-    def query_by_type(self, *args, **kwargs):
-        """Retrieve all objects of the given STIX object type.
-
-        Translate query_by_type() call to the appropriate DataSource call.
-
-        Args:
-            obj_type (str): The STIX object type to retrieve.
-            filters (list, optional): A list of additional filters to apply to
-                the query.
-
-        Returns:
-            list: The STIX objects that matched the query.
-
-        """
-        try:
-            return self.source.query_by_type(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
-
     def creator_of(self, *args, **kwargs):
         """Retrieve the Identity refered to by the object's `created_by_ref`.
 
@@ -568,35 +549,6 @@ class CompositeDataSource(DataSource):
 
         return all_data
 
-    def query_by_type(self, *args, **kwargs):
-        """Retrieve all objects of the given STIX object type.
-
-        Federate the query to all DataSources attached to the
-        Composite Data Source.
-
-        Args:
-            obj_type (str): The STIX object type to retrieve.
-            filters (list, optional): A list of additional filters to apply to
-                the query.
-
-        Returns:
-            list: The STIX objects that matched the query.
-
-        """
-        if not self.has_data_sources():
-            raise AttributeError('CompositeDataSource has no data sources')
-
-        results = []
-        for ds in self.data_sources:
-            results.extend(ds.query_by_type(*args, **kwargs))
-
-        # remove exact duplicates (where duplicates are STIX 2.0
-        # objects with the same 'id' and 'modified' values)
-        if len(results) > 0:
-            results = deduplicate(results)
-
-        return results
-
     def relationships(self, *args, **kwargs):
         """Retrieve Relationships involving the given STIX object.
 

From 5791810906fce96e82280e297cf384e911c83491 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 5 Apr 2018 16:44:44 -0400
Subject: [PATCH 079/150] Ensure all guide doc output cells have [Out] label

Fixes #105.
---
 docs/guide/creating.ipynb    |  92 ++--
 docs/guide/custom.ipynb      | 658 +++++++++++++++++++++----
 docs/guide/datastore.ipynb   | 111 ++---
 docs/guide/environment.ipynb | 370 +++++++-------
 docs/guide/filesystem.ipynb  | 926 ++++++++++++++++++++++++-----------
 docs/guide/markings.ipynb    | 414 ++++++++--------
 docs/guide/memory.ipynb      | 188 ++-----
 docs/guide/parsing.ipynb     | 303 ++++++++++--
 docs/guide/serializing.ipynb |  38 +-
 docs/guide/taxii.ipynb       |  20 +-
 docs/guide/ts_support.ipynb  |  37 +-
 docs/guide/versioning.ipynb  |  72 +--
 docs/guide/workbench.ipynb   | 376 ++++++++++++--
 13 files changed, 2422 insertions(+), 1183 deletions(-)

diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb
index 3e22061..d9464e7 100644
--- a/docs/guide/creating.ipynb
+++ b/docs/guide/creating.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -25,7 +24,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -146,15 +144,15 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
-       "    "created": "2017-09-26T23:33:39.829Z",\n",
-       "    "modified": "2017-09-26T23:33:39.829Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "id": "indicator--548af3be-39d7-4a3e-93c2-1a63cccf8951",\n",
+       "    "created": "2018-04-05T18:32:24.193Z",\n",
+       "    "modified": "2018-04-05T18:32:24.193Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-09-26T23:33:39.829952Z"\n",
+       "    "valid_from": "2018-04-05T18:32:24.193659Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -188,9 +186,7 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "indicator2 = Indicator(type='indicator',\n",
@@ -295,7 +291,7 @@
     {
      "data": {
       "text/plain": [
-       "u'File hash for malware variant'"
+       "'File hash for malware variant'"
       ]
      },
      "execution_count": 8,
@@ -322,7 +318,7 @@
     {
      "data": {
       "text/plain": [
-       "u'File hash for malware variant'"
+       "'File hash for malware variant'"
       ]
      },
      "execution_count": 9,
@@ -469,9 +465,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--d7fd675d-94eb-4d95-b0bc-b3c5e28e8ed2",\n",
-       "    "created": "2017-09-26T23:33:56.908Z",\n",
-       "    "modified": "2017-09-26T23:33:56.908Z",\n",
+       "    "id": "malware--3d7f0c1c-616a-4868-aa7b-150821d2a429",\n",
+       "    "created": "2018-04-05T18:32:46.584Z",\n",
+       "    "modified": "2018-04-05T18:32:46.584Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "labels": [\n",
        "        "remote-access-trojan"\n",
@@ -592,12 +588,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "relationship",\n",
-       "    "id": "relationship--637aa3b1-d4b8-4bc4-85e7-77cc82b198a3",\n",
-       "    "created": "2017-09-26T23:34:01.765Z",\n",
-       "    "modified": "2017-09-26T23:34:01.765Z",\n",
+       "    "id": "relationship--34ddc7b4-4965-4615-b286-1c8bbaa1e7db",\n",
+       "    "created": "2018-04-05T18:32:49.474Z",\n",
+       "    "modified": "2018-04-05T18:32:49.474Z",\n",
        "    "relationship_type": "indicates",\n",
-       "    "source_ref": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
-       "    "target_ref": "malware--d7fd675d-94eb-4d95-b0bc-b3c5e28e8ed2"\n",
+       "    "source_ref": "indicator--548af3be-39d7-4a3e-93c2-1a63cccf8951",\n",
+       "    "target_ref": "malware--3d7f0c1c-616a-4868-aa7b-150821d2a429"\n",
        "}\n",
        "
\n"
       ],
@@ -704,12 +700,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "relationship",\n",
-       "    "id": "relationship--70fe77c2-ab00-4181-a2dc-fe5567d971ca",\n",
-       "    "created": "2017-09-26T23:34:03.923Z",\n",
-       "    "modified": "2017-09-26T23:34:03.923Z",\n",
+       "    "id": "relationship--0a646403-f7e7-4cfd-b945-cab5cde05857",\n",
+       "    "created": "2018-04-05T18:32:51.417Z",\n",
+       "    "modified": "2018-04-05T18:32:51.417Z",\n",
        "    "relationship_type": "indicates",\n",
-       "    "source_ref": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
-       "    "target_ref": "malware--d7fd675d-94eb-4d95-b0bc-b3c5e28e8ed2"\n",
+       "    "source_ref": "indicator--548af3be-39d7-4a3e-93c2-1a63cccf8951",\n",
+       "    "target_ref": "malware--3d7f0c1c-616a-4868-aa7b-150821d2a429"\n",
        "}\n",
        "
\n"
       ],
@@ -814,26 +810,26 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "bundle",\n",
-       "    "id": "bundle--2536c43d-c874-418e-886c-20a22120d8cb",\n",
+       "    "id": "bundle--f83477e5-f853-47e1-a267-43f3aa1bd5b0",\n",
        "    "spec_version": "2.0",\n",
        "    "objects": [\n",
        "        {\n",
        "            "type": "indicator",\n",
-       "            "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
-       "            "created": "2017-09-26T23:33:39.829Z",\n",
-       "            "modified": "2017-09-26T23:33:39.829Z",\n",
-       "            "labels": [\n",
-       "                "malicious-activity"\n",
-       "            ],\n",
+       "            "id": "indicator--548af3be-39d7-4a3e-93c2-1a63cccf8951",\n",
+       "            "created": "2018-04-05T18:32:24.193Z",\n",
+       "            "modified": "2018-04-05T18:32:24.193Z",\n",
        "            "name": "File hash for malware variant",\n",
        "            "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "            "valid_from": "2017-09-26T23:33:39.829952Z"\n",
+       "            "valid_from": "2018-04-05T18:32:24.193659Z",\n",
+       "            "labels": [\n",
+       "                "malicious-activity"\n",
+       "            ]\n",
        "        },\n",
        "        {\n",
        "            "type": "malware",\n",
-       "            "id": "malware--d7fd675d-94eb-4d95-b0bc-b3c5e28e8ed2",\n",
-       "            "created": "2017-09-26T23:33:56.908Z",\n",
-       "            "modified": "2017-09-26T23:33:56.908Z",\n",
+       "            "id": "malware--3d7f0c1c-616a-4868-aa7b-150821d2a429",\n",
+       "            "created": "2018-04-05T18:32:46.584Z",\n",
+       "            "modified": "2018-04-05T18:32:46.584Z",\n",
        "            "name": "Poison Ivy",\n",
        "            "labels": [\n",
        "                "remote-access-trojan"\n",
@@ -841,12 +837,12 @@
        "        },\n",
        "        {\n",
        "            "type": "relationship",\n",
-       "            "id": "relationship--637aa3b1-d4b8-4bc4-85e7-77cc82b198a3",\n",
-       "            "created": "2017-09-26T23:34:01.765Z",\n",
-       "            "modified": "2017-09-26T23:34:01.765Z",\n",
+       "            "id": "relationship--34ddc7b4-4965-4615-b286-1c8bbaa1e7db",\n",
+       "            "created": "2018-04-05T18:32:49.474Z",\n",
+       "            "modified": "2018-04-05T18:32:49.474Z",\n",
        "            "relationship_type": "indicates",\n",
-       "            "source_ref": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
-       "            "target_ref": "malware--d7fd675d-94eb-4d95-b0bc-b3c5e28e8ed2"\n",
+       "            "source_ref": "indicator--548af3be-39d7-4a3e-93c2-1a63cccf8951",\n",
+       "            "target_ref": "malware--3d7f0c1c-616a-4868-aa7b-150821d2a429"\n",
        "        }\n",
        "    ]\n",
        "}\n",
@@ -871,21 +867,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb
index 7f30401..bc19749 100644
--- a/docs/guide/custom.ipynb
+++ b/docs/guide/custom.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -23,9 +22,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -70,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -99,7 +99,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -175,9 +175,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--00c5743f-2d5e-4d66-88f1-1842584f4519",\n",
-       "    "created": "2017-11-09T16:17:44.596Z",\n",
-       "    "modified": "2017-11-09T16:17:44.596Z",\n",
+       "    "id": "identity--87aac643-341b-413a-b702-ea5820416155",\n",
+       "    "created": "2018-04-05T18:38:10.269Z",\n",
+       "    "modified": "2018-04-05T18:38:10.269Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -188,7 +188,7 @@
        ""
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -211,6 +211,117 @@
     "Alternatively, setting ``allow_custom`` to ``True`` will allow custom properties without requiring a ``custom_properties`` dictionary."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "identity",\n",
+       "    "id": "identity--a1ad0a6f-39ab-4642-9a72-aaa198b1eee2",\n",
+       "    "created": "2018-04-05T18:38:12.270Z",\n",
+       "    "modified": "2018-04-05T18:38:12.270Z",\n",
+       "    "name": "John Smith",\n",
+       "    "identity_class": "individual",\n",
+       "    "x_foo": "bar"\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "identity2 = Identity(name=\"John Smith\",\n",
+    "                     identity_class=\"individual\",\n",
+    "                     x_foo=\"bar\",\n",
+    "                     allow_custom=True)\n",
+    "print(identity2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.core.rst#stix2.core.parse):"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 6,
@@ -287,15 +398,7 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
-       "    "x_foo": "bar",\n",
-       "    "type": "identity",\n",
-       "    "id": "identity--1e8188eb-245f-400b-839d-7f612169c514",\n",
-       "    "created": "2017-09-26T21:02:22.708Z",\n",
-       "    "modified": "2017-09-26T21:02:22.708Z",\n",
-       "    "name": "John Smith",\n",
-       "    "identity_class": "individual"\n",
-       "}\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
bar\n",
        "
\n"
       ],
       "text/plain": [
@@ -307,34 +410,6 @@
      "output_type": "execute_result"
     }
    ],
-   "source": [
-    "identity2 = Identity(name=\"John Smith\",\n",
-    "                     identity_class=\"individual\",\n",
-    "                     x_foo=\"bar\",\n",
-    "                     allow_custom=True)\n",
-    "print(identity2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.core.rst#stix2.core.parse):"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "bar\n"
-     ]
-    }
-   ],
    "source": [
     "from stix2 import parse\n",
     "\n",
@@ -364,10 +439,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 7,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import CustomObject, properties\n",
@@ -391,7 +464,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -467,9 +540,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-animal",\n",
-       "    "id": "x-animal--caebdf17-9d2a-4c84-8864-7406326618f0",\n",
-       "    "created": "2017-09-26T21:02:34.724Z",\n",
-       "    "modified": "2017-09-26T21:02:34.724Z",\n",
+       "    "id": "x-animal--b1e4fe7f-7985-451d-855c-6ba5c265b22a",\n",
+       "    "created": "2018-04-05T18:38:19.790Z",\n",
+       "    "modified": "2018-04-05T18:38:19.790Z",\n",
        "    "species": "lion",\n",
        "    "animal_class": "mammal"\n",
        "}\n",
@@ -479,7 +552,7 @@
        ""
       ]
      },
-     "execution_count": 9,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -499,7 +572,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -525,15 +598,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "shark\n"
-     ]
+     "data": {
+      "text/html": [
+       "
shark\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -558,7 +706,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -593,7 +741,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -678,7 +826,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -709,16 +857,172 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "foobaz\n",
-      "5\n"
-     ]
+     "data": {
+      "text/html": [
+       "
foobaz\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
5\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -759,7 +1063,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -843,7 +1147,7 @@
        ""
       ]
      },
-     "execution_count": 15,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -851,10 +1155,10 @@
    "source": [
     "from stix2 import File, CustomExtension\n",
     "\n",
-    "@CustomExtension(File, 'x-new-ext', [\n",
-    "    ('property1', properties.StringProperty(required=True)),\n",
-    "    ('property2', properties.IntegerProperty()),\n",
-    "])\n",
+    "@CustomExtension(File, 'x-new-ext', {\n",
+    "    'property1': properties.StringProperty(required=True),\n",
+    "    'property2': properties.IntegerProperty(),\n",
+    "})\n",
     "class NewExtension():\n",
     "    pass\n",
     "\n",
@@ -872,16 +1176,172 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "bla\n",
-      "50\n"
-     ]
+     "data": {
+      "text/html": [
+       "
bla\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
50\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -918,21 +1378,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index ac27a82..474c719 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -23,9 +22,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,36 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "collapsed": true
-   },
-   "outputs": [],
-   "source": [
-    "# without this configuration, only last print() call is outputted in cells\n",
-    "from IPython.core.interactiveshell import InteractiveShell\n",
-    "InteractiveShell.ast_node_interactivity = \"all\""
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -328,10 +315,10 @@
     "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n",
     "\n",
     "# create FileSystemStore\n",
-    "fs = FileSystemSource(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/\")\n",
+    "fs = FileSystemSource(\"/tmp/stix2_source\")\n",
     "\n",
     "# create TAXIICollectionSource\n",
-    "colxn = Collection('https://test.freetaxii.com:8000/osint/collections/a9c22eaf-0f3e-482c-8bb4-45ae09e75d9b/')\n",
+    "colxn = Collection('http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/')\n",
     "ts = TAXIICollectionSource(colxn)\n",
     "\n",
     "# add them both to the CompositeDataSource\n",
@@ -344,7 +331,7 @@
     "\n",
     "# get an object that is only in the TAXII collection\n",
     "ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n",
-    "print(ind)\n"
+    "print(ind)"
    ]
   },
   {
@@ -389,10 +376,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 7,
+   "metadata": {},
    "outputs": [],
    "source": [
     "import sys\n",
@@ -423,16 +408,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 9,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import MemoryStore, FileSystemStore, FileSystemSource\n",
     "\n",
-    "fs = FileSystemStore(\"/home/michael/Desktop/sample_stix2_data\")\n",
-    "fs_source = FileSystemSource(\"/home/michael/Desktop/sample_stix2_data\")\n",
+    "fs = FileSystemStore(\"/tmp/stix2_store\")\n",
+    "fs_source = FileSystemSource(\"/tmp/stix2_source\")\n",
     "\n",
     "# attach filter to FileSystemStore\n",
     "fs.source.filters.add(f)\n",
@@ -466,7 +449,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -492,7 +475,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -568,9 +551,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--be3baac0-9aba-48a8-81e4-4408b1c379a8",\n",
-       "    "created": "2017-11-21T22:14:45.213Z",\n",
-       "    "modified": "2017-11-21T22:14:45.213Z",\n",
+       "    "id": "identity--b67cf8d4-cc1a-4bb7-9402-fffcff17c9a9",\n",
+       "    "created": "2018-04-05T20:43:54.117Z",\n",
+       "    "modified": "2018-04-05T20:43:54.117Z",\n",
        "    "name": "John Doe",\n",
        "    "identity_class": "individual"\n",
        "}\n",
@@ -580,7 +563,7 @@
        ""
       ]
      },
-     "execution_count": 14,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -598,7 +581,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -607,7 +590,7 @@
        "3"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -626,16 +609,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]"
+       "[Relationship(type='relationship', id='relationship--3b9cb248-5c2c-425d-85d0-680bfef6e69d', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='indicates', source_ref='indicator--61deb2a5-305a-490e-83b3-9839a9677368', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d')]"
       ]
      },
-     "execution_count": 27,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -653,16 +636,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--7eb7f5cd-8bf2-4f7c-8756-84c0b5693b9a', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'targets', source_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4', target_ref='identity--be3baac0-9aba-48a8-81e4-4408b1c379a8')]"
+       "[Relationship(type='relationship', id='relationship--8d322508-423b-4d51-be85-a95ad083f8af', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='targets', source_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d', target_ref='identity--b67cf8d4-cc1a-4bb7-9402-fffcff17c9a9')]"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -680,17 +663,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4'),\n",
-       " Relationship(type='relationship', id='relationship--3c759d40-c92a-430e-aab6-77d5c5763302', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'uses', source_ref='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]"
+       "[Relationship(type='relationship', id='relationship--3b9cb248-5c2c-425d-85d0-680bfef6e69d', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='indicates', source_ref='indicator--61deb2a5-305a-490e-83b3-9839a9677368', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d'),\n",
+       " Relationship(type='relationship', id='relationship--93e5afe0-d1fb-4315-8d08-10951f7a99b6', created='2018-04-05T20:43:54.134Z', modified='2018-04-05T20:43:54.134Z', relationship_type='uses', source_ref='campaign--edfd885c-bc31-4051-9bc2-08e057542d56', target_ref='malware--9fe343d8-edf7-4f4a-bb6c-a221fb75142d')]"
       ]
      },
-     "execution_count": 30,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -708,16 +691,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[Campaign(type='campaign', id='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', created='2017-11-21T22:14:45.213Z', modified='2017-11-21T22:14:45.213Z', name=u'Charge', description=u'Attack!')]"
+       "[Campaign(type='campaign', id='campaign--edfd885c-bc31-4051-9bc2-08e057542d56', created='2018-04-05T20:43:54.117Z', modified='2018-04-05T20:43:54.117Z', name='Charge', description='Attack!')]"
       ]
      },
-     "execution_count": 42,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -729,21 +712,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "cti-python-stix2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "cti-python-stix2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb
index 3ece7c4..fdc57c8 100644
--- a/docs/guide/environment.ipynb
+++ b/docs/guide/environment.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -25,7 +24,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -68,9 +68,7 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import Environment, MemoryStore\n",
@@ -87,18 +85,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 6,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import CompositeDataSource, FileSystemSink, FileSystemSource, MemorySource\n",
     "\n",
     "src = CompositeDataSource()\n",
-    "src.add_data_sources([MemorySource(), FileSystemSource(\"/tmp/stix_source\")])\n",
+    "src.add_data_sources([MemorySource(), FileSystemSource(\"/tmp/stix2_source\")])\n",
     "env2 = Environment(source=src,\n",
-    "                   sink=FileSystemSink(\"/tmp/stix_sink\"))"
+    "                   sink=FileSystemSink(\"/tmp/stix2_sink\"))"
    ]
   },
   {
@@ -110,10 +106,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 7,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from stix2 import Indicator\n",
@@ -131,139 +125,6 @@
     "You can retrieve STIX objects from the [DataSources](../api/stix2.datastore.rst#stix2.datastore.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), [creator_of()](../api/stix2.datastore.rst#stix2.datastore.DataSource.creator_of), [related_to()](../api/stix2.datastore.rst#stix2.datastore.DataSource.related_to), and [relationships()](../api/stix2.datastore.rst#stix2.datastore.DataSource.relationships) just as you would for a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource)."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--01234567-89ab-cdef-0123-456789abcdef",\n",
-       "    "created": "2017-10-02T13:20:39.373Z",\n",
-       "    "modified": "2017-10-02T13:20:39.373Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-02T13:20:39.3737Z"\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "print(env.get(\"indicator--01234567-89ab-cdef-0123-456789abcdef\"))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Creating STIX Objects With Defaults\n",
-    "\n",
-    "To create STIX objects with default values for certain properties, use an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory). For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "collapsed": true
-   },
-   "outputs": [],
-   "source": [
-    "from stix2 import Indicator, ObjectFactory\n",
-    "\n",
-    "factory = ObjectFactory(created_by_ref=\"identity--311b2d2d-f010-5473-83ec-1edf84858f4c\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "collapsed": true
-   },
-   "source": [
-    "Once you've set up the [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory), use its [create()](../api/stix2.environment.rst#stix2.environment.ObjectFactory.create) method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 8,
@@ -342,15 +203,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--c92ad60d-449d-4adf-86b3-4e5951a8f480",\n",
-       "    "created_by_ref": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",\n",
-       "    "created": "2017-10-02T13:23:00.607Z",\n",
-       "    "modified": "2017-10-02T13:23:00.607Z",\n",
+       "    "id": "indicator--01234567-89ab-cdef-0123-456789abcdef",\n",
+       "    "created": "2018-04-05T19:27:53.923Z",\n",
+       "    "modified": "2018-04-05T19:27:53.923Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:27:53.923548Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-02T13:23:00.607216Z"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -363,6 +223,138 @@
      "output_type": "execute_result"
     }
    ],
+   "source": [
+    "print(env.get(\"indicator--01234567-89ab-cdef-0123-456789abcdef\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Creating STIX Objects With Defaults\n",
+    "\n",
+    "To create STIX objects with default values for certain properties, use an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory). For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from stix2 import Indicator, ObjectFactory\n",
+    "\n",
+    "factory = ObjectFactory(created_by_ref=\"identity--311b2d2d-f010-5473-83ec-1edf84858f4c\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true
+   },
+   "source": [
+    "Once you've set up the [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory), use its [create()](../api/stix2.environment.rst#stix2.environment.ObjectFactory.create) method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--c1b421c0-9c6b-4276-9b73-1b8684a5a0d2",\n",
+       "    "created_by_ref": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",\n",
+       "    "created": "2018-04-05T19:28:48.776Z",\n",
+       "    "modified": "2018-04-05T19:28:48.776Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:28:48.776442Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "ind = factory.create(Indicator,\n",
     "                     labels=[\"malicious-activity\"],\n",
@@ -388,7 +380,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -464,14 +456,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--ae206b9f-8723-4fcf-beb7-8b1b9a2570ab",\n",
+       "    "id": "indicator--30a3b39c-5f57-4e7f-9eaf-e1abcb643da4",\n",
        "    "created": "2017-09-25T18:07:46.255Z",\n",
        "    "modified": "2017-09-25T18:07:46.255Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:28:53.268567Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-02T13:23:05.790562Z"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -479,7 +471,7 @@
        ""
       ]
      },
-     "execution_count": 9,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -498,7 +490,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -574,15 +566,15 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--a8e2be68-b496-463f-9ff4-f620046e7cf2",\n",
+       "    "id": "indicator--6c5bbaaf-6dac-44b0-a0df-86c27b3f6ecb",\n",
        "    "created_by_ref": "identity--962cabe5-f7f3-438a-9169-585a8c971d12",\n",
        "    "created": "2017-09-25T18:07:46.255Z",\n",
        "    "modified": "2017-09-25T18:07:46.255Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:29:56.55129Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-02T13:23:08.32424Z"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -590,7 +582,7 @@
        ""
       ]
      },
-     "execution_count": 10,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -614,7 +606,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -690,15 +682,15 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--89ba04ea-cce9-47a3-acd3-b6379ce51581",\n",
+       "    "id": "indicator--d1b8c3f6-1de1-44c1-b079-3df307224a0d",\n",
        "    "created_by_ref": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",\n",
-       "    "created": "2017-10-02T13:23:29.629Z",\n",
-       "    "modified": "2017-10-02T13:23:29.629Z",\n",
+       "    "created": "2018-04-05T19:29:59.605Z",\n",
+       "    "modified": "2018-04-05T19:29:59.605Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:29:59.605463Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-02T13:23:29.629857Z"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -706,7 +698,7 @@
        ""
       ]
      },
-     "execution_count": 11,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -725,21 +717,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb
index cff73d6..28a1843 100644
--- a/docs/guide/filesystem.ipynb
+++ b/docs/guide/filesystem.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -25,7 +24,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -120,57 +120,129 @@
     "### FileSystem Examples\n",
     "\n",
     "#### FileSystemStore\n",
-    "        "
+    "\n",
+    "Use the FileSystemStore when you want to both retrieve STIX content from the file system and push STIX content to it, too."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"modified\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"name\": \"PowerDuke\",\n",
-      "    \"description\": \"PowerDuke is a backdoor that was used by APT29 in 2016. It has primarily been delivered through Microsoft Word or Excel attachments containing malicious macros.[[Citation: Volexity PowerDuke November 2016]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0139\",\n",
-      "            \"external_id\": \"S0139\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Volexity PowerDuke November 2016\",\n",
-      "            \"description\": \"Adair, S.. (2016, November 9). PowerDuke: Widespread Post-Election Spear Phishing Campaigns Targeting Think Tanks and NGOs. Retrieved January 11, 2017.\",\n",
-      "            \"url\": \"https://www.volexity.com/blog/2016/11/09/powerduke-post-election-spear-phishing-campaigns-targeting-think-tanks-and-ngos/\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n"
-     ]
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "malware",\n",
+       "    "id": "malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:33:19.746Z",\n",
+       "    "modified": "2017-05-31T21:33:19.746Z",\n",
+       "    "name": "PowerDuke",\n",
+       "    "description": "PowerDuke is a backdoor that was used by APT29 in 2016. It has primarily been delivered through Microsoft Word or Excel attachments containing malicious macros.[[Citation: Volexity PowerDuke November 2016]]",\n",
+       "    "labels": [\n",
+       "        "malware"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "mitre-attack",\n",
+       "            "url": "https://attack.mitre.org/wiki/Software/S0139",\n",
+       "            "external_id": "S0139"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Volexity PowerDuke November 2016",\n",
+       "            "description": "Adair, S.. (2016, November 9). PowerDuke: Widespread Post-Election Spear Phishing Campaigns Targeting Think Tanks and NGOs. Retrieved January 11, 2017.",\n",
+       "            "url": "https://www.volexity.com/blog/2016/11/09/powerduke-post-election-spear-phishing-campaigns-targeting-think-tanks-and-ngos/"\n",
+       "        }\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "from stix2 import FileSystemStore\n",
     "\n",
-    "\"\"\"\n",
-    "Working with the FileSystemStore, where STIX content can be retrieved and pushed to a file system.\n",
-    "\"\"\"\n",
-    "\n",
     "# create FileSystemStore\n",
-    "fs = FileSystemStore(\"/home/michael/Desktop/sample_stix2_data\")\n",
+    "fs = FileSystemStore(\"/tmp/stix2_store\")\n",
     "\n",
     "# retrieve STIX2 content from FileSystemStore\n",
     "ap = fs.get(\"attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6\")\n",
@@ -221,54 +293,128 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "#### FileSystemSource - (if STIX content is only to be retrieved from FileSystem)"
+    "#### FileSystemSource\n",
+    "\n",
+    "Use the FileSystemSource when you only want to retrieve STIX content from the file system."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{\n",
-      "    \"type\": \"attack-pattern\",\n",
-      "    \"id\": \"attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:30:54.176Z\",\n",
-      "    \"modified\": \"2017-05-31T21:30:54.176Z\",\n",
-      "    \"name\": \"Indicator Removal from Tools\",\n",
-      "    \"description\": \"If a malicious...command-line parameters, Process monitoring\",\n",
-      "    \"kill_chain_phases\": [\n",
-      "        {\n",
-      "            \"kill_chain_name\": \"mitre-attack\",\n",
-      "            \"phase_name\": \"defense-evasion\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Technique/T1066\",\n",
-      "            \"external_id\": \"T1066\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n"
-     ]
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "attack-pattern",\n",
+       "    "id": "attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:30:54.176Z",\n",
+       "    "modified": "2017-05-31T21:30:54.176Z",\n",
+       "    "name": "Indicator Removal from Tools",\n",
+       "    "description": "If a malicious...command-line parameters, Process monitoring",\n",
+       "    "kill_chain_phases": [\n",
+       "        {\n",
+       "            "kill_chain_name": "mitre-attack",\n",
+       "            "phase_name": "defense-evasion"\n",
+       "        }\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "mitre-attack",\n",
+       "            "url": "https://attack.mitre.org/wiki/Technique/T1066",\n",
+       "            "external_id": "T1066"\n",
+       "        }\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "from stix2 import FileSystemSource\n",
-    "\"\"\"\n",
-    "Working with FileSystemSource for retrieving STIX content.\n",
-    "\"\"\"\n",
     "\n",
     "# create FileSystemSource\n",
-    "fs_source = FileSystemSource(\"/home/michael/Desktop/sample_stix2_data\")\n",
+    "fs_source = FileSystemSource(\"/tmp/stix2_source\")\n",
     "\n",
     "# retrieve STIX 2 objects\n",
     "ap = fs_source.get(\"attack-pattern--00d0b012-8a03-410e-95de-5826bf542de6\")\n",
@@ -279,149 +425,336 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--0f862b01-99da-47cc-9bdb-db4a86a95bb1\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:32:54.772Z\",\n",
-      "    \"modified\": \"2017-05-31T21:32:54.772Z\",\n",
-      "    \"name\": \"Emissary\",\n",
-      "    \"description\": \"Emissary is a Trojan that has been used by Lotus Blossom. It shares code with Elise, with both Trojans being part of a malware group referred to as LStudio.[[Citation: Lotus Blossom Dec 2015]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0082\",\n",
-      "            \"external_id\": \"S0082\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Lotus Blossom Dec 2015\",\n",
-      "            \"description\": \"Falcone, R. and Miller-Osborn, J.. (2015, December 18). Attack on French Diplomat Linked to Operation Lotus Blossom. Retrieved February 15, 2016.\",\n",
-      "            \"url\": \"http://researchcenter.paloaltonetworks.com/2015/12/attack-on-french-diplomat-linked-to-operation-lotus-blossom/\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n",
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--2a6f4c7b-e690-4cc7-ab6b-1f821fb6b80b\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:32:33.348Z\",\n",
-      "    \"modified\": \"2017-05-31T21:32:33.348Z\",\n",
-      "    \"name\": \"LOWBALL\",\n",
-      "    \"description\": \"LOWBALL is malware used by admin@338. It was used in August 2015 in email messages targeting Hong Kong-based media organizations.[[Citation: FireEye admin@338]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0042\",\n",
-      "            \"external_id\": \"S0042\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"FireEye admin@338\",\n",
-      "            \"description\": \"FireEye Threat Intelligence. (2015, December 1). China-based Cyber Threat Group Uses Dropbox for Malware Communications and Targets Hong Kong Media Outlets. Retrieved December 4, 2015.\",\n",
-      "            \"url\": \"https://www.fireeye.com/blog/threat-research/2015/11/china-based-threat.html\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n",
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"modified\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"name\": \"PowerDuke\",\n",
-      "    \"description\": \"PowerDuke is a backdoor that was used by APT29 in 2016. It has primarily been delivered through Microsoft Word or Excel attachments containing malicious macros.[[Citation: Volexity PowerDuke November 2016]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0139\",\n",
-      "            \"external_id\": \"S0139\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Volexity PowerDuke November 2016\",\n",
-      "            \"description\": \"Adair, S.. (2016, November 9). PowerDuke: Widespread Post-Election Spear Phishing Campaigns Targeting Think Tanks and NGOs. Retrieved January 11, 2017.\",\n",
-      "            \"url\": \"https://www.volexity.com/blog/2016/11/09/powerduke-post-election-spear-phishing-campaigns-targeting-think-tanks-and-ngos/\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n",
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--0db09158-6e48-4e7c-8ce7-2b10b9c0c039\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:32:55.126Z\",\n",
-      "    \"modified\": \"2017-05-31T21:32:55.126Z\",\n",
-      "    \"name\": \"Misdat\",\n",
-      "    \"description\": \"Misdat is a backdoor that was used by Dust Storm from 2010 to 2011.[[Citation: Cylance Dust Storm]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0083\",\n",
-      "            \"external_id\": \"S0083\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Cylance Dust Storm\",\n",
-      "            \"description\": \"Gross, J. (2016, February 23). Operation Dust Storm. Retrieved February 25, 2016.\",\n",
-      "            \"url\": \"https://www.cylance.com/hubfs/2015%20cylance%20website/assets/operation-dust-storm/Op%20Dust%20Storm%20Report.pdf?t=1456259131512\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n",
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--1d808f62-cf63-4063-9727-ff6132514c22\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:33:06.433Z\",\n",
-      "    \"modified\": \"2017-05-31T21:33:06.433Z\",\n",
-      "    \"name\": \"WEBC2\",\n",
-      "    \"description\": \"WEBC2 is a backdoor used by APT1 to retrieve a Web page from a predetermined C2 server.[[Citation: Mandiant APT1 Appendix]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0109\",\n",
-      "            \"external_id\": \"S0109\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Mandiant APT1 Appendix\",\n",
-      "            \"description\": \"Mandiant. (n.d.). Appendix C (Digital) - The Malware Arsenal. Retrieved July 18, 2016.\",\n",
-      "            \"url\": \"https://www.fireeye.com/content/dam/fireeye-www/services/pdfs/mandiant-apt1-report-appendix.zip\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n"
-     ]
+     "data": {
+      "text/html": [
+       "
malware--96b08451-b27a-4ff6-893f-790e26393a8e\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
malware--b42378e0-f147-496f-992a-26a49705395b\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
malware--6b616fc1-1505-48e3-8b2c-0d19337bff38\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -434,46 +767,95 @@
     "mals = fs_source.query(query)\n",
     "\n",
     "for mal in mals:\n",
-    "    print(mal)"
+    "    print(mal.id)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{\n",
-      "    \"type\": \"malware\",\n",
-      "    \"id\": \"malware--00c3bfcb-99bd-4767-8c03-b08f585f5c8a\",\n",
-      "    \"created_by_ref\": \"identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5\",\n",
-      "    \"created\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"modified\": \"2017-05-31T21:33:19.746Z\",\n",
-      "    \"name\": \"PowerDuke\",\n",
-      "    \"description\": \"PowerDuke is a backdoor that was used by APT29 in 2016. It has primarily been delivered through Microsoft Word or Excel attachments containing malicious macros.[[Citation: Volexity PowerDuke November 2016]]\",\n",
-      "    \"labels\": [\n",
-      "        \"malware\"\n",
-      "    ],\n",
-      "    \"external_references\": [\n",
-      "        {\n",
-      "            \"source_name\": \"mitre-attack\",\n",
-      "            \"url\": \"https://attack.mitre.org/wiki/Software/S0139\",\n",
-      "            \"external_id\": \"S0139\"\n",
-      "        },\n",
-      "        {\n",
-      "            \"source_name\": \"Volexity PowerDuke November 2016\",\n",
-      "            \"description\": \"Adair, S.. (2016, November 9). PowerDuke: Widespread Post-Election Spear Phishing Campaigns Targeting Think Tanks and NGOs. Retrieved January 11, 2017.\",\n",
-      "            \"url\": \"https://www.volexity.com/blog/2016/11/09/powerduke-post-election-spear-phishing-campaigns-targeting-think-tanks-and-ngos/\"\n",
-      "        }\n",
-      "    ],\n",
-      "    \"object_marking_refs\": [\n",
-      "        \"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168\"\n",
-      "    ]\n",
-      "}\n"
-     ]
+     "data": {
+      "text/html": [
+       "
malware--92ec0cbd-2c30-44a2-b270-73f4ec949841\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -484,30 +866,28 @@
     "\n",
     "# for visual purposes\n",
     "for mal in mals:\n",
-    "    print(mal)"
+    "    print(mal.id)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "#### FileSystemSink - (if STIX content is only to be pushed to FileSystem)"
+    "#### FileSystemSink\n",
+    "\n",
+    "Use the FileSystemSink when you only want to push STIX content to the file system."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 10,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "from stix2 import FileSystemSink, Campaign\n",
-    "\"\"\"\n",
-    "Working with FileSystemSink for pushing STIX content.\n",
-    "\"\"\"\n",
+    "from stix2 import FileSystemSink, Campaign, Indicator\n",
+    "\n",
     "# create FileSystemSink\n",
-    "fs_sink = FileSystemSink(\"/home/michael/Desktop/sample_stix2_data\")\n",
+    "fs_sink = FileSystemSink(\"/tmp/stix2_sink\")\n",
     "\n",
     "# create STIX objects and add to sink\n",
     "camp = Campaign(name=\"The Crusades\",\n",
@@ -532,21 +912,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb
index fcacf47..8230daf 100644
--- a/docs/guide/markings.ipynb
+++ b/docs/guide/markings.ipynb
@@ -2,9 +2,8 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 6,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -23,9 +22,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 5,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -68,127 +68,127 @@
     "To create an object with a (predefined) TLP marking to an object, just provide it as a keyword argument to the constructor. The TLP markings can easily be imported from python-stix2."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "indicator",\n",
-       "    "id": "indicator--65ff0082-bb92-4812-9b74-b144b858297f",\n",
-       "    "created": "2017-11-13T14:42:14.641Z",\n",
-       "    "modified": "2017-11-13T14:42:14.641Z",\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-11-13T14:42:14.641818Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
-       "    "object_marking_refs": [\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
-       "    ]\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from stix2 import Indicator, TLP_AMBER\n",
-    "\n",
-    "indicator = Indicator(labels=[\"malicious-activity\"],\n",
-    "                      pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n",
-    "                      object_marking_refs=TLP_AMBER)\n",
-    "print(indicator)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "If you’re creating your own marking (for example, a ``Statement`` marking), first create the statement marking:"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 7,
    "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
+       "    "created": "2018-04-05T19:49:47.924Z",\n",
+       "    "modified": "2018-04-05T19:49:47.924Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
+       "    ]\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from stix2 import Indicator, TLP_AMBER\n",
+    "\n",
+    "indicator = Indicator(labels=[\"malicious-activity\"],\n",
+    "                      pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n",
+    "                      object_marking_refs=TLP_AMBER)\n",
+    "print(indicator)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you’re creating your own marking (for example, a ``Statement`` marking), first create the statement marking:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -263,8 +263,8 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "marking-definition",\n",
-       "    "id": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
-       "    "created": "2017-11-13T14:43:30.558058Z",\n",
+       "    "id": "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
+       "    "created": "2018-04-05T19:49:53.98008Z",\n",
        "    "definition_type": "statement",\n",
        "    "definition": {\n",
        "        "statement": "Copyright 2017, Example Corp"\n",
@@ -276,7 +276,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -300,7 +300,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -376,16 +376,16 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--526cda4e-6745-4cd6-852f-0750c6a79784",\n",
-       "    "created": "2017-10-04T14:43:09.586Z",\n",
-       "    "modified": "2017-10-04T14:43:09.586Z",\n",
+       "    "id": "indicator--7caeab49-2472-41bb-a988-2f990aea99bd",\n",
+       "    "created": "2018-04-05T19:49:55.763Z",\n",
+       "    "modified": "2018-04-05T19:49:55.763Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:55.763364Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:43:09.586133Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53"\n",
+       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -394,7 +394,7 @@
        ""
       ]
      },
-     "execution_count": 5,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -408,7 +408,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -484,14 +484,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--1505b789-fcd2-48ee-bea9-3b20627a4abd",\n",
-       "    "created": "2017-10-04T14:43:20.049Z",\n",
-       "    "modified": "2017-10-04T14:43:20.049Z",\n",
+       "    "id": "indicator--4eb21bbe-b8a9-4348-86cf-1ed52f9abdd7",\n",
+       "    "created": "2018-04-05T19:49:57.248Z",\n",
+       "    "modified": "2018-04-05T19:49:57.248Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:57.248658Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:43:20.049166Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -502,7 +502,7 @@
        ""
       ]
      },
-     "execution_count": 6,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -523,7 +523,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -599,9 +599,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--f7128008-f6ab-4d43-a8a2-a681651268f8",\n",
-       "    "created": "2017-11-13T14:43:34.857Z",\n",
-       "    "modified": "2017-11-13T14:43:34.857Z",\n",
+       "    "id": "malware--ef1eddbb-b5a5-47e0-b607-75b9870d8d91",\n",
+       "    "created": "2018-04-05T19:49:59.103Z",\n",
+       "    "modified": "2018-04-05T19:49:59.103Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "description": "A ransomware related to ...",\n",
        "    "labels": [\n",
@@ -609,7 +609,7 @@
        "    ],\n",
        "    "granular_markings": [\n",
        "        {\n",
-       "            "marking_ref": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
+       "            "marking_ref": "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
        "            "selectors": [\n",
        "                "description"\n",
        "            ]\n",
@@ -628,7 +628,7 @@
        ""
       ]
      },
-     "execution_count": 8,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -661,7 +661,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -705,7 +705,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -781,17 +781,17 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T15:03:46.599Z",\n",
+       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
+       "    "created": "2018-04-05T19:49:47.924Z",\n",
+       "    "modified": "2018-04-05T19:50:03.387Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82",\n",
-       "        "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53"\n",
+       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
+       "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -800,7 +800,7 @@
        ""
       ]
      },
-     "execution_count": 21,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -819,7 +819,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -895,14 +895,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T15:03:54.290Z",\n",
+       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
+       "    "created": "2018-04-05T19:49:47.924Z",\n",
+       "    "modified": "2018-04-05T19:50:05.109Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -913,7 +913,7 @@
        ""
       ]
      },
-     "execution_count": 22,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -932,7 +932,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -1008,17 +1008,17 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T15:04:04.218Z",\n",
+       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
+       "    "created": "2018-04-05T19:49:47.924Z",\n",
+       "    "modified": "2018-04-05T19:50:06.773Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z",\n",
        "    "object_marking_refs": [\n",
-       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da",\n",
-       "        "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53"\n",
+       "        "marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368",\n",
+       "        "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -1027,7 +1027,7 @@
        ""
       ]
      },
-     "execution_count": 23,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1048,7 +1048,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -1124,14 +1124,14 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T14:54:39.331Z",\n",
+       "    "id": "indicator--95a71cff-fad0-4ffb-a641-8a6eaa642290",\n",
+       "    "created": "2018-04-05T19:49:47.924Z",\n",
+       "    "modified": "2018-04-05T19:50:08.616Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2018-04-05T19:49:47.924708Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
-       "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -1139,7 +1139,7 @@
        ""
       ]
      },
-     "execution_count": 12,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1167,17 +1167,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da',\n",
-       " 'marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53']"
+       "['marking-definition--13680b12-3d19-4b42-abe6-0d31effe5368',\n",
+       " 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']"
       ]
      },
-     "execution_count": 19,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1195,7 +1195,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -1204,7 +1204,7 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1224,7 +1224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
@@ -1233,7 +1233,7 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 19,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1251,7 +1251,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -1260,7 +1260,7 @@
        "True"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1271,7 +1271,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
@@ -1280,7 +1280,7 @@
        "True"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1291,7 +1291,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 22,
    "metadata": {
     "scrolled": true
    },
@@ -1302,7 +1302,7 @@
        "False"
       ]
      },
-     "execution_count": 18,
+     "execution_count": 22,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1314,21 +1314,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb
index 79a8f33..6b6d5cb 100644
--- a/docs/guide/memory.ipynb
+++ b/docs/guide/memory.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -25,7 +24,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -151,12 +151,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--2f61e4e7-0891-4e09-b79a-66f5e594fec0",\n",
-       "    "created": "2017-11-17T17:01:31.590Z",\n",
-       "    "modified": "2017-11-17T17:01:31.590Z",\n",
+       "    "id": "indicator--41a960c7-a6d4-406d-9156-0069cb3bd40d",\n",
+       "    "created": "2018-04-05T19:50:41.222Z",\n",
+       "    "modified": "2018-04-05T19:50:41.222Z",\n",
        "    "description": "Crusades C2 implant",\n",
        "    "pattern": "[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']",\n",
-       "    "valid_from": "2017-11-17T17:01:31.590939Z",\n",
+       "    "valid_from": "2018-04-05T19:50:41.222522Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ]\n",
@@ -267,12 +267,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--ddb765ba-ff1e-4285-bf33-1f6d08f583d6",\n",
-       "    "created": "2017-11-17T17:01:31.799Z",\n",
-       "    "modified": "2017-11-17T17:01:31.799Z",\n",
+       "    "id": "indicator--ba2a7acb-a3ac-420b-9288-09988aa99408",\n",
+       "    "created": "2018-04-05T19:50:43.343Z",\n",
+       "    "modified": "2018-04-05T19:50:43.343Z",\n",
        "    "description": "Crusades stage 2 implant variant",\n",
        "    "pattern": "[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']",\n",
-       "    "valid_from": "2017-11-17T17:01:31.799228Z",\n",
+       "    "valid_from": "2018-04-05T19:50:43.343298Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ]\n",
@@ -386,9 +386,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--e8170e70-522f-4ec3-aa22-afb55bfad0b0",\n",
-       "    "created": "2017-11-17T17:01:31.806Z",\n",
-       "    "modified": "2017-11-17T17:01:31.806Z",\n",
+       "    "id": "malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52",\n",
+       "    "created": "2018-04-05T19:50:43.346Z",\n",
+       "    "modified": "2018-04-05T19:50:43.346Z",\n",
        "    "name": "Alexios",\n",
        "    "labels": [\n",
        "        "rootkit"\n",
@@ -412,120 +412,6 @@
     "print(mal)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "report",\n",
-       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
-       "    "created": "2017-05-08T18:34:08.042Z",\n",
-       "    "modified": "2017-05-08T18:34:08.042Z",\n",
-       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
-       "    "published": "2017-05-08T10:24:11.011Z",\n",
-       "    "object_refs": [\n",
-       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
-       "    ],\n",
-       "    "labels": [\n",
-       "        "threat-report"\n",
-       "    ]\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from stix2 import Filter\n",
-    "\n",
-    "# add json formatted string to MemoryStore\n",
-    "# Again, would NOT manually create json-formatted string\n",
-    "# but taken as an output form from another source\n",
-    "report = '{\"type\": \"report\",\"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\"labels\": [\"threat-report\"], \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\", \"published\": \"2017-05-08T10:24:11.011Z\", \"object_refs\":[\"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"], \"created\": \"2017-05-08T18:34:08.042Z\", \"modified\": \"2017-05-08T18:34:08.042Z\"}'\n",
-    "\n",
-    "mem.add(report)\n",
-    "\n",
-    "print(mem.get(\"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\"))"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -610,17 +496,13 @@
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
-       "    "type": "report",\n",
-       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
-       "    "created": "2017-05-08T18:34:08.042Z",\n",
-       "    "modified": "2017-05-08T18:34:08.042Z",\n",
-       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
-       "    "published": "2017-05-08T10:24:11.011Z",\n",
-       "    "object_refs": [\n",
-       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
-       "    ],\n",
+       "    "type": "malware",\n",
+       "    "id": "malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52",\n",
+       "    "created": "2018-04-05T19:50:43.346Z",\n",
+       "    "modified": "2018-04-05T19:50:43.346Z",\n",
+       "    "name": "Alexios",\n",
        "    "labels": [\n",
-       "        "threat-report"\n",
+       "        "rootkit"\n",
        "    ]\n",
        "}\n",
        "
\n"
@@ -643,30 +525,30 @@
     "# load(add) STIX content from json file into MemoryStore\n",
     "mem_2.load_from_file(\"path_to_target_file.json\")\n",
     "\n",
-    "report = mem_2.get(\"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\")\n",
+    "report = mem_2.get(\"malware--9e9b87ce-2b2b-455a-8d5b-26384ccc8d52\")\n",
     "\n",
-    "# for visualpurposes\n",
+    "# for visual purposes\n",
     "print(report)"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "cti-python-stix2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "cti-python-stix2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb
index b3460b3..284125e 100644
--- a/docs/guide/parsing.ipynb
+++ b/docs/guide/parsing.ipynb
@@ -4,7 +4,6 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -25,7 +24,6 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -70,15 +70,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
+     "data": {
+      "text/html": [
+       "
<class 'stix2.v20.sdo.ObservedData'>\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
     },
     {
      "data": {
@@ -174,7 +249,7 @@
        ""
       ]
      },
-     "execution_count": 10,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -214,15 +289,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
+     "data": {
+      "text/html": [
+       "
<class 'stix2.v20.sdo.Identity'>\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
     },
     {
      "data": {
@@ -309,7 +459,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -338,15 +488,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
+     "data": {
+      "text/html": [
+       "
<class 'stix2.v20.sdo.CourseOfAction'>\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
     },
     {
      "data": {
@@ -434,13 +659,13 @@
        ""
       ]
      },
-     "execution_count": 8,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "file_handle = open(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json\")\n",
+    "file_handle = open(\"/tmp/stix2_store/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json\")\n",
     "\n",
     "obj = parse(file_handle)\n",
     "print(type(obj))\n",
@@ -486,27 +711,27 @@
     "\n",
     "# create TAXIICollectionSource\n",
     "colxn = Collection('http://taxii_url')\n",
-    "ts = TAXIICollectionSource(colxn, allow_custom=True)\n"
+    "ts = TAXIICollectionSource(colxn, allow_custom=True)"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb
index 1046d9f..e58302e 100644
--- a/docs/guide/serializing.ipynb
+++ b/docs/guide/serializing.ipynb
@@ -31,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -142,12 +144,12 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48",\n",
-       "    "created": "2018-03-13T19:49:53.392Z",\n",
-       "    "modified": "2018-03-13T19:49:53.392Z",\n",
+       "    "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3",\n",
+       "    "created": "2018-04-05T20:01:20.012Z",\n",
+       "    "modified": "2018-04-05T20:01:20.012Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2018-03-13T19:49:53.392627Z",\n",
+       "    "valid_from": "2018-04-05T20:01:20.012209Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ]\n",
@@ -256,7 +258,7 @@
        ".highlight .vg { color: #19177C } /* Name.Variable.Global */\n",
        ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n",
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
-       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"name": "File hash for malware variant", "labels": ["malicious-activity"], "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "type": "indicator", "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48", "created": "2018-03-13T19:49:53.392Z", "modified": "2018-03-13T19:49:53.392Z", "valid_from": "2018-03-13T19:49:53.392627Z"}\n",
+       ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{"name": "File hash for malware variant", "labels": ["malicious-activity"], "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "type": "indicator", "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3", "created": "2018-04-05T20:01:20.012Z", "modified": "2018-04-05T20:01:20.012Z", "valid_from": "2018-04-05T20:01:20.012209Z"}\n",
        "
\n"
       ],
       "text/plain": [
@@ -362,10 +364,10 @@
        "    ],\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--85773ceb-c768-45f6-bb04-b4d813809e48",\n",
-       "    "created": "2018-03-13T19:49:53.392Z",\n",
-       "    "modified": "2018-03-13T19:49:53.392Z",\n",
-       "    "valid_from": "2018-03-13T19:49:53.392627Z"\n",
+       "    "id": "indicator--4336ace8-d985-413a-8e32-f749ba268dc3",\n",
+       "    "created": "2018-04-05T20:01:20.012Z",\n",
+       "    "modified": "2018-04-05T20:01:20.012Z",\n",
+       "    "valid_from": "2018-04-05T20:01:20.012209Z"\n",
        "}\n",
        "
\n"
       ],
diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb
index 4045a98..4e81388 100644
--- a/docs/guide/taxii.ipynb
+++ b/docs/guide/taxii.ipynb
@@ -33,23 +33,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb
index 6ceef99..2264eb5 100644
--- a/docs/guide/ts_support.ipynb
+++ b/docs/guide/ts_support.ipynb
@@ -23,9 +23,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +32,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -236,7 +237,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -328,7 +329,7 @@
        ""
       ]
      },
-     "execution_count": 3,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -396,21 +397,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/versioning.ipynb b/docs/guide/versioning.ipynb
index fb3b866..6074d00 100644
--- a/docs/guide/versioning.ipynb
+++ b/docs/guide/versioning.ipynb
@@ -2,9 +2,8 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -23,9 +22,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {
-    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -33,23 +31,25 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from IPython.display import HTML\n",
+    "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
     "\n",
-    "original_print = print\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        return HTML('{}'.format(\n",
-    "                    formatter.get_style_defs('.highlight'),\n",
-    "                    highlight(string, JsonLexer(), formatter)))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        original_print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
-    "print = json_print"
+    "globals()['print'] = json_print"
    ]
   },
   {
@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -144,15 +144,15 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--92bb1ae4-db9c-4d6e-8ded-ef7280b4439a",\n",
+       "    "id": "indicator--dd052ff6-e404-444b-beb9-eae96d1e79ea",\n",
        "    "created": "2016-01-01T08:00:00.000Z",\n",
-       "    "modified": "2017-09-26T23:39:07.149Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "modified": "2018-04-05T20:02:51.161Z",\n",
        "    "name": "File hash for Foobar malware",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-09-26T23:39:07.132129Z"\n",
+       "    "valid_from": "2018-04-05T20:02:51.138312Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -160,7 +160,7 @@
        ""
       ]
      },
-     "execution_count": 3,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -187,7 +187,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {
     "scrolled": true
    },
@@ -216,7 +216,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -292,16 +292,16 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--92bb1ae4-db9c-4d6e-8ded-ef7280b4439a",\n",
+       "    "id": "indicator--dd052ff6-e404-444b-beb9-eae96d1e79ea",\n",
        "    "created": "2016-01-01T08:00:00.000Z",\n",
-       "    "modified": "2017-09-26T23:39:09.463Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "modified": "2018-04-05T20:02:54.704Z",\n",
        "    "name": "File hash for Foobar malware",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-09-26T23:39:07.132129Z",\n",
-       "    "revoked": true\n",
+       "    "valid_from": "2018-04-05T20:02:51.138312Z",\n",
+       "    "revoked": true,\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
        "}\n",
        "
\n"
       ],
@@ -309,7 +309,7 @@
        ""
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -322,21 +322,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.12"
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/workbench.ipynb b/docs/guide/workbench.ipynb
index 9cb099a..6594841 100644
--- a/docs/guide/workbench.ipynb
+++ b/docs/guide/workbench.ipynb
@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 2,
    "metadata": {
     "nbsphinx": "hidden"
    },
@@ -31,20 +31,23 @@
     "# JSON output syntax highlighting\n",
     "from __future__ import print_function\n",
     "from pygments import highlight\n",
-    "from pygments.lexers import JsonLexer\n",
+    "from pygments.lexers import JsonLexer, TextLexer\n",
     "from pygments.formatters import HtmlFormatter\n",
-    "from six.moves import builtins\n",
     "from IPython.display import display, HTML\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
     "\n",
     "def json_print(inpt):\n",
     "    string = str(inpt)\n",
+    "    formatter = HtmlFormatter()\n",
     "    if string[0] == '{':\n",
-    "        formatter = HtmlFormatter()\n",
-    "        display(HTML('{}'.format(\n",
-    "                     formatter.get_style_defs('.highlight'),\n",
-    "                     highlight(string, JsonLexer(), formatter))))\n",
+    "        lexer = JsonLexer()\n",
     "    else:\n",
-    "        builtins.print(inpt)\n",
+    "        lexer = TextLexer()\n",
+    "    return HTML('{}'.format(\n",
+    "                formatter.get_style_defs('.highlight'),\n",
+    "                highlight(string, lexer, formatter)))\n",
     "\n",
     "globals()['print'] = json_print"
    ]
@@ -65,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,7 +86,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 4,
    "metadata": {
     "scrolled": true
    },
@@ -141,17 +144,254 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\n",
-      "indicates\n",
-      "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\n"
-     ]
+     "data": {
+      "text/html": [
+       "
indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
indicates\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -164,7 +404,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -255,8 +495,9 @@
        ""
       ]
      },
+     "execution_count": 8,
      "metadata": {},
-     "output_type": "display_data"
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -274,7 +515,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -366,7 +607,7 @@
        ""
       ]
      },
-     "execution_count": 5,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -388,7 +629,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -405,7 +646,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -421,15 +662,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "ACME Threat Intel Co.\n"
-     ]
+     "data": {
+      "text/html": [
+       "
ACME Threat Intel Co.\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [

From 8f58f29358bdb8686ac8388052bf666f771fe2c9 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:16:04 -0400
Subject: [PATCH 080/150] Moved from test_datastore.py -> new file
 test_datastore_filesystem.py

---
 stix2/test/test_datastore_filesystem.py | 530 ++++++++++++++++++++++++
 1 file changed, 530 insertions(+)
 create mode 100644 stix2/test/test_datastore_filesystem.py

diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py
new file mode 100644
index 0000000..bd1f02c
--- /dev/null
+++ b/stix2/test/test_datastore_filesystem.py
@@ -0,0 +1,530 @@
+import json
+import os
+import shutil
+
+import pytest
+
+from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink,
+                   FileSystemSource, FileSystemStore, Filter, Identity,
+                   Indicator, Malware, Relationship, properties)
+
+from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
+                                  IDENTITY_KWARGS, INDICATOR_ID,
+                                  INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS,
+                                  RELATIONSHIP_IDS)
+
+FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
+
+
+@pytest.fixture
+def fs_store():
+    # create
+    yield FileSystemStore(FS_PATH)
+
+    # remove campaign dir
+    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
+
+
+@pytest.fixture
+def fs_source():
+    # create
+    fs = FileSystemSource(FS_PATH)
+    assert fs.stix_dir == FS_PATH
+    yield fs
+
+    # remove campaign dir
+    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
+
+
+@pytest.fixture
+def fs_sink():
+    # create
+    fs = FileSystemSink(FS_PATH)
+    assert fs.stix_dir == FS_PATH
+    yield fs
+
+    # remove campaign dir
+    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
+
+
+@pytest.fixture
+def bad_json_files():
+    # create erroneous JSON files for tests to make sure handled gracefully
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f:
+        f.write("Im not a JSON file")
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f:
+        f.write("Im not a JSON formatted file")
+
+    yield True  # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"))
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"))
+
+
+@pytest.fixture
+def bad_stix_files():
+    # create erroneous STIX JSON files for tests to make sure handled correctly
+
+    # bad STIX object
+    stix_obj = {
+        "id": "intrusion-set--test-bad-stix",
+        "spec_version": "2.0"
+        # no "type" field
+    }
+
+    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f:
+        f.write(json.dumps(stix_obj))
+
+    yield True  # dummy yield so can have teardown
+
+    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"))
+
+
+@pytest.fixture(scope='module')
+def rel_fs_store():
+    cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
+    mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
+    rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
+    rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
+    stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
+    fs = FileSystemStore(FS_PATH)
+    for o in stix_objs:
+        fs.add(o)
+    yield fs
+
+    for o in stix_objs:
+        os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
+
+
+def test_filesystem_source_nonexistent_folder():
+    with pytest.raises(ValueError) as excinfo:
+        FileSystemSource('nonexistent-folder')
+    assert "for STIX data does not exist" in str(excinfo)
+
+
+def test_filesystem_sink_nonexistent_folder():
+    with pytest.raises(ValueError) as excinfo:
+        FileSystemSink('nonexistent-folder')
+    assert "for STIX data does not exist" in str(excinfo)
+
+
+def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
+    # this tests the handling of two bad json files
+    #  - one file should just be skipped (silently) as its a ".txt" extension
+    #  - one file should be parsed and raise Exception bc its not JSON
+    try:
+        fs_source.get("intrusion-set--test-bad-json")
+    except TypeError as e:
+        assert "intrusion-set--test-bad-json" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
+def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
+    # this tests handling of bad STIX json object
+    try:
+        fs_source.get("intrusion-set--test-non-stix")
+    except TypeError as e:
+        assert "intrusion-set--test-non-stix" in str(e)
+        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+
+
+def test_filesytem_source_get_object(fs_source):
+    # get object
+    mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")
+    assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"
+    assert mal.name == "Rover"
+
+
+def test_filesytem_source_get_nonexistent_object(fs_source):
+    ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38")
+    assert ind is None
+
+
+def test_filesytem_source_all_versions(fs_source):
+    # all versions - (currently not a true all versions call as FileSystem cant have multiple versions)
+    id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5")
+    assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5"
+    assert id_.name == "The MITRE Corporation"
+    assert id_.type == "identity"
+
+
+def test_filesytem_source_query_single(fs_source):
+    # query2
+    is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")])
+    assert len(is_2) == 1
+
+    is_2 = is_2[0]
+    assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a"
+    assert is_2.type == "attack-pattern"
+
+
+def test_filesytem_source_query_multiple(fs_source):
+    # query
+    intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")])
+    assert len(intrusion_sets) == 2
+    assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets]
+    assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets]
+
+    is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0]
+    assert "DragonOK" in is_1.aliases
+    assert len(is_1.external_references) == 4
+
+
+def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source):
+    # add python stix object
+    camp1 = Campaign(name="Hannibal",
+                     objective="Targeting Italian and Spanish Diplomat internet accounts",
+                     aliases=["War Elephant"])
+
+    fs_sink.add(camp1)
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json"))
+
+    camp1_r = fs_source.get(camp1.id)
+    assert camp1_r.id == camp1.id
+    assert camp1_r.name == "Hannibal"
+    assert "War Elephant" in camp1_r.aliases
+
+    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
+
+
+def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source):
+    # add stix object dict
+    camp2 = {
+        "name": "Aurelius",
+        "type": "campaign",
+        "objective": "German and French Intelligence Services",
+        "aliases": ["Purple Robes"],
+        "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a",
+        "created": "2017-05-31T21:31:53.197755Z"
+    }
+
+    fs_sink.add(camp2)
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json"))
+
+    camp2_r = fs_source.get(camp2["id"])
+    assert camp2_r.id == camp2["id"]
+    assert camp2_r.name == camp2["name"]
+    assert "Purple Robes" in camp2_r.aliases
+
+    os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json"))
+
+
+def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source):
+    # add stix bundle dict
+    bund = {
+        "type": "bundle",
+        "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a",
+        "spec_version": "2.0",
+        "objects": [
+            {
+                "name": "Atilla",
+                "type": "campaign",
+                "objective": "Bulgarian, Albanian and Romanian Intelligence Services",
+                "aliases": ["Huns"],
+                "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a",
+                "created": "2017-05-31T21:31:53.197755Z"
+            }
+        ]
+    }
+
+    fs_sink.add(bund)
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json"))
+
+    camp3_r = fs_source.get(bund["objects"][0]["id"])
+    assert camp3_r.id == bund["objects"][0]["id"]
+    assert camp3_r.name == bund["objects"][0]["name"]
+    assert "Huns" in camp3_r.aliases
+
+    os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json"))
+
+
+def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source):
+    # add json-encoded stix obj
+    camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\
+            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}'
+
+    fs_sink.add(camp4)
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json"))
+
+    camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a")
+    assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a"
+    assert camp4_r.name == "Ghengis Khan"
+
+    os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json"))
+
+
+def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source):
+    # add json-encoded stix bundle
+    bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \
+            ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \
+            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}'
+    fs_sink.add(bund2)
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json"))
+
+    camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a")
+    assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a"
+    assert camp5_r.name == "Spartacus"
+
+    os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json"))
+
+
+def test_filesystem_sink_add_objects_list(fs_sink, fs_source):
+    # add list of objects
+    camp6 = Campaign(name="Comanche",
+                     objective="US Midwest manufacturing firms, oil refineries, and businesses",
+                     aliases=["Horse Warrior"])
+
+    camp7 = {
+        "name": "Napolean",
+        "type": "campaign",
+        "objective": "Central and Eastern Europe military commands and departments",
+        "aliases": ["The Frenchmen"],
+        "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a",
+        "created": "2017-05-31T21:31:53.197755Z"
+    }
+
+    fs_sink.add([camp6, camp7])
+
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json"))
+    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json"))
+
+    camp6_r = fs_source.get(camp6.id)
+    assert camp6_r.id == camp6.id
+    assert "Horse Warrior" in camp6_r.aliases
+
+    camp7_r = fs_source.get(camp7["id"])
+    assert camp7_r.id == camp7["id"]
+    assert "The Frenchmen" in camp7_r.aliases
+
+    # remove all added objects
+    os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json"))
+    os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json"))
+
+
+def test_filesystem_store_get_stored_as_bundle(fs_store):
+    coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f")
+    assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f"
+    assert coa.type == "course-of-action"
+
+
+def test_filesystem_store_get_stored_as_object(fs_store):
+    coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd")
+    assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd"
+    assert coa.type == "course-of-action"
+
+
+def test_filesystem_store_all_versions(fs_store):
+    # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored)
+    rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0]
+    assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1"
+    assert rel.type == "relationship"
+
+
+def test_filesystem_store_query(fs_store):
+    # query()
+    tools = fs_store.query([Filter("labels", "in", "tool")])
+    assert len(tools) == 2
+    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools]
+    assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools]
+
+
+def test_filesystem_store_query_single_filter(fs_store):
+    query = Filter("labels", "in", "tool")
+    tools = fs_store.query(query)
+    assert len(tools) == 2
+    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools]
+    assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools]
+
+
+def test_filesystem_store_empty_query(fs_store):
+    results = fs_store.query()  # returns all
+    assert len(results) == 26
+    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results]
+    assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results]
+
+
+def test_filesystem_store_query_multiple_filters(fs_store):
+    fs_store.source.filters.add(Filter("labels", "in", "tool"))
+    tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966"))
+    assert len(tools) == 1
+    assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966"
+
+
+def test_filesystem_store_query_dont_include_type_folder(fs_store):
+    results = fs_store.query(Filter("type", "!=", "tool"))
+    assert len(results) == 24
+
+
+def test_filesystem_store_add(fs_store):
+    # add()
+    camp1 = Campaign(name="Great Heathen Army",
+                     objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England",
+                     aliases=["Ragnar"])
+    fs_store.add(camp1)
+
+    camp1_r = fs_store.get(camp1.id)
+    assert camp1_r.id == camp1.id
+    assert camp1_r.name == camp1.name
+
+    # remove
+    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
+
+
+def test_filesystem_store_add_as_bundle():
+    fs_store = FileSystemStore(FS_PATH, bundlify=True)
+
+    camp1 = Campaign(name="Great Heathen Army",
+                     objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England",
+                     aliases=["Ragnar"])
+    fs_store.add(camp1)
+
+    with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file:
+        assert '"type": "bundle"' in bundle_file.read()
+
+    camp1_r = fs_store.get(camp1.id)
+    assert camp1_r.id == camp1.id
+    assert camp1_r.name == camp1.name
+
+    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
+
+
+def test_filesystem_add_bundle_object(fs_store):
+    bundle = Bundle()
+    fs_store.add(bundle)
+
+
+def test_filesystem_store_add_invalid_object(fs_store):
+    ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a')  # tuple isn't valid
+    with pytest.raises(TypeError) as excinfo:
+        fs_store.add(ind)
+    assert 'stix_data must be' in str(excinfo.value)
+    assert 'a STIX object' in str(excinfo.value)
+    assert 'JSON formatted STIX' in str(excinfo.value)
+    assert 'JSON formatted STIX bundle' in str(excinfo.value)
+
+
+def test_filesystem_object_with_custom_property(fs_store):
+    camp = Campaign(name="Scipio Africanus",
+                    objective="Defeat the Carthaginians",
+                    x_empire="Roman",
+                    allow_custom=True)
+
+    fs_store.add(camp, True)
+
+    camp_r = fs_store.get(camp.id)
+    assert camp_r.id == camp.id
+    assert camp_r.x_empire == camp.x_empire
+
+
+def test_filesystem_object_with_custom_property_in_bundle(fs_store):
+    camp = Campaign(name="Scipio Africanus",
+                    objective="Defeat the Carthaginians",
+                    x_empire="Roman",
+                    allow_custom=True)
+
+    bundle = Bundle(camp, allow_custom=True)
+    fs_store.add(bundle)
+
+    camp_r = fs_store.get(camp.id)
+    assert camp_r.id == camp.id
+    assert camp_r.x_empire == camp.x_empire
+
+
+def test_filesystem_custom_object(fs_store):
+    @CustomObject('x-new-obj', [
+        ('property1', properties.StringProperty(required=True)),
+    ])
+    class NewObj():
+        pass
+
+    newobj = NewObj(property1='something')
+    fs_store.add(newobj)
+
+    newobj_r = fs_store.get(newobj.id)
+    assert newobj_r.id == newobj.id
+    assert newobj_r.property1 == 'something'
+
+    # remove dir
+    shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True)
+
+
+def test_relationships(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.relationships(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_type(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.relationships(mal, relationship_type='indicates')
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[0]
+
+
+def test_relationships_by_source(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[1]
+
+
+def test_relationships_by_target(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_type(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_source(rel_fs_store):
+    with pytest.raises(ValueError) as excinfo:
+        rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True)
+
+    assert 'not both' in str(excinfo.value)
+
+
+def test_related_to(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.related_to(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_source(rel_fs_store):
+    resp = rel_fs_store.related_to(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_target(rel_fs_store):
+    resp = rel_fs_store.related_to(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)

From 5c4472cbbdd945e5c4cd6bd981d4e7b18d9656fd Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:17:19 -0400
Subject: [PATCH 081/150] Moved from test_datastore.py -> new file
 test_datastore_filters.py

---
 stix2/test/test_datastore_filters.py | 280 +++++++++++++++++++++++++++
 1 file changed, 280 insertions(+)
 create mode 100644 stix2/test/test_datastore_filters.py

diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py
new file mode 100644
index 0000000..4f66ff5
--- /dev/null
+++ b/stix2/test/test_datastore_filters.py
@@ -0,0 +1,280 @@
+import pytest
+
+from stix2.datastore.filters import Filter, apply_common_filters
+
+
+def test_filter_ops_check():
+    # invalid filters - non supported operators
+
+    with pytest.raises(ValueError) as excinfo:
+        # create Filter that has an operator that is not allowed
+        Filter('modified', '*', 'not supported operator')
+    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
+
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "%", "4")
+    assert "Filter operator '%' not supported for specified property" in str(excinfo.value)
+
+
+def test_filter_value_type_check():
+    # invalid filters - non supported value types
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter('created', '=', object())
+    # On Python 2, the type of object() is `` On Python 3, it's ``.
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", complex(2, -1))
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        Filter("type", "=", set([16, 23]))
+    assert any([s in str(excinfo.value) for s in ["", "''"]])
+    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
+
+
+def test_filter_type_underscore_check():
+    # check that Filters where property="type", value (name) doesnt have underscores
+    with pytest.raises(ValueError) as excinfo:
+        Filter("type", "=", "oh_underscore")
+    assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
+
+
+def test_apply_common_filters():
+    stix_objs = [
+        {
+            "created": "2017-01-27T13:49:53.997Z",
+            "description": "\n\nTITLE:\n\tPoison Ivy",
+            "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+            "labels": [
+                "remote-access-trojan"
+            ],
+            "modified": "2017-01-27T13:49:53.997Z",
+            "name": "Poison Ivy",
+            "type": "malware"
+        },
+        {
+            "created": "2014-05-08T09:00:00.000Z",
+            "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+            "labels": [
+                "file-hash-watchlist"
+            ],
+            "modified": "2014-05-08T09:00:00.000Z",
+            "name": "File hash for Poison Ivy variant",
+            "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
+            "type": "indicator",
+            "valid_from": "2014-05-08T09:00:00.000000Z"
+        },
+        {
+            "created": "2014-05-08T09:00:00.000Z",
+            "granular_markings": [
+                {
+                    "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
+                    "selectors": [
+                        "relationship_type"
+                    ]
+                }
+            ],
+            "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
+            "modified": "2014-05-08T09:00:00.000Z",
+            "object_marking_refs": [
+                "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
+            ],
+            "relationship_type": "indicates",
+            "revoked": True,
+            "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+            "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+            "type": "relationship"
+        },
+        {
+            "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
+            "created": "2016-02-14T00:00:00.000Z",
+            "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
+            "modified": "2016-02-14T00:00:00.000Z",
+            "type": "vulnerability",
+            "name": "CVE-2014-0160",
+            "description": "The (1) TLS...",
+            "external_references": [
+                {
+                    "source_name": "cve",
+                    "external_id": "CVE-2014-0160"
+                }
+            ],
+            "labels": ["heartbleed", "has-logo"]
+        }
+    ]
+
+    filters = [
+        Filter("type", "!=", "relationship"),
+        Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
+        Filter("labels", "in", "remote-access-trojan"),
+        Filter("created", ">", "2015-01-01T01:00:00.000Z"),
+        Filter("revoked", "=", True),
+        Filter("revoked", "!=", True),
+        Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
+        Filter("granular_markings.selectors", "in", "relationship_type"),
+        Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
+        Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
+        Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
+        Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
+        Filter("granular_markings.selectors", "in", "description"),
+        Filter("external_references.source_name", "=", "CVE"),
+    ]
+
+    # "Return any object whose type is not relationship"
+    resp = list(apply_common_filters(stix_objs, [filters[0]]))
+    ids = [r['id'] for r in resp]
+    assert stix_objs[0]['id'] in ids
+    assert stix_objs[1]['id'] in ids
+    assert stix_objs[3]['id'] in ids
+    assert len(ids) == 3
+
+    # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
+    resp = list(apply_common_filters(stix_objs, [filters[1]]))
+    assert resp[0]['id'] == stix_objs[2]['id']
+    assert len(resp) == 1
+
+    # "Return any object that contains remote-access-trojan in labels"
+    resp = list(apply_common_filters(stix_objs, [filters[2]]))
+    assert resp[0]['id'] == stix_objs[0]['id']
+    assert len(resp) == 1
+
+    # "Return any object created after 2015-01-01T01:00:00.000Z"
+    resp = list(apply_common_filters(stix_objs, [filters[3]]))
+    assert resp[0]['id'] == stix_objs[0]['id']
+    assert len(resp) == 2
+
+    # "Return any revoked object"
+    resp = list(apply_common_filters(stix_objs, [filters[4]]))
+    assert resp[0]['id'] == stix_objs[2]['id']
+    assert len(resp) == 1
+
+    # "Return any object whose not revoked"
+    # Note that if 'revoked' property is not present in object.
+    # Currently we can't use such an expression to filter for... :(
+    resp = list(apply_common_filters(stix_objs, [filters[5]]))
+    assert len(resp) == 0
+
+    # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
+    resp = list(apply_common_filters(stix_objs, [filters[6]]))
+    assert resp[0]['id'] == stix_objs[2]['id']
+    assert len(resp) == 1
+
+    # "Return any object that contains relationship_type in their selectors AND
+    # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
+    resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
+    assert resp[0]['id'] == stix_objs[2]['id']
+    assert len(resp) == 1
+
+    # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
+    resp = list(apply_common_filters(stix_objs, [filters[9]]))
+    assert resp[0]['id'] == stix_objs[3]['id']
+    assert len(resp) == 1
+
+    # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
+    resp = list(apply_common_filters(stix_objs, [filters[10]]))
+    assert len(resp) == 1
+
+    # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
+    resp = list(apply_common_filters(stix_objs, [filters[11]]))
+    assert len(resp) == 0
+
+    # "Return any object that contains description in its selectors" (None)
+    resp = list(apply_common_filters(stix_objs, [filters[12]]))
+    assert len(resp) == 0
+
+    # "Return any object that object that matches CVE in source_name" (None, case sensitive)
+    resp = list(apply_common_filters(stix_objs, [filters[13]]))
+    assert len(resp) == 0
+
+
+def test_filters0():
+    # "Return any object modified before 2017-01-28T13:49:53.935Z"
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
+    assert resp[0]['id'] == STIX_OBJS2[1]['id']
+    assert len(resp) == 2
+
+
+def test_filters1():
+    # "Return any object modified after 2017-01-28T13:49:53.935Z"
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
+    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    assert len(resp) == 1
+
+
+def test_filters2():
+    # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
+    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    assert len(resp) == 3
+
+
+def test_filters3():
+    # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
+    assert resp[0]['id'] == STIX_OBJS2[1]['id']
+    assert len(resp) == 2
+
+
+def test_filters4():
+    # Assert invalid Filter cannot be created
+    with pytest.raises(ValueError) as excinfo:
+        Filter("modified", "?", "2017-01-27T13:49:53.935Z")
+    assert str(excinfo.value) == ("Filter operator '?' not supported "
+                                  "for specified property: 'modified'")
+
+
+def test_filters5():
+    # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
+    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    assert len(resp) == 1
+
+
+def test_filters6():
+    # Test filtering on non-common property
+    resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")]))
+    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    assert len(resp) == 3
+
+
+def test_filters7():
+    # Test filtering on embedded property
+    stix_objects = list(STIX_OBJS2) + [{
+        "type": "observed-data",
+        "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
+        "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 50,
+        "objects": {
+            "0": {
+                "type": "file",
+                "hashes": {
+                    "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f"
+                },
+                "extensions": {
+                    "pdf-ext": {
+                        "version": "1.7",
+                        "document_info_dict": {
+                            "Title": "Sample document",
+                            "Author": "Adobe Systems Incorporated",
+                            "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh",
+                            "Producer": "Acrobat Distiller 3.01 for Power Macintosh",
+                            "CreationDate": "20070412090123-02"
+                        },
+                        "pdfid0": "DFCE52BD827ECF765649852119D",
+                        "pdfid1": "57A1E0F9ED2AE523E313C"
+                    }
+                }
+            }
+        }
+    }]
+    resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
+    assert resp[0]['id'] == stix_objects[3]['id']
+    assert len(resp) == 1

From 59ff0c4f26d71aed662bee4ce1bab691f9530af6 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:17:49 -0400
Subject: [PATCH 082/150] Moved from test_datastore.py -> new file
 test_datastore_memory.py

---
 stix2/test/test_datastore_memory.py | 86 +++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 stix2/test/test_datastore_memory.py

diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
new file mode 100644
index 0000000..b027640
--- /dev/null
+++ b/stix2/test/test_datastore_memory.py
@@ -0,0 +1,86 @@
+import pytest
+
+from stix2.datastore import CompositeDataSource, make_id
+from stix2.datastore.memory import MemorySource, MemorySink
+
+
+def test_add_remove_composite_datasource():
+    cds = CompositeDataSource()
+    ds1 = MemorySource()
+    ds2 = MemorySource()
+    ds3 = MemorySink()
+
+    with pytest.raises(TypeError) as excinfo:
+        cds.add_data_sources([ds1, ds2, ds1, ds3])
+    assert str(excinfo.value) == ("DataSource (to be added) is not of type "
+                                  "stix2.DataSource. DataSource type is ''")
+
+    cds.add_data_sources([ds1, ds2, ds1])
+
+    assert len(cds.get_all_data_sources()) == 2
+
+    cds.remove_data_sources([ds1.id, ds2.id])
+
+    assert len(cds.get_all_data_sources()) == 0
+
+
+def test_composite_datasource_operations():
+    BUNDLE1 = dict(id="bundle--%s" % make_id(),
+                   objects=STIX_OBJS1,
+                   spec_version="2.0",
+                   type="bundle")
+    cds1 = CompositeDataSource()
+    ds1_1 = MemorySource(stix_data=BUNDLE1)
+    ds1_2 = MemorySource(stix_data=STIX_OBJS2)
+
+    cds2 = CompositeDataSource()
+    ds2_1 = MemorySource(stix_data=BUNDLE1)
+    ds2_2 = MemorySource(stix_data=STIX_OBJS2)
+
+    cds1.add_data_sources([ds1_1, ds1_2])
+    cds2.add_data_sources([ds2_1, ds2_2])
+
+    indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+
+    # In STIX_OBJS2 changed the 'modified' property to a later time...
+    assert len(indicators) == 2
+
+    cds1.add_data_sources([cds2])
+
+    indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+
+    assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
+    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
+    assert indicator["type"] == "indicator"
+
+    query1 = [
+        Filter("type", "=", "indicator")
+    ]
+
+    query2 = [
+        Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")
+    ]
+
+    cds1.filters.update(query2)
+
+    results = cds1.query(query1)
+
+    # STIX_OBJS2 has indicator with later time, one with different id, one with
+    # original time in STIX_OBJS1
+    assert len(results) == 3
+
+    indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+
+    assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
+    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
+    assert indicator["type"] == "indicator"
+
+    # There is only one indicator with different ID. Since we use the same data
+    # when deduplicated, only two indicators (one with different modified).
+    results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert len(results) == 2
+
+    # Since we have filters already associated with our CompositeSource providing
+    # nothing returns the same as cds1.query(query1) (the associated query is query2)
+    results = cds1.query([])
+    assert len(results) == 3

From c2e7cbb3e3c3676f14f24212b863a013f0429eb0 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:18:05 -0400
Subject: [PATCH 083/150] Moved from test_datastore.py -> new file
 test_datastore_taxii.py

---
 stix2/test/test_datastore_taxii.py | 78 ++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 stix2/test/test_datastore_taxii.py

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
new file mode 100644
index 0000000..0a293b7
--- /dev/null
+++ b/stix2/test/test_datastore_taxii.py
@@ -0,0 +1,78 @@
+COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
+
+
+class MockTAXIIClient(object):
+    """Mock for taxii2_client.TAXIIClient"""
+    pass
+
+
+@pytest.fixture
+def collection():
+    return Collection(COLLECTION_URL, MockTAXIIClient())
+
+
+def test_ds_taxii(collection):
+    ds = taxii.TAXIICollectionSource(collection)
+    assert ds.collection is not None
+
+
+def test_ds_taxii_name(collection):
+    ds = taxii.TAXIICollectionSource(collection)
+    assert ds.collection is not None
+
+
+def test_parse_taxii_filters():
+    query = [
+        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
+        Filter("id", "=", "taxii stix object ID"),
+        Filter("type", "=", "taxii stix object ID"),
+        Filter("version", "=", "first"),
+        Filter("created_by_ref", "=", "Bane"),
+    ]
+
+    taxii_filters_expected = set([
+        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
+        Filter("id", "=", "taxii stix object ID"),
+        Filter("type", "=", "taxii stix object ID"),
+        Filter("version", "=", "first")
+    ])
+
+    ds = taxii.TAXIICollectionSource(collection)
+
+    taxii_filters = ds._parse_taxii_filters(query)
+
+    assert taxii_filters == taxii_filters_expected
+
+
+def test_add_get_remove_filter():
+    ds = taxii.TAXIICollectionSource(collection)
+
+    # First 3 filters are valid, remaining properties are erroneous in some way
+    valid_filters = [
+        Filter('type', '=', 'malware'),
+        Filter('id', '!=', 'stix object id'),
+        Filter('labels', 'in', ["heartbleed", "malicious-activity"]),
+    ]
+
+    assert len(ds.filters) == 0
+
+    ds.filters.add(valid_filters[0])
+    assert len(ds.filters) == 1
+
+    # Addin the same filter again will have no effect since `filters` uses a set
+    ds.filters.add(valid_filters[0])
+    assert len(ds.filters) == 1
+
+    ds.filters.add(valid_filters[1])
+    assert len(ds.filters) == 2
+    ds.filters.add(valid_filters[2])
+    assert len(ds.filters) == 3
+
+    assert set(valid_filters) == ds.filters
+
+    # remove
+    ds.filters.remove(valid_filters[0])
+
+    assert len(ds.filters) == 2
+
+    ds.filters.update(valid_filters)

From 80e4706ac4ea45ca9ced0d61b90889f420a966c1 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:19:51 -0400
Subject: [PATCH 084/150] Add some test objs previously in test_datastore.py

---
 stix2/test/conftest.py | 106 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py
index 9f61bc2..acbb8b0 100644
--- a/stix2/test/conftest.py
+++ b/stix2/test/conftest.py
@@ -46,3 +46,109 @@ def malware(uuid4, clock):
 @pytest.fixture
 def relationship(uuid4, clock):
     return stix2.Relationship(**RELATIONSHIP_KWARGS)
+
+
+@pytest.fixture
+def stix1_objs():
+    ind1 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind2 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind3 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.936Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind4 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind5 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    return [ind1, ind2, ind3, ind4, ind5]
+
+
+@pytest.fixture
+def stix2_objs():
+    ind6 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-31T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind7 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    ind8 = {
+        "created": "2017-01-27T13:49:53.935Z",
+        "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
+        "labels": [
+            "url-watchlist"
+        ],
+        "modified": "2017-01-27T13:49:53.935Z",
+        "name": "Malicious site hosting downloader",
+        "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
+        "type": "indicator",
+        "valid_from": "2017-01-27T13:49:53.935382Z"
+    }
+    return [ind6, ind7, ind8]

From 0d1729bbd7000d7c5f4d9eaf661743cf12d11587 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:20:39 -0400
Subject: [PATCH 085/150] Deleted test_filesystem in favor of
 test_datastore_filesystem.py

---
 stix2/test/test_filesystem.py | 529 ----------------------------------
 1 file changed, 529 deletions(-)
 delete mode 100644 stix2/test/test_filesystem.py

diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
deleted file mode 100644
index f59136e..0000000
--- a/stix2/test/test_filesystem.py
+++ /dev/null
@@ -1,529 +0,0 @@
-import json
-import os
-import shutil
-
-import pytest
-
-from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink,
-                   FileSystemSource, FileSystemStore, Filter, Identity,
-                   Indicator, Malware, Relationship, properties)
-
-from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
-                        IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
-                        MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
-
-FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
-
-
-@pytest.fixture
-def fs_store():
-    # create
-    yield FileSystemStore(FS_PATH)
-
-    # remove campaign dir
-    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
-
-
-@pytest.fixture
-def fs_source():
-    # create
-    fs = FileSystemSource(FS_PATH)
-    assert fs.stix_dir == FS_PATH
-    yield fs
-
-    # remove campaign dir
-    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
-
-
-@pytest.fixture
-def fs_sink():
-    # create
-    fs = FileSystemSink(FS_PATH)
-    assert fs.stix_dir == FS_PATH
-    yield fs
-
-    # remove campaign dir
-    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
-
-
-@pytest.fixture
-def bad_json_files():
-    # create erroneous JSON files for tests to make sure handled gracefully
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f:
-        f.write("Im not a JSON file")
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f:
-        f.write("Im not a JSON formatted file")
-
-    yield True  # dummy yield so can have teardown
-
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"))
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"))
-
-
-@pytest.fixture
-def bad_stix_files():
-    # create erroneous STIX JSON files for tests to make sure handled correctly
-
-    # bad STIX object
-    stix_obj = {
-        "id": "intrusion-set--test-bad-stix",
-        "spec_version": "2.0"
-        # no "type" field
-    }
-
-    with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f:
-        f.write(json.dumps(stix_obj))
-
-    yield True  # dummy yield so can have teardown
-
-    os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"))
-
-
-@pytest.fixture(scope='module')
-def rel_fs_store():
-    cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
-    idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
-    ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
-    mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
-    rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
-    rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
-    rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
-    stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
-    fs = FileSystemStore(FS_PATH)
-    for o in stix_objs:
-        fs.add(o)
-    yield fs
-
-    for o in stix_objs:
-        os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
-
-
-def test_filesystem_source_nonexistent_folder():
-    with pytest.raises(ValueError) as excinfo:
-        FileSystemSource('nonexistent-folder')
-    assert "for STIX data does not exist" in str(excinfo)
-
-
-def test_filesystem_sink_nonexistent_folder():
-    with pytest.raises(ValueError) as excinfo:
-        FileSystemSink('nonexistent-folder')
-    assert "for STIX data does not exist" in str(excinfo)
-
-
-def test_filesystem_source_bad_json_file(fs_source, bad_json_files):
-    # this tests the handling of two bad json files
-    #  - one file should just be skipped (silently) as its a ".txt" extension
-    #  - one file should be parsed and raise Exception bc its not JSON
-    try:
-        fs_source.get("intrusion-set--test-bad-json")
-    except TypeError as e:
-        assert "intrusion-set--test-bad-json" in str(e)
-        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
-
-
-def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
-    # this tests handling of bad STIX json object
-    try:
-        fs_source.get("intrusion-set--test-non-stix")
-    except TypeError as e:
-        assert "intrusion-set--test-non-stix" in str(e)
-        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
-
-
-def test_filesytem_source_get_object(fs_source):
-    # get object
-    mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")
-    assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"
-    assert mal.name == "Rover"
-
-
-def test_filesytem_source_get_nonexistent_object(fs_source):
-    ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38")
-    assert ind is None
-
-
-def test_filesytem_source_all_versions(fs_source):
-    # all versions - (currently not a true all versions call as FileSystem cant have multiple versions)
-    id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5")
-    assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5"
-    assert id_.name == "The MITRE Corporation"
-    assert id_.type == "identity"
-
-
-def test_filesytem_source_query_single(fs_source):
-    # query2
-    is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")])
-    assert len(is_2) == 1
-
-    is_2 = is_2[0]
-    assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a"
-    assert is_2.type == "attack-pattern"
-
-
-def test_filesytem_source_query_multiple(fs_source):
-    # query
-    intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")])
-    assert len(intrusion_sets) == 2
-    assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets]
-    assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets]
-
-    is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0]
-    assert "DragonOK" in is_1.aliases
-    assert len(is_1.external_references) == 4
-
-
-def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source):
-    # add python stix object
-    camp1 = Campaign(name="Hannibal",
-                     objective="Targeting Italian and Spanish Diplomat internet accounts",
-                     aliases=["War Elephant"])
-
-    fs_sink.add(camp1)
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json"))
-
-    camp1_r = fs_source.get(camp1.id)
-    assert camp1_r.id == camp1.id
-    assert camp1_r.name == "Hannibal"
-    assert "War Elephant" in camp1_r.aliases
-
-    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
-
-
-def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source):
-    # add stix object dict
-    camp2 = {
-        "name": "Aurelius",
-        "type": "campaign",
-        "objective": "German and French Intelligence Services",
-        "aliases": ["Purple Robes"],
-        "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a",
-        "created": "2017-05-31T21:31:53.197755Z"
-    }
-
-    fs_sink.add(camp2)
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json"))
-
-    camp2_r = fs_source.get(camp2["id"])
-    assert camp2_r.id == camp2["id"]
-    assert camp2_r.name == camp2["name"]
-    assert "Purple Robes" in camp2_r.aliases
-
-    os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json"))
-
-
-def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source):
-    # add stix bundle dict
-    bund = {
-        "type": "bundle",
-        "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a",
-        "spec_version": "2.0",
-        "objects": [
-            {
-                "name": "Atilla",
-                "type": "campaign",
-                "objective": "Bulgarian, Albanian and Romanian Intelligence Services",
-                "aliases": ["Huns"],
-                "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a",
-                "created": "2017-05-31T21:31:53.197755Z"
-            }
-        ]
-    }
-
-    fs_sink.add(bund)
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json"))
-
-    camp3_r = fs_source.get(bund["objects"][0]["id"])
-    assert camp3_r.id == bund["objects"][0]["id"]
-    assert camp3_r.name == bund["objects"][0]["name"]
-    assert "Huns" in camp3_r.aliases
-
-    os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json"))
-
-
-def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source):
-    # add json-encoded stix obj
-    camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\
-            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}'
-
-    fs_sink.add(camp4)
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json"))
-
-    camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a")
-    assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a"
-    assert camp4_r.name == "Ghengis Khan"
-
-    os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json"))
-
-
-def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source):
-    # add json-encoded stix bundle
-    bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \
-            ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \
-            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}'
-    fs_sink.add(bund2)
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json"))
-
-    camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a")
-    assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a"
-    assert camp5_r.name == "Spartacus"
-
-    os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json"))
-
-
-def test_filesystem_sink_add_objects_list(fs_sink, fs_source):
-    # add list of objects
-    camp6 = Campaign(name="Comanche",
-                     objective="US Midwest manufacturing firms, oil refineries, and businesses",
-                     aliases=["Horse Warrior"])
-
-    camp7 = {
-        "name": "Napolean",
-        "type": "campaign",
-        "objective": "Central and Eastern Europe military commands and departments",
-        "aliases": ["The Frenchmen"],
-        "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a",
-        "created": "2017-05-31T21:31:53.197755Z"
-    }
-
-    fs_sink.add([camp6, camp7])
-
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json"))
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json"))
-
-    camp6_r = fs_source.get(camp6.id)
-    assert camp6_r.id == camp6.id
-    assert "Horse Warrior" in camp6_r.aliases
-
-    camp7_r = fs_source.get(camp7["id"])
-    assert camp7_r.id == camp7["id"]
-    assert "The Frenchmen" in camp7_r.aliases
-
-    # remove all added objects
-    os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json"))
-    os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json"))
-
-
-def test_filesystem_store_get_stored_as_bundle(fs_store):
-    coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f")
-    assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f"
-    assert coa.type == "course-of-action"
-
-
-def test_filesystem_store_get_stored_as_object(fs_store):
-    coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd")
-    assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd"
-    assert coa.type == "course-of-action"
-
-
-def test_filesystem_store_all_versions(fs_store):
-    # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored)
-    rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0]
-    assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1"
-    assert rel.type == "relationship"
-
-
-def test_filesystem_store_query(fs_store):
-    # query()
-    tools = fs_store.query([Filter("labels", "in", "tool")])
-    assert len(tools) == 2
-    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools]
-    assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools]
-
-
-def test_filesystem_store_query_single_filter(fs_store):
-    query = Filter("labels", "in", "tool")
-    tools = fs_store.query(query)
-    assert len(tools) == 2
-    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools]
-    assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools]
-
-
-def test_filesystem_store_empty_query(fs_store):
-    results = fs_store.query()  # returns all
-    assert len(results) == 26
-    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results]
-    assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results]
-
-
-def test_filesystem_store_query_multiple_filters(fs_store):
-    fs_store.source.filters.add(Filter("labels", "in", "tool"))
-    tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966"))
-    assert len(tools) == 1
-    assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966"
-
-
-def test_filesystem_store_query_dont_include_type_folder(fs_store):
-    results = fs_store.query(Filter("type", "!=", "tool"))
-    assert len(results) == 24
-
-
-def test_filesystem_store_add(fs_store):
-    # add()
-    camp1 = Campaign(name="Great Heathen Army",
-                     objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England",
-                     aliases=["Ragnar"])
-    fs_store.add(camp1)
-
-    camp1_r = fs_store.get(camp1.id)
-    assert camp1_r.id == camp1.id
-    assert camp1_r.name == camp1.name
-
-    # remove
-    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
-
-
-def test_filesystem_store_add_as_bundle():
-    fs_store = FileSystemStore(FS_PATH, bundlify=True)
-
-    camp1 = Campaign(name="Great Heathen Army",
-                     objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England",
-                     aliases=["Ragnar"])
-    fs_store.add(camp1)
-
-    with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file:
-        assert '"type": "bundle"' in bundle_file.read()
-
-    camp1_r = fs_store.get(camp1.id)
-    assert camp1_r.id == camp1.id
-    assert camp1_r.name == camp1.name
-
-    shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
-
-
-def test_filesystem_add_bundle_object(fs_store):
-    bundle = Bundle()
-    fs_store.add(bundle)
-
-
-def test_filesystem_store_add_invalid_object(fs_store):
-    ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a')  # tuple isn't valid
-    with pytest.raises(TypeError) as excinfo:
-        fs_store.add(ind)
-    assert 'stix_data must be' in str(excinfo.value)
-    assert 'a STIX object' in str(excinfo.value)
-    assert 'JSON formatted STIX' in str(excinfo.value)
-    assert 'JSON formatted STIX bundle' in str(excinfo.value)
-
-
-def test_filesystem_object_with_custom_property(fs_store):
-    camp = Campaign(name="Scipio Africanus",
-                    objective="Defeat the Carthaginians",
-                    x_empire="Roman",
-                    allow_custom=True)
-
-    fs_store.add(camp, True)
-
-    camp_r = fs_store.get(camp.id)
-    assert camp_r.id == camp.id
-    assert camp_r.x_empire == camp.x_empire
-
-
-def test_filesystem_object_with_custom_property_in_bundle(fs_store):
-    camp = Campaign(name="Scipio Africanus",
-                    objective="Defeat the Carthaginians",
-                    x_empire="Roman",
-                    allow_custom=True)
-
-    bundle = Bundle(camp, allow_custom=True)
-    fs_store.add(bundle)
-
-    camp_r = fs_store.get(camp.id)
-    assert camp_r.id == camp.id
-    assert camp_r.x_empire == camp.x_empire
-
-
-def test_filesystem_custom_object(fs_store):
-    @CustomObject('x-new-obj', [
-        ('property1', properties.StringProperty(required=True)),
-    ])
-    class NewObj():
-        pass
-
-    newobj = NewObj(property1='something')
-    fs_store.add(newobj)
-
-    newobj_r = fs_store.get(newobj.id)
-    assert newobj_r.id == newobj.id
-    assert newobj_r.property1 == 'something'
-
-    # remove dir
-    shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True)
-
-
-def test_relationships(rel_fs_store):
-    mal = rel_fs_store.get(MALWARE_ID)
-    resp = rel_fs_store.relationships(mal)
-
-    assert len(resp) == 3
-    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
-    assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp)
-    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
-
-
-def test_relationships_by_type(rel_fs_store):
-    mal = rel_fs_store.get(MALWARE_ID)
-    resp = rel_fs_store.relationships(mal, relationship_type='indicates')
-
-    assert len(resp) == 1
-    assert resp[0]['id'] == RELATIONSHIP_IDS[0]
-
-
-def test_relationships_by_source(rel_fs_store):
-    resp = rel_fs_store.relationships(MALWARE_ID, source_only=True)
-
-    assert len(resp) == 1
-    assert resp[0]['id'] == RELATIONSHIP_IDS[1]
-
-
-def test_relationships_by_target(rel_fs_store):
-    resp = rel_fs_store.relationships(MALWARE_ID, target_only=True)
-
-    assert len(resp) == 2
-    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
-    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
-
-
-def test_relationships_by_target_and_type(rel_fs_store):
-    resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
-
-    assert len(resp) == 1
-    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
-
-
-def test_relationships_by_target_and_source(rel_fs_store):
-    with pytest.raises(ValueError) as excinfo:
-        rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True)
-
-    assert 'not both' in str(excinfo.value)
-
-
-def test_related_to(rel_fs_store):
-    mal = rel_fs_store.get(MALWARE_ID)
-    resp = rel_fs_store.related_to(mal)
-
-    assert len(resp) == 3
-    assert any(x['id'] == CAMPAIGN_ID for x in resp)
-    assert any(x['id'] == INDICATOR_ID for x in resp)
-    assert any(x['id'] == IDENTITY_ID for x in resp)
-
-
-def test_related_to_by_source(rel_fs_store):
-    resp = rel_fs_store.related_to(MALWARE_ID, source_only=True)
-
-    assert len(resp) == 1
-    assert any(x['id'] == IDENTITY_ID for x in resp)
-
-
-def test_related_to_by_target(rel_fs_store):
-    resp = rel_fs_store.related_to(MALWARE_ID, target_only=True)
-
-    assert len(resp) == 2
-    assert any(x['id'] == CAMPAIGN_ID for x in resp)
-    assert any(x['id'] == INDICATOR_ID for x in resp)

From cd85683f2b8c110981e9c5c21a4743a00a05433e Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:23:48 -0400
Subject: [PATCH 086/150] WIP: Cleaned test_datastore to only include base
 store tests

---
 stix2/test/test_datastore.py | 661 +++++------------------------------
 1 file changed, 95 insertions(+), 566 deletions(-)

diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index e80e8d8..0121a85 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -1,581 +1,110 @@
 import pytest
-from taxii2client import Collection
 
-from stix2 import Filter, MemorySink, MemorySource
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
-                             make_id, taxii)
-from stix2.datastore.filters import apply_common_filters
-from stix2.utils import deduplicate
-
-COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
+                             DataStoreMixin)
+from stix2.datastore.filters import Filter
+from stix2.test.constants import CAMPAIGN_MORE_KWARGS
 
 
-class MockTAXIIClient(object):
-    """Mock for taxii2_client.TAXIIClient"""
-    pass
-
-
-@pytest.fixture
-def collection():
-    return Collection(COLLECTION_URL, MockTAXIIClient())
-
-
-IND1 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND2 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND3 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.936Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND4 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND5 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND6 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-31T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND7 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-IND8 = {
-    "created": "2017-01-27T13:49:53.935Z",
-    "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
-    "labels": [
-        "url-watchlist"
-    ],
-    "modified": "2017-01-27T13:49:53.935Z",
-    "name": "Malicious site hosting downloader",
-    "pattern": "[url:value = 'http://x4z9arb.cn/4712']",
-    "type": "indicator",
-    "valid_from": "2017-01-27T13:49:53.935382Z"
-}
-
-STIX_OBJS2 = [IND6, IND7, IND8]
-STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5]
-
-
-def test_ds_abstract_class_smoke():
+def test_datasource_abstract_class_raises_error():
     with pytest.raises(TypeError):
         DataSource()
 
+
+def test_datasink_abstract_class_raises_error():
     with pytest.raises(TypeError):
         DataSink()
 
 
-def test_ds_taxii(collection):
-    ds = taxii.TAXIICollectionSource(collection)
-    assert ds.collection is not None
+def test_datastore_smoke():
+    assert DataStoreMixin() is not None
 
 
-def test_ds_taxii_name(collection):
-    ds = taxii.TAXIICollectionSource(collection)
-    assert ds.collection is not None
-
-
-def test_parse_taxii_filters():
-    query = [
-        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
-        Filter("id", "=", "taxii stix object ID"),
-        Filter("type", "=", "taxii stix object ID"),
-        Filter("version", "=", "first"),
-        Filter("created_by_ref", "=", "Bane"),
-    ]
-
-    taxii_filters_expected = set([
-        Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
-        Filter("id", "=", "taxii stix object ID"),
-        Filter("type", "=", "taxii stix object ID"),
-        Filter("version", "=", "first")
-    ])
-
-    ds = taxii.TAXIICollectionSource(collection)
-
-    taxii_filters = ds._parse_taxii_filters(query)
-
-    assert taxii_filters == taxii_filters_expected
-
-
-def test_add_get_remove_filter():
-    ds = taxii.TAXIICollectionSource(collection)
-
-    # First 3 filters are valid, remaining properties are erroneous in some way
-    valid_filters = [
-        Filter('type', '=', 'malware'),
-        Filter('id', '!=', 'stix object id'),
-        Filter('labels', 'in', ["heartbleed", "malicious-activity"]),
-    ]
-
-    assert len(ds.filters) == 0
-
-    ds.filters.add(valid_filters[0])
-    assert len(ds.filters) == 1
-
-    # Addin the same filter again will have no effect since `filters` uses a set
-    ds.filters.add(valid_filters[0])
-    assert len(ds.filters) == 1
-
-    ds.filters.add(valid_filters[1])
-    assert len(ds.filters) == 2
-    ds.filters.add(valid_filters[2])
-    assert len(ds.filters) == 3
-
-    assert set(valid_filters) == ds.filters
-
-    # remove
-    ds.filters.remove(valid_filters[0])
-
-    assert len(ds.filters) == 2
-
-    ds.filters.update(valid_filters)
-
-
-def test_filter_ops_check():
-    # invalid filters - non supported operators
-
-    with pytest.raises(ValueError) as excinfo:
-        # create Filter that has an operator that is not allowed
-        Filter('modified', '*', 'not supported operator')
-    assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'"
-
-    with pytest.raises(ValueError) as excinfo:
-        Filter("type", "%", "4")
-    assert "Filter operator '%' not supported for specified property" in str(excinfo.value)
-
-
-def test_filter_value_type_check():
-    # invalid filters - non supported value types
-
-    with pytest.raises(TypeError) as excinfo:
-        Filter('created', '=', object())
-    # On Python 2, the type of object() is `` On Python 3, it's ``.
-    assert any([s in str(excinfo.value) for s in ["", "''"]])
-    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
-
-    with pytest.raises(TypeError) as excinfo:
-        Filter("type", "=", complex(2, -1))
-    assert any([s in str(excinfo.value) for s in ["", "''"]])
-    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
-
-    with pytest.raises(TypeError) as excinfo:
-        Filter("type", "=", set([16, 23]))
-    assert any([s in str(excinfo.value) for s in ["", "''"]])
-    assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
-
-
-def test_filter_type_underscore_check():
-    # check that Filters where property="type", value (name) doesnt have underscores
-    with pytest.raises(ValueError) as excinfo:
-        Filter("type", "=", "oh_underscore")
-    assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
-
-
-def test_apply_common_filters():
-    stix_objs = [
-        {
-            "created": "2017-01-27T13:49:53.997Z",
-            "description": "\n\nTITLE:\n\tPoison Ivy",
-            "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "labels": [
-                "remote-access-trojan"
-            ],
-            "modified": "2017-01-27T13:49:53.997Z",
-            "name": "Poison Ivy",
-            "type": "malware"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "labels": [
-                "file-hash-watchlist"
-            ],
-            "modified": "2014-05-08T09:00:00.000Z",
-            "name": "File hash for Poison Ivy variant",
-            "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
-            "type": "indicator",
-            "valid_from": "2014-05-08T09:00:00.000000Z"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "granular_markings": [
-                {
-                    "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
-                    "selectors": [
-                        "relationship_type"
-                    ]
-                }
-            ],
-            "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
-            "modified": "2014-05-08T09:00:00.000Z",
-            "object_marking_refs": [
-                "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
-            ],
-            "relationship_type": "indicates",
-            "revoked": True,
-            "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "type": "relationship"
-        },
-        {
-            "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
-            "created": "2016-02-14T00:00:00.000Z",
-            "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
-            "modified": "2016-02-14T00:00:00.000Z",
-            "type": "vulnerability",
-            "name": "CVE-2014-0160",
-            "description": "The (1) TLS...",
-            "external_references": [
-                {
-                    "source_name": "cve",
-                    "external_id": "CVE-2014-0160"
-                }
-            ],
-            "labels": ["heartbleed", "has-logo"]
-        }
-    ]
-
-    filters = [
-        Filter("type", "!=", "relationship"),
-        Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
-        Filter("labels", "in", "remote-access-trojan"),
-        Filter("created", ">", "2015-01-01T01:00:00.000Z"),
-        Filter("revoked", "=", True),
-        Filter("revoked", "!=", True),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "relationship_type"),
-        Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
-        Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
-        Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "description"),
-        Filter("external_references.source_name", "=", "CVE"),
-    ]
-
-    # "Return any object whose type is not relationship"
-    resp = list(apply_common_filters(stix_objs, [filters[0]]))
-    ids = [r['id'] for r in resp]
-    assert stix_objs[0]['id'] in ids
-    assert stix_objs[1]['id'] in ids
-    assert stix_objs[3]['id'] in ids
-    assert len(ids) == 3
-
-    # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
-    resp = list(apply_common_filters(stix_objs, [filters[1]]))
-    assert resp[0]['id'] == stix_objs[2]['id']
-    assert len(resp) == 1
-
-    # "Return any object that contains remote-access-trojan in labels"
-    resp = list(apply_common_filters(stix_objs, [filters[2]]))
-    assert resp[0]['id'] == stix_objs[0]['id']
-    assert len(resp) == 1
-
-    # "Return any object created after 2015-01-01T01:00:00.000Z"
-    resp = list(apply_common_filters(stix_objs, [filters[3]]))
-    assert resp[0]['id'] == stix_objs[0]['id']
-    assert len(resp) == 2
-
-    # "Return any revoked object"
-    resp = list(apply_common_filters(stix_objs, [filters[4]]))
-    assert resp[0]['id'] == stix_objs[2]['id']
-    assert len(resp) == 1
-
-    # "Return any object whose not revoked"
-    # Note that if 'revoked' property is not present in object.
-    # Currently we can't use such an expression to filter for... :(
-    resp = list(apply_common_filters(stix_objs, [filters[5]]))
-    assert len(resp) == 0
-
-    # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
-    resp = list(apply_common_filters(stix_objs, [filters[6]]))
-    assert resp[0]['id'] == stix_objs[2]['id']
-    assert len(resp) == 1
-
-    # "Return any object that contains relationship_type in their selectors AND
-    # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
-    resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
-    assert resp[0]['id'] == stix_objs[2]['id']
-    assert len(resp) == 1
-
-    # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
-    resp = list(apply_common_filters(stix_objs, [filters[9]]))
-    assert resp[0]['id'] == stix_objs[3]['id']
-    assert len(resp) == 1
-
-    # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
-    resp = list(apply_common_filters(stix_objs, [filters[10]]))
-    assert len(resp) == 1
-
-    # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
-    resp = list(apply_common_filters(stix_objs, [filters[11]]))
-    assert len(resp) == 0
-
-    # "Return any object that contains description in its selectors" (None)
-    resp = list(apply_common_filters(stix_objs, [filters[12]]))
-    assert len(resp) == 0
-
-    # "Return any object that object that matches CVE in source_name" (None, case sensitive)
-    resp = list(apply_common_filters(stix_objs, [filters[13]]))
-    assert len(resp) == 0
-
-
-def test_filters0():
-    # "Return any object modified before 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[1]['id']
-    assert len(resp) == 2
-
-
-def test_filters1():
-    # "Return any object modified after 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
-    assert len(resp) == 1
-
-
-def test_filters2():
-    # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
-    assert len(resp) == 3
-
-
-def test_filters3():
-    # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[1]['id']
-    assert len(resp) == 2
-
-
-def test_filters4():
-    # Assert invalid Filter cannot be created
-    with pytest.raises(ValueError) as excinfo:
-        Filter("modified", "?", "2017-01-27T13:49:53.935Z")
-    assert str(excinfo.value) == ("Filter operator '?' not supported "
-                                  "for specified property: 'modified'")
-
-
-def test_filters5():
-    # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
-    assert len(resp) == 1
-
-
-def test_filters6():
-    # Test filtering on non-common property
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
-    assert len(resp) == 3
-
-
-def test_filters7():
-    # Test filtering on embedded property
-    stix_objects = list(STIX_OBJS2) + [{
-        "type": "observed-data",
-        "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
-        "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
-        "created": "2016-04-06T19:58:16.000Z",
-        "modified": "2016-04-06T19:58:16.000Z",
-        "first_observed": "2015-12-21T19:00:00Z",
-        "last_observed": "2015-12-21T19:00:00Z",
-        "number_observed": 50,
-        "objects": {
-            "0": {
-                "type": "file",
-                "hashes": {
-                    "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f"
-                },
-                "extensions": {
-                    "pdf-ext": {
-                        "version": "1.7",
-                        "document_info_dict": {
-                            "Title": "Sample document",
-                            "Author": "Adobe Systems Incorporated",
-                            "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh",
-                            "Producer": "Acrobat Distiller 3.01 for Power Macintosh",
-                            "CreationDate": "20070412090123-02"
-                        },
-                        "pdfid0": "DFCE52BD827ECF765649852119D",
-                        "pdfid1": "57A1E0F9ED2AE523E313C"
-                    }
-                }
-            }
-        }
-    }]
-    resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
-    assert resp[0]['id'] == stix_objects[3]['id']
-    assert len(resp) == 1
-
-
-def test_deduplicate():
-    unique = deduplicate(STIX_OBJS1)
-
-    # Only 3 objects are unique
-    # 2 id's vary
-    # 2 modified times vary for a particular id
-
-    assert len(unique) == 3
-
-    ids = [obj['id'] for obj in unique]
-    mods = [obj['modified'] for obj in unique]
-
-    assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids
-    assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids
-    assert "2017-01-27T13:49:53.935Z" in mods
-    assert "2017-01-27T13:49:53.936Z" in mods
-
-
-def test_add_remove_composite_datasource():
-    cds = CompositeDataSource()
-    ds1 = MemorySource()
-    ds2 = MemorySource()
-    ds3 = MemorySink()
-
-    with pytest.raises(TypeError) as excinfo:
-        cds.add_data_sources([ds1, ds2, ds1, ds3])
-    assert str(excinfo.value) == ("DataSource (to be added) is not of type "
-                                  "stix2.DataSource. DataSource type is ''")
-
-    cds.add_data_sources([ds1, ds2, ds1])
-
-    assert len(cds.get_all_data_sources()) == 2
-
-    cds.remove_data_sources([ds1.id, ds2.id])
-
-    assert len(cds.get_all_data_sources()) == 0
-
-
-def test_composite_datasource_operations():
-    BUNDLE1 = dict(id="bundle--%s" % make_id(),
-                   objects=STIX_OBJS1,
-                   spec_version="2.0",
-                   type="bundle")
-    cds1 = CompositeDataSource()
-    ds1_1 = MemorySource(stix_data=BUNDLE1)
-    ds1_2 = MemorySource(stix_data=STIX_OBJS2)
-
-    cds2 = CompositeDataSource()
-    ds2_1 = MemorySource(stix_data=BUNDLE1)
-    ds2_2 = MemorySource(stix_data=STIX_OBJS2)
-
-    cds1.add_data_sources([ds1_1, ds1_2])
-    cds2.add_data_sources([ds2_1, ds2_2])
-
-    indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
-
-    # In STIX_OBJS2 changed the 'modified' property to a later time...
-    assert len(indicators) == 2
-
-    cds1.add_data_sources([cds2])
-
-    indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
-
-    assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
-    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
-    assert indicator["type"] == "indicator"
-
-    query1 = [
-        Filter("type", "=", "indicator")
-    ]
-
-    query2 = [
-        Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")
-    ]
-
-    cds1.filters.update(query2)
-
-    results = cds1.query(query1)
-
-    # STIX_OBJS2 has indicator with later time, one with different id, one with
-    # original time in STIX_OBJS1
-    assert len(results) == 3
-
-    indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
-
-    assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
-    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
-    assert indicator["type"] == "indicator"
-
-    # There is only one indicator with different ID. Since we use the same data
-    # when deduplicated, only two indicators (one with different modified).
-    results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
-    assert len(results) == 2
-
-    # Since we have filters already associated with our CompositeSource providing
-    # nothing returns the same as cds1.query(query1) (the associated query is query2)
-    results = cds1.query([])
-    assert len(results) == 3
-
-
-def test_composite_datastore_no_datasource():
-    cds = CompositeDataSource()
-
+def test_datastore_get_raises():
     with pytest.raises(AttributeError) as excinfo:
-        cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
-    assert 'CompositeDataSource has no data source' in str(excinfo.value)
+        DataStoreMixin().get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_all_versions_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_query_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().query([Filter("type", "=", "indicator")])
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_creator_of_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().creator_of(CAMPAIGN_MORE_KWARGS)
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_relationships_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().relationships(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+                                       target_only=True)
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_related_to_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().related_to(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+                                    target_only=True)
+    assert "DataStoreMixin has no data source to query" == str(excinfo.value)
+
+
+def test_datastore_add_raises():
+    with pytest.raises(AttributeError) as excinfo:
+        DataStoreMixin().add(CAMPAIGN_MORE_KWARGS)
+    assert "DataStoreMixin has no data sink to put objects in" == str(excinfo.value)
+
+
+def test_composite_datastore_get_raises_error():
+    with pytest.raises(AttributeError) as excinfo:
+        CompositeDataSource().get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert "CompositeDataSource has no data sources" == str(excinfo.value)
+
+
+def test_composite_datastore_all_versions_raises_error():
+    with pytest.raises(AttributeError) as excinfo:
+        CompositeDataSource().all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert "CompositeDataSource has no data sources" == str(excinfo.value)
+
+
+def test_composite_datastore_query_raises_error():
+    with pytest.raises(AttributeError) as excinfo:
+        CompositeDataSource().query([Filter("type", "=", "indicator")])
+    assert "CompositeDataSource has no data sources" == str(excinfo.value)
+
+
+def test_composite_datastore_relationships_raises_error():
+    with pytest.raises(AttributeError) as excinfo:
+        CompositeDataSource().relationships(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+                                            target_only=True)
+    assert "CompositeDataSource has no data sources" == str(excinfo.value)
+
+
+def test_composite_datastore_related_to_raises_error():
+    with pytest.raises(AttributeError) as excinfo:
+        CompositeDataSource().related_to(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
+                                         target_only=True)
+    assert "CompositeDataSource has no data sources" == str(excinfo.value)
+
+
+def test_composite_datastore_add_data_source_raises_error():
+    with pytest.raises(TypeError) as excinfo:
+        ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
+        CompositeDataSource().add_data_source(ind)
+    assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value)
+
+
+def test_composite_datastore_add_data_sources_raises_error():
+    with pytest.raises(TypeError) as excinfo:
+        ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
+        CompositeDataSource().add_data_sources(ind)
+    assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value)

From 1ff640de3ce5bce56642c4fa35f22c8958208e20 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:24:15 -0400
Subject: [PATCH 087/150] Moved some test methods here

---
 stix2/test/test_utils.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py
index cbe5b0f..4705ec8 100644
--- a/stix2/test/test_utils.py
+++ b/stix2/test/test_utils.py
@@ -82,3 +82,21 @@ def test_get_dict_invalid(data):
 ])
 def test_get_type_from_id(stix_id, typ):
     assert stix2.utils.get_type_from_id(stix_id) == typ
+
+
+def test_deduplicate(stix1_objs):
+    unique = stix2.utils.deduplicate(stix1_objs)
+
+    # Only 3 objects are unique
+    # 2 id's vary
+    # 2 modified times vary for a particular id
+
+    assert len(unique) == 3
+
+    ids = [obj['id'] for obj in unique]
+    mods = [obj['modified'] for obj in unique]
+
+    assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids
+    assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids
+    assert "2017-01-27T13:49:53.935Z" in mods
+    assert "2017-01-27T13:49:53.936Z" in mods

From 4d35cc6f2878f8320e665d7414d6e5205860a528 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 6 Apr 2018 14:25:06 -0400
Subject: [PATCH 088/150] WIP: Update tox.ini

---
 tox.ini | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/tox.ini b/tox.ini
index bfc8c1b..884288a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,15 +1,15 @@
 [tox]
-envlist = py27,py34,py35,py36,pycodestyle,isort-check
+envlist = py27,py34,py35,py36,pycodestyle,isort-check,taxii-datastore
 
 [testenv]
 deps =
-    -U
-    tox
-    pytest
-    pytest-cov
-    coverage
+  -U
+  tox
+  pytest
+  pytest-cov
+  coverage
 commands =
-    py.test --cov=stix2 stix2/test/ --cov-report term-missing
+  py.test --cov=stix2 stix2/test/ --cov-report term-missing
 
 passenv = CI TRAVIS TRAVIS_*
 
@@ -22,20 +22,28 @@ commands =
   flake8
 
 [pycodestyle]
-max-line-length=160
+max-line-length = 160
 
 [flake8]
-max-line-length=160
+max-line-length = 160
 
 [testenv:isort-check]
 deps = isort
 commands =
-    isort -rc stix2 examples -df
-    isort -rc stix2 examples -c
+  isort -rc stix2 examples -df
+  isort -rc stix2 examples -c
+
+[testenv:taxii-datastore]
+deps =
+  git+https://github.com/oasis-open/cti-taxii-client.git#egg=taxii2-client
+  git+https://github.com/oasis-open/cti-taxii-server.git#egg=medallion
+
+commands =
+  py.test --cov=stix2 stix2/test/test_datastore_taxii.py --cov-report term-missing
 
 [travis]
 python =
-  2.7: py27, pycodestyle
+  2.7: py27, pycodestyle, taxii-datastore
   3.4: py34, pycodestyle
   3.5: py35, pycodestyle
-  3.6: py36, pycodestyle
+  3.6: py36, pycodestyle, taxii-datastore

From 1d06e642a4a9b5c202af20af87f3bd3cabaa2c6c Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 9 Apr 2018 09:55:29 -0400
Subject: [PATCH 089/150] Bump and pin nbsphinx version

---
 requirements.txt   | 2 +-
 stix2/workbench.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index d6abb63..32e08f2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 bumpversion
 ipython
-nbsphinx>=0.3.0
+nbsphinx==0.3.2
 pre-commit
 pytest
 pytest-cov
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 9e31b50..b47968e 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -131,7 +131,7 @@ def _setup_workbench():
     for obj_type in STIX_OBJS:
         new_class_dict = {
             '__new__': _constructor_wrapper(obj_type),
-            '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
+            '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS)
         }
         new_class = type(obj_type.__name__, (), new_class_dict)
 

From f83d9a56b56a77c505397f63efb68655ada80791 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 9 Apr 2018 13:29:53 -0400
Subject: [PATCH 090/150] Clean up documentation

---
 docs/api_ref.rst            |  2 +-
 docs/datastore_api.rst      | 39 -------------------------------------
 docs/guide.rst              |  3 +++
 docs/guide/creating.ipynb   |  2 +-
 docs/guide/datastore.ipynb  | 15 ++++++++++----
 docs/guide/filesystem.ipynb | 12 ++++++------
 docs/guide/parsing.ipynb    | 10 +++++-----
 docs/guide/ts_support.ipynb |  6 +++---
 docs/index.rst              | 19 ++++++++++++++----
 docs/overview.rst           | 25 ++++++++++--------------
 docs/roadmap.rst            | 17 ----------------
 stix2/datastore/__init__.py |  2 +-
 stix2/environment.py        |  3 +++
 stix2/utils.py              |  4 ++--
 stix2/v20/sdo.py            |  3 ++-
 15 files changed, 63 insertions(+), 99 deletions(-)
 delete mode 100644 docs/datastore_api.rst
 delete mode 100644 docs/roadmap.rst

diff --git a/docs/api_ref.rst b/docs/api_ref.rst
index ffc328c..dc66401 100644
--- a/docs/api_ref.rst
+++ b/docs/api_ref.rst
@@ -7,6 +7,6 @@ functions in the ``stix2`` API, as given by the package's docstrings.
 .. note::
     All the classes and functions detailed in the pages below are importable
     directly from `stix2`. See also:
-    :ref:`How imports will work `.
+    :ref:`How imports work `.
 
 .. automodule:: stix2
diff --git a/docs/datastore_api.rst b/docs/datastore_api.rst
deleted file mode 100644
index 5ecf09f..0000000
--- a/docs/datastore_api.rst
+++ /dev/null
@@ -1,39 +0,0 @@
-.. _datastore_api:
-
-DataStore API
-=============
-
-.. warning::
-
-    The DataStore API is still in the planning stages and may be subject to
-    major changes. We encourage anyone with feedback to contact the maintainers
-    to help ensure the API meets a large variety of use cases.
-
-One prominent feature of python-stix2 will be an interface for connecting
-different backend data stores containing STIX content. This will allow a uniform
-interface for querying and saving STIX content, and allow higher level code to
-be written without regard to the underlying data storage format. python-stix2
-will define the API and contain some default implementations of this API, but
-developers are encouraged to write their own implementations.
-
-Potential functions of the API include:
-
-* get a STIX Object by ID (returns the most recent version).
-* get all versions of a STIX object by ID.
-* get all relationships involving a given object, and all related objects.
-* save an object.
-* query for objects that match certain criteria (query syntax TBD).
-
-For all queries, the API will include a "filter" interface that can be used to
-either explicitly include or exclude results with certain criteria. For example,
-
-* only trust content from a set of object creators.
-* exclude content from certain (untrusted) object creators.
-* only include content with a confidence above a certain threshold (once
-  confidence is added to STIX).
-* only return content that can be shared with external parties (in other words,
-  that has TLP:GREEN markings).
-
-Additionally, the python-stix2 library will contain a "composite" data store,
-which implements the DataStore API while delegating functionality to one or more
-"child" data stores.
diff --git a/docs/guide.rst b/docs/guide.rst
index 80d2fb3..b722be3 100644
--- a/docs/guide.rst
+++ b/docs/guide.rst
@@ -1,6 +1,9 @@
 User's Guide
 ============
 
+This section of documentation contains guides and tutorials on how to use the
+``stix2`` library.
+
 .. toctree::
    :glob:
 
diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb
index d9464e7..61bbe15 100644
--- a/docs/guide/creating.ipynb
+++ b/docs/guide/creating.ipynb
@@ -377,7 +377,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To update the properties of an object, see the [Versioning](guide/versioning.ipynb) section."
+    "To update the properties of an object, see the [Versioning](versioning.ipynb) section."
    ]
   },
   {
diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index 474c719..7d40930 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -58,9 +58,9 @@
    "source": [
     "# DataStore API\n",
     "\n",
-    "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin), [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) constructs: a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) for both pulling and pushing.\n",
+    "The ``stix2`` library features an interface for pulling and pushing STIX 2 content. This interface consists of [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin), [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) and [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) constructs: a [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource) for pulling STIX 2 content, a [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) for pushing STIX 2 content, and a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) for both pulling and pushing.\n",
     "\n",
-    "The DataStore, [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource), [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites."
+    "The DataStore, [DataSource](../api/stix2.datastore.rst#stix2.datastore.DataSource), [DataSink](../api/stix2.datastore.rst#stix2.datastore.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. The ``stix2`` library provides the DataStore suites of [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites."
    ]
   },
   {
@@ -340,9 +340,16 @@
    "source": [
     "## Filters\n",
     "\n",
-    "The CTI Python STIX2 DataStore suites - [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst) - all use the [Filters](../api/datastore/stix2.datastore.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) so that every future query placed to that [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin).\n",
+    "The ``stix2`` DataStore suites - [FileSystem](../api/datastore/stix2.datastore.filesystem.rst), [Memory](../api/datastore/stix2.datastore.memory.rst), and [TAXII](../api/datastore/stix2.datastore.taxii.rst) - all use the [Filters](../api/datastore/stix2.datastore.filters.rst) module to allow for the querying of STIX content. Filters can be used to explicitly include or exclude results with certain criteria. For example:\n",
     "\n",
-    "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX2 objects can be filtered on. In addition, TAXII2 Filtering parameters for fields can also be used in filters.\n",
+    "* only trust content from a set of object creators\n",
+    "* exclude content from certain (untrusted) object creators\n",
+    "* only include content with a confidence above a certain threshold (once confidence is added to STIX 2)\n",
+    "* only return content that can be shared with external parties (e.g. only content that has TLP:GREEN markings)\n",
+    "\n",
+    "Filters can be created and supplied with every call to `query()`, and/or attached to a [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) so that every future query placed to that [DataStore](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.datastore.rst#stix2.datastore.DataStoreMixin).\n",
+    "\n",
+    "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX 2 objects can be filtered on. In addition, TAXII 2 Filtering parameters for fields can also be used in filters.\n",
     "\n",
     "TAXII2 filter fields:\n",
     "\n",
diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb
index 28a1843..b3aca88 100644
--- a/docs/guide/filesystem.ipynb
+++ b/docs/guide/filesystem.ipynb
@@ -58,9 +58,9 @@
    "source": [
     "## FileSystem \n",
     "\n",
-    "The FileSystem suite contains [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore), [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n",
+    "The FileSystem suite contains [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore), [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX 2 content. \n",
     "\n",
-    "The directory and file structure of the intended STIX2 content should be:\n",
+    "The directory and file structure of the intended STIX 2 content should be:\n",
     "\n",
     "```\n",
     "stix2_content/\n",
@@ -82,7 +82,7 @@
     "    /STIX2 Domain Object type\n",
     "```\n",
     "\n",
-    "The master STIX2 content directory contains subdirectories, each of which aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\", etc.). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n",
+    "The master STIX 2 content directory contains subdirectories, each of which aligns to a STIX 2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\", etc.). Within each STIX 2 domain object subdirectory are JSON files that are STIX 2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX 2 domain object found within that file. A real example of the FileSystem directory structure:\n",
     "\n",
     "```\n",
     "stix2_content/\n",
@@ -107,13 +107,13 @@
     "    /vulnerability\n",
     "```\n",
     "\n",
-    "[FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) that point the same file directory.\n",
+    "[FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is intended for use cases where STIX 2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) that point the same file directory.\n",
     "\n",
-    "For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n",
+    "For use cases where STIX 2 content will only be retrieved or pushed, then a [FileSystemSource](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource) and [FileSystemSink](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX 2 content will be retrieved from one distinct file directory and pushed to another.\n",
     "\n",
     "### FileSystem API\n",
     "\n",
-    "A note on [get()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.get), [all_versions()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.all_versions), and [query()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n",
+    "A note on [get()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.get), [all_versions()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.all_versions), and [query()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemStore) retrieves STIX 2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n",
     "\n",
     "A note on [add()](../api/datastore/stix2.datastore.filesystem.rst#stix2.datastore.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n",
     "\n",
diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb
index 284125e..4bd026f 100644
--- a/docs/guide/parsing.ipynb
+++ b/docs/guide/parsing.ipynb
@@ -676,20 +676,20 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Parsing Custom STIX Content"
+    "### Parsing Custom STIX Content"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument *allow_custom=True*. When *allow_custom* is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX2 domain objects and/or previously defined custom defined STIX2 objects. If the conversion cannot be completed (and *allow_custom* is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX2 content as valid STIX2 objects and return them. **Warning: Specifying *allow_custom* may lead to critical errors if further processing (searching, filtering, modifying etc...) of the custom STIX2 content occurs where the custom STIX2 content supplied is not valid STIX2**. This is an axiomatic possibility as the STIX2 library cannot guarantee proper processing of unknown custom STIX2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n",
+    "Parsing custom STIX objects and/or STIX objects with custom properties is also completed easily with [parse()](../api/stix2.core.rst#stix2.core.parse). Just supply the keyword argument ``allow_custom=True``. When ``allow_custom`` is specified, [parse()](../api/stix2.core.rst#stix2.core.parse) will attempt to convert the supplied STIX content to known STIX 2 domain objects and/or previously defined [custom STIX 2 objects](custom.ipynb). If the conversion cannot be completed (and ``allow_custom`` is specified), [parse()](../api/stix2.core.rst#stix2.core.parse) will treat the supplied STIX 2 content as valid STIX 2 objects and return them. **Warning: Specifying allow_custom may lead to critical errors if further processing (searching, filtering, modifying etc...) of the custom content occurs where the custom content supplied is not valid STIX 2**. This is an axiomatic possibility as the ``stix2`` library cannot guarantee proper processing of unknown custom STIX 2 objects that were explicitly flagged to be allowed, and thus may not be valid.\n",
     "\n",
-    "For examples on parsing STIX2 objects with custom STIX properties, see [Custom STIX Content:Custom Properties](custom.ipynb#Custom-Properties)\n",
+    "For examples of parsing STIX 2 objects with custom STIX properties, see [Custom STIX Content: Custom Properties](custom.ipynb#Custom-Properties)\n",
     "\n",
-    "For examples on parsing defined custom STIX2 objects, see [Custom STIX Content: Custom STIX Object Types](custom.ipynb#Custom-STIX-Object-Types)\n",
+    "For examples of parsing defined custom STIX 2 objects, see [Custom STIX Content: Custom STIX Object Types](custom.ipynb#Custom-STIX-Object-Types)\n",
     "\n",
-    "For the case where it is desired to retrieve STIX2 content from a source (e.g. file system, TAXII) that may possibly have custom STIX2 content unknown to the user, the user can create a STIX2 DataStore/Source with the flag *allow_custom=True*. As aforementioned this will configure the DataStore/Source to allow for unknown STIX2 content to be returned (albeit not converted to full STIX2 domain objects and properties); notable processing capabilites of the STIX2 library may be precluded by the unknown STIX2 content, if the content is not valid or actual STIX2 domain objects and properties."
+    "For retrieving STIX 2 content from a source (e.g. file system, TAXII) that may possibly have custom STIX 2 content unknown to the user, the user can create a STIX 2 DataStore/Source with the flag ``allow_custom=True``. As mentioned, this will configure the DataStore/Source to allow for unknown STIX 2 content to be returned (albeit not converted to full STIX 2 domain objects and properties); the ``stix2`` library may preclude processing the unknown content, if the content is not valid or actual STIX 2 domain objects and properties."
    ]
   },
   {
diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb
index 2264eb5..445e263 100644
--- a/docs/guide/ts_support.ipynb
+++ b/docs/guide/ts_support.ipynb
@@ -59,7 +59,7 @@
    "source": [
     "## Technical Specification Support\n",
     "\n",
-    "### How imports will work\n",
+    "### How imports work\n",
     "\n",
     "Imports can be used in different ways depending on the use case and support levels.\n",
     "\n",
@@ -229,7 +229,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### How parsing will work\n",
+    "### How parsing works\n",
     "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n",
     "\n",
     "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:"
@@ -363,7 +363,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### How will custom content work\n",
+    "### How custom content works\n",
     "\n",
     "[CustomObject](../api/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject), [CustomObservable](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable), [CustomMarking](../api/stix2.v20.common.rst#stix2.v20.common.CustomMarking) and [CustomExtension](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n",
     "\n",
diff --git a/docs/index.rst b/docs/index.rst
index 62d07ff..dd39903 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -3,8 +3,21 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to stix2's documentation!
-=================================
+STIX 2 Python API Documentation
+===============================
+
+.. warning::
+
+    Prior to version 1.0, all APIs are considered unstable and subject to change.
+
+Welcome to the STIX 2 Python API's documentation. This library is designed to
+help you work with STIX 2 content. For more information about STIX 2, see the
+`website `_ of the OASIS Cyber Threat Intelligence
+Technical Committee.
+
+Get started with an `overview `_ of the library, then take a look
+at the `guides and tutorials `_ to see how to use it. For information
+about a specific class or function, see the `API reference `_.
 
 .. toctree::
    :maxdepth: 3
@@ -13,8 +26,6 @@ Welcome to stix2's documentation!
    overview
    guide
    api_ref
-   datastore_api
-   roadmap
    contributing
 
 
diff --git a/docs/overview.rst b/docs/overview.rst
index 0306a22..0396ee8 100644
--- a/docs/overview.rst
+++ b/docs/overview.rst
@@ -4,7 +4,7 @@ Overview
 Goals
 -----
 
-High level goals/principles of the python-stix2 library:
+High level goals/principles of the Python ``stix2`` library:
 
 1. It should be as easy as possible (but no easier!) to perform common tasks of
    producing, consuming, and processing STIX 2 content.
@@ -17,22 +17,22 @@ Design Decisions
 ----------------
 
 To accomplish these goals, and to incorporate lessons learned while developing
-python-stix (for STIX 1.x), several decisions influenced the design of
-python-stix2:
+``python-stix`` (for STIX 1.x), several decisions influenced the design of the
+``stix2`` library:
 
 1. All data structures are immutable by default. In contrast to python-stix,
    where users would create an object and then assign attributes to it, in
-   python-stix2 all properties must be provided when creating the object.
+   ``stix2`` all properties must be provided when creating the object.
 2. Where necessary, library objects should act like ``dict``'s. When treated as
    a ``str``, the JSON reprentation of the object should be used.
 3. Core Python data types (including numeric types, ``datetime``) should be used
    when appropriate, and serialized to the correct format in JSON as specified
-   in the STIX 2.0 spec.
+   in the STIX 2 spec.
 
 Architecture
 ------------
 
-The `stix2` library APIs are divided into three logical layers, representing
+The ``stix2`` library is divided into three logical layers, representing
 different levels of abstraction useful in different types of scripts and larger
 applications. It is possible to combine multiple layers in the same program,
 and the higher levels build on the layers below.
@@ -41,7 +41,7 @@ and the higher levels build on the layers below.
 Object Layer
 ^^^^^^^^^^^^
 
-The lowest layer, **Object Layer**, is where Python objects representing STIX 2
+The lowest layer, the **Object Layer**, is where Python objects representing STIX 2
 data types (such as SDOs, SROs, and Cyber Observable Objects, as well as
 non-top-level objects like External References, Kill Chain phases, and Cyber
 Observable extensions) are created, and can be serialized and deserialized
@@ -57,8 +57,6 @@ not implemented as references between the Python objects themselves, but by
 simply having the same values in ``id`` and reference properties. There is no
 referential integrity maintained by the ``stix2`` library.
 
-*This layer is mostly complete.*
-
 Environment Layer
 ^^^^^^^^^^^^^^^^^
 
@@ -79,8 +77,7 @@ intelligence ecosystem.
 Each of these components can be used individually, or combined as part of an
 ``Environment``. These ``Environment`` objects allow different settings to be
 used by different users of a multi-user application (such as a web application).
-
-*This layer is mostly complete.*
+For more information, check out `this Environment tutorial `_.
 
 Workbench Layer
 ^^^^^^^^^^^^^^^
@@ -89,9 +86,7 @@ The highest layer of the ``stix2`` APIs is the **Workbench Layer**, designed for
 a single user in a highly-interactive analytical environment (such as a `Jupyter
 Notebook `_). It builds on the lower layers of the API,
 while hiding most of their complexity. Unlike the other layers, this layer is
-designed to be used directly by end users. For users who are comfortable with,
+designed to be used directly by end users. For users who are comfortable with
 Python, the Workbench Layer makes it easy to quickly interact with STIX data
 from a variety of sources without needing to write and run one-off Python
-scripts.
-
-*This layer is currently being developed.*
+scripts. For more information, check out `this Workbench tutorial `_.
diff --git a/docs/roadmap.rst b/docs/roadmap.rst
deleted file mode 100644
index 7fb9d8b..0000000
--- a/docs/roadmap.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-Development Roadmap
-===================
-
-.. warning::
-
-    Prior to version 1.0, all APIs are considered unstable and subject to
-    change.
-
-This is a list of (planned) features before version 1.0 is released.
-
-* Serialization of all STIX and Cyber Observable objects to JSON.
-* De-serialization (parsing) of all STIX and Cyber Observable objects.
-* APIs for versioning (revising and revoking) STIX objects.
-* APIs for marking STIX objects and interpreting markings of STIX objects.
-* :ref:`datastore_api`, providing a common interface for querying sources
-  of STIX content (such as objects in memory, on a filesystem, in a database, or
-  via a TAXII feed).
diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 442a242..4ca9d5f 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -1,4 +1,4 @@
-"""Python STIX 2.0 DataStore API
+"""Python STIX 2.0 DataStore API.
 
 .. autosummary::
    :toctree: datastore
diff --git a/stix2/environment.py b/stix2/environment.py
index e40e991..a456e95 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -1,3 +1,6 @@
+"""Python STIX 2.0 Environment API.
+"""
+
 import copy
 
 from .core import parse as _parse
diff --git a/stix2/utils.py b/stix2/utils.py
index 37ff166..9febd78 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -263,11 +263,11 @@ def get_class_hierarchy_names(obj):
 
 
 def remove_custom_stix(stix_obj):
-    """remove any custom STIX objects or properties
+    """Remove any custom STIX objects or properties.
 
     Warning: This function is a best effort utility, in that
     it will remove custom objects and properties based on the
-    type names; i.e. if "x-" prefixes object types, and "x_"
+    type names; i.e. if "x-" prefixes object types, and "x\\_"
     prefixes property types. According to the STIX2 spec,
     those naming conventions are a SHOULDs not MUSTs, meaning
     that valid custom STIX content may ignore those conventions
diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index 7ccc3e3..060b9f0 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -1,4 +1,5 @@
-"""STIX 2.0 Domain Objects"""
+"""STIX 2.0 Domain Objects.
+"""
 
 from collections import OrderedDict
 

From b851afba01e77ed972238a89fc3da1a32930067c Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 9 Apr 2018 14:58:52 -0400
Subject: [PATCH 091/150] Req. custom extension properties as list of tuples

---
 docs/guide/custom.ipynb   |  8 +++----
 stix2/test/test_custom.py | 50 +++++++++++++++++++++++----------------
 stix2/v20/observables.py  |  4 ++--
 3 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb
index bc19749..e651084 100644
--- a/docs/guide/custom.ipynb
+++ b/docs/guide/custom.ipynb
@@ -1155,10 +1155,10 @@
    "source": [
     "from stix2 import File, CustomExtension\n",
     "\n",
-    "@CustomExtension(File, 'x-new-ext', {\n",
-    "    'property1': properties.StringProperty(required=True),\n",
-    "    'property2': properties.IntegerProperty(),\n",
-    "})\n",
+    "@CustomExtension(File, 'x-new-ext', [\n",
+    "    ('property1', properties.StringProperty(required=True)),\n",
+    "    ('property2', properties.IntegerProperty()),\n",
+    "])\n",
     "class NewExtension():\n",
     "    pass\n",
     "\n",
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index cc8b32b..918c7f0 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -435,10 +435,10 @@ def test_observed_data_with_custom_observable_object():
     assert ob_data.objects['0'].property1 == 'something'
 
 
-@stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext', {
-    'property1': stix2.properties.StringProperty(required=True),
-    'property2': stix2.properties.IntegerProperty(),
-})
+@stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext', [
+    ('property1', stix2.properties.StringProperty(required=True)),
+    ('property2', stix2.properties.IntegerProperty()),
+])
 class NewExtension():
     def __init__(self, property2=None, **kwargs):
         if property2 and property2 < 10:
@@ -485,9 +485,9 @@ def test_custom_extension_invalid_observable():
     class Foo(object):
         pass
     with pytest.raises(ValueError) as excinfo:
-        @stix2.observables.CustomExtension(Foo, 'x-new-ext', {
-            'property1': stix2.properties.StringProperty(required=True),
-        })
+        @stix2.observables.CustomExtension(Foo, 'x-new-ext', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ])
         class FooExtension():
             pass  # pragma: no cover
     assert str(excinfo.value) == "'observable' must be a valid Observable class!"
@@ -495,9 +495,9 @@ def test_custom_extension_invalid_observable():
     class Bar(stix2.observables._Observable):
         pass
     with pytest.raises(ValueError) as excinfo:
-        @stix2.observables.CustomExtension(Bar, 'x-new-ext', {
-            'property1': stix2.properties.StringProperty(required=True),
-        })
+        @stix2.observables.CustomExtension(Bar, 'x-new-ext', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ])
         class BarExtension():
             pass
     assert "Unknown observable type" in str(excinfo.value)
@@ -506,9 +506,9 @@ def test_custom_extension_invalid_observable():
     class Baz(stix2.observables._Observable):
         _type = 'Baz'
     with pytest.raises(ValueError) as excinfo:
-        @stix2.observables.CustomExtension(Baz, 'x-new-ext', {
-            'property1': stix2.properties.StringProperty(required=True),
-        })
+        @stix2.observables.CustomExtension(Baz, 'x-new-ext', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ])
         class BazExtension():
             pass
     assert "Unknown observable type" in str(excinfo.value)
@@ -520,21 +520,29 @@ def test_custom_extension_no_properties():
         @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', None)
         class BarExtension():
             pass
-    assert "'properties' must be a dict!" in str(excinfo.value)
+    assert "Must supply a list, containing tuples." in str(excinfo.value)
 
 
 def test_custom_extension_empty_properties():
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', [])
+        class BarExtension():
+            pass
+    assert "Must supply a list, containing tuples." in str(excinfo.value)
+
+
+def test_custom_extension_dict_properties():
     with pytest.raises(ValueError) as excinfo:
         @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', {})
         class BarExtension():
             pass
-    assert "'properties' must be a dict!" in str(excinfo.value)
+    assert "Must supply a list, containing tuples." in str(excinfo.value)
 
 
 def test_custom_extension_no_init_1():
-    @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-extension', {
-        'property1': stix2.properties.StringProperty(required=True),
-    })
+    @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-extension', [
+        ('property1', stix2.properties.StringProperty(required=True)),
+    ])
     class NewExt():
         pass
 
@@ -543,9 +551,9 @@ def test_custom_extension_no_init_1():
 
 
 def test_custom_extension_no_init_2():
-    @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', {
-        'property1': stix2.properties.StringProperty(required=True),
-    })
+    @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', [
+        ('property1', stix2.properties.StringProperty(required=True)),
+    ])
     class NewExt2(object):
         pass
 
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 83600b0..dc1289b 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -1034,8 +1034,8 @@ def CustomExtension(observable=None, type='x-custom-observable', properties=None
                 'extensions': ExtensionsProperty(enclosing_type=_type),
             }
 
-            if not isinstance(properties, dict) or not properties:
-                raise ValueError("'properties' must be a dict!")
+            if not properties or not isinstance(properties, list):
+                raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]")
 
             _properties.update(properties)
 

From 5c5ca1f21cd4887db271bd7d97c975c962a909c0 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 9 Apr 2018 15:18:29 -0400
Subject: [PATCH 092/150] Move 'extensions' property to custom Observables

... from custom Observable extensions (an extension doesn't need an
'extensions' property).
---
 stix2/test/test_custom.py | 5 +++++
 stix2/v20/observables.py  | 7 ++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 918c7f0..a14503f 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -602,3 +602,8 @@ def test_register_custom_object():
     stix2._register_type(CustomObject2)
     # Note that we will always check against newest OBJ_MAP.
     assert (CustomObject2._type, CustomObject2) in stix2.OBJ_MAP.items()
+
+
+def test_extension_property_location():
+    assert 'extensions' in stix2.v20.observables.OBJ_MAP_OBSERVABLE['x-new-observable']._properties
+    assert 'extensions' not in stix2.v20.observables.EXT_MAP['domain-name']['x-new-ext']._properties
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index dc1289b..39a8f19 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -979,6 +979,9 @@ def CustomObservable(type='x-custom-observable', properties=None):
                                      "is not a ListProperty containing ObjectReferenceProperty." % prop_name)
 
             _properties.update(properties)
+            _properties.update([
+                ('extensions', ExtensionsProperty(enclosing_type=_type)),
+            ])
 
             def __init__(self, **kwargs):
                 _Observable.__init__(self, **kwargs)
@@ -1030,9 +1033,7 @@ def CustomExtension(observable=None, type='x-custom-observable', properties=None
         class _Custom(cls, _Extension):
 
             _type = type
-            _properties = {
-                'extensions': ExtensionsProperty(enclosing_type=_type),
-            }
+            _properties = OrderedDict()
 
             if not properties or not isinstance(properties, list):
                 raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]")

From b633fd37856ccd15257d7d6bae3223411a90e9ef Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 10 Apr 2018 12:54:27 -0400
Subject: [PATCH 093/150] WIP: Allow custom observables, extensions

---
 stix2/test/test_custom.py | 31 +++++++++++++++++++++++++++++++
 stix2/v20/observables.py  | 15 +++++++++++----
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index cc8b32b..a50819b 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -363,6 +363,7 @@ def test_parse_custom_observable_object():
     }"""
 
     nt = stix2.parse_observable(nt_string, [])
+    assert isinstance(nt, stix2.core._STIXBase)
     assert nt.property1 == 'something'
 
 
@@ -376,6 +377,32 @@ def test_parse_unregistered_custom_observable_object():
         stix2.parse_observable(nt_string)
     assert "Can't parse unknown observable type" in str(excinfo.value)
 
+    parsed_custom = stix2.parse_observable(nt_string, allow_custom=True)
+    assert parsed_custom['property1'] == 'something'
+    with pytest.raises(AttributeError) as excinfo:
+        assert parsed_custom.property1 == 'something'
+    assert not isinstance(parsed_custom, stix2.core._STIXBase)
+
+
+def test_parse_observed_data_with_custom_observable():
+    input_str = """{
+        "type": "observed-data",
+        "id": "observed-data--dc20c4ca-a2a3-4090-a5d5-9558c3af4758",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "x-foobar-observable",
+                "property1": "something"
+            }
+        }
+    }"""
+    parsed = stix2.parse(input_str, allow_custom=True)
+    assert parsed.objects['0']['property1'] == 'something'
+
 
 def test_parse_invalid_custom_observable_object():
     nt_string = """{
@@ -585,6 +612,10 @@ def test_parse_observable_with_unregistered_custom_extension():
         stix2.parse_observable(input_str)
     assert "Can't parse Unknown extension type" in str(excinfo.value)
 
+    parsed_ob = stix2.parse_observable(input_str, allow_custom=True)
+    assert parsed_ob['extensions']['x-foobar-ext']['property1'] == 'foo'
+    assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase)
+
 
 def test_register_custom_object():
     # Not the way to register custom object.
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 83600b0..4449527 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -923,15 +923,22 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
     try:
         obj_class = OBJ_MAP_OBSERVABLE[obj['type']]
     except KeyError:
+        if allow_custom:
+            # flag allows for unknown custom objects too, but will not
+            # be parsed into STIX observable object, just returned as is
+            return obj
         raise ParseError("Can't parse unknown observable type '%s'! For custom observables, "
                          "use the CustomObservable decorator." % obj['type'])
 
     if 'extensions' in obj and obj['type'] in EXT_MAP:
         for name, ext in obj['extensions'].items():
-            if name not in EXT_MAP[obj['type']]:
-                raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
-            ext_class = EXT_MAP[obj['type']][name]
-            obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
+            try:
+                ext_class = EXT_MAP[obj['type']][name]
+            except KeyError:
+                if not allow_custom:
+                    raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
+            else:  # extension was found
+                obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
 
     return obj_class(allow_custom=allow_custom, **obj)
 

From 27647091a5a59f91e86bf38490900f14964b5289 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 11 Apr 2018 13:36:52 -0400
Subject: [PATCH 094/150] WIP - just at stash point

---
 stix2/datastore/__init__.py   | 33 +++++++++++-----------
 stix2/datastore/filesystem.py | 25 ++++++-----------
 stix2/datastore/filters.py    | 53 +++++++++++++++++++++++++++++++++++
 stix2/datastore/memory.py     | 20 ++++++-------
 stix2/datastore/taxii.py      | 37 ++++++++++--------------
 5 files changed, 102 insertions(+), 66 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 78f7555..c43f309 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -16,7 +16,7 @@ import uuid
 
 from six import with_metaclass
 
-from stix2.datastore.filters import Filter
+from stix2.datastore.filters import Filter, FilterSet
 from stix2.utils import deduplicate
 
 
@@ -220,13 +220,13 @@ class DataSource(with_metaclass(ABCMeta)):
 
     Attributes:
         id (str): A unique UUIDv4 to identify this DataSource.
-        filters (set): A collection of filters attached to this DataSource.
+        filters (FilterSet): A collection of filters attached to this DataSource.
 
     """
     def __init__(self):
         super(DataSource, self).__init__()
         self.id = make_id()
-        self.filters = set()
+        self.filters = FilterSet()
 
     @abstractmethod
     def get(self, stix_id):
@@ -420,7 +420,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): the id of the STIX object to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is attached
                 to another parent CompositeDataSource), not user supplied.
 
@@ -432,11 +432,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
-        all_filters.update(self.filters)
+        all_filters = FilterSet()
+
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # for every configured Data Source, call its retrieve handler
         for ds in self.data_sources:
@@ -466,7 +467,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): id of the STIX objects to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -478,12 +479,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
+        all_filters = FilterSet()
 
-        all_filters.update(self.filters)
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # retrieve STIX objects from all configured data sources
         for ds in self.data_sources:
@@ -505,7 +506,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             query (list): list of filters to search on.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -517,17 +518,17 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         if not query:
-            # don't mess with the query (i.e. convert to a set, as that's done
+            # don't mess with the query (i.e. deduplicate, as that's done
             # within the specific DataSources that are called)
             query = []
 
         all_data = []
+        all_filters = FilterSet()
 
-        all_filters = set()
-        all_filters.update(self.filters)
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # federate query to all attached data sources,
         # pass composite filters to id
diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index a6f31cf..c13b02c 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -8,7 +8,7 @@ import os
 
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate, get_class_hierarchy_names
 
 
@@ -165,7 +165,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -195,7 +195,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -217,7 +217,7 @@ class FileSystemSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -231,20 +231,13 @@ class FileSystemSource(DataSource):
 
         all_data = []
 
-        if query is None:
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+        query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # extract any filters that are for "type" or "id" , as we can then do
         # filtering before reading in the STIX objects. A STIX 'type' filter
@@ -343,8 +336,8 @@ class FileSystemSource(DataSource):
         search space of a FileSystemStore (or FileSystemSink).
 
         """
-        file_filters = set()
+        file_filters = []
         for filter_ in query:
             if filter_.property == "id" or filter_.property == "type":
-                file_filters.add(filter_)
+                file_filters.append(filter_)
         return file_filters
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 9065b61..0946694 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -17,6 +17,23 @@ except NameError:
     pass
 
 
+def deduplicate_filters(filters):
+    """utility for deduplicating list of filters, this
+    is used when 'set()' cannot be used as one of the
+    filter values is a dict (or non-hashable type)
+
+    Args:
+        filters (list): a list of filters
+
+    Returns: list of unique filters
+    """
+    unique_filters = []
+    for filter_ in filters:
+        if filter_ not in unique_filters:
+            unique_filters.append(filter_)
+    return unique_filters
+
+
 def _check_filter_components(prop, op, value):
     """Check that filter meets minimum validity.
 
@@ -168,3 +185,39 @@ def _check_filter(filter_, stix_obj):
     else:
         # Check if property matches
         return filter_._check_property(stix_obj[prop])
+
+
+class FilterSet(object):
+    """ """
+
+    def __init__(self, filters=None):
+        """ """
+        self._filters = []
+        if filters:
+            self.add(filters)
+
+    def __iter__(self):
+        """ """
+        for f in self._filters:
+            yield f
+
+    def add(self, filters):
+        """ """
+        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+            filters = [filters]
+
+        for f in filters:
+            if f not in self._filters:
+                self._filters.append(f)
+
+        return
+
+    def remove(self, filters):
+        """ """
+        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+            filters = [filters]
+
+        for f in filters:
+            self._filters.remove(f)
+
+        return
diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index e057271..e6f0fd2 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -18,7 +18,7 @@ import os
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 
 
 def _add(store, stix_data=None, version=None):
@@ -197,7 +197,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -236,7 +236,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX 2 object to retrieve.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -258,7 +258,7 @@ class MemorySource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -269,19 +269,15 @@ class MemorySource(DataSource):
 
         """
         if query is None:
-            query = set()
+            query = FilterSet()
         else:
-            if not isinstance(query, list):
-                # make sure don't make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+            query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # Apply STIX common property filters.
         all_data = list(apply_common_filters(self._data.values(), query))
diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 0a58763..faa2669 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -6,7 +6,7 @@ from requests.exceptions import HTTPError
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate
 
 TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
@@ -120,7 +120,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -132,11 +132,12 @@ class TAXIICollectionSource(DataSource):
 
         """
         # combine all query filters
-        query = set()
+        query = FilterSet()
+
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # dont extract TAXII filters from query (to send to TAXII endpoint)
         # as directly retrieveing a STIX object by ID
@@ -164,7 +165,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -198,7 +199,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -209,20 +210,13 @@ class TAXIICollectionSource(DataSource):
                 parsed into python STIX objects and then returned.
 
         """
-        if query is None:
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
+        query = FilterSet(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # parse taxii query params (that can be applied remotely)
         taxii_filters = self._parse_taxii_filters(query)
@@ -268,17 +262,16 @@ class TAXIICollectionSource(DataSource):
 
 
         Args:
-            query (set): set of filters to extract which ones are TAXII
+            query (list): list of filters to extract which ones are TAXII
                 specific.
 
-        Returns:
-            taxii_filters (set): set of the TAXII filters
+        Returns: a list of the TAXII filters
 
         """
-        taxii_filters = set()
+        taxii_filters = []
 
         for filter_ in query:
             if filter_.property in TAXII_FILTERS:
-                taxii_filters.add(filter_)
+                taxii_filters.append(filter_)
 
         return taxii_filters

From 76f1474a8b6a5d11dbec170f29ed9219e5721128 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 11 Apr 2018 14:43:44 -0400
Subject: [PATCH 095/150] Pin sphinx version for ReadTheDocs

See also: https://github.com/rtfd/readthedocs.org/issues/3148
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 32e08f2..5ac0b37 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ nbsphinx==0.3.2
 pre-commit
 pytest
 pytest-cov
-sphinx
+sphinx<1.6
 sphinx-prompt
 tox
 

From 1d22c757ef79a02a0ea0b22962a5009ee8e82ce8 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 11 Apr 2018 15:40:52 -0400
Subject: [PATCH 096/150] Improve Tox test harness

- No need for both pycodestyle and flake8; flake8 includes the former.
- Use the proper pytest invocation.
---
 tox.ini | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/tox.ini b/tox.ini
index ac4e89f..86cd4ee 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,py35,py36,pycodestyle,isort-check
+envlist = py27,py34,py35,py36,style,isort-check
 
 [testenv]
 deps =
@@ -10,22 +10,17 @@ deps =
     coverage
     taxii2-client
 commands =
-    py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
-    py.test stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
+    pytest --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
+    pytest stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
 
 passenv = CI TRAVIS TRAVIS_*
 
-[testenv:pycodestyle]
+[testenv:style]
 deps =
   flake8
-  pycodestyle
 commands =
-  pycodestyle ./stix2
   flake8
 
-[pycodestyle]
-max-line-length=160
-
 [flake8]
 max-line-length=160
 
@@ -37,7 +32,7 @@ commands =
 
 [travis]
 python =
-  2.7: py27, pycodestyle
-  3.4: py34, pycodestyle
-  3.5: py35, pycodestyle
-  3.6: py36, pycodestyle
+  2.7: py27, style
+  3.4: py34, style
+  3.5: py35, style
+  3.6: py36, style

From 3e048ef325ba129a71c03708c2f07c953e26c5de Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 11 Apr 2018 15:46:17 -0400
Subject: [PATCH 097/150] Fix deprecated 3.6 backslash-character pairs

(https://docs.python.org/3/whatsnew/3.6.html#deprecated-python-behavior)
---
 stix2/test/test_external_reference.py  | 4 ++--
 stix2/test/test_malware.py             | 2 +-
 stix2/test/test_pattern_expressions.py | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py
index 2b79f01..9b90998 100644
--- a/stix2/test/test_external_reference.py
+++ b/stix2/test/test_external_reference.py
@@ -42,7 +42,7 @@ def test_external_reference_capec():
     )
 
     assert str(ref) == CAPEC
-    assert re.match("ExternalReference\(source_name=u?'capec', external_id=u?'CAPEC-550'\)", repr(ref))
+    assert re.match("ExternalReference\\(source_name=u?'capec', external_id=u?'CAPEC-550'\\)", repr(ref))
 
 
 CAPEC_URL = """{
@@ -109,7 +109,7 @@ def test_external_reference_offline():
     )
 
     assert str(ref) == OFFLINE
-    assert re.match("ExternalReference\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\)", repr(ref))
+    assert re.match("ExternalReference\\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\\)", repr(ref))
     # Yikes! This works
     assert eval("stix2." + repr(ref)) == ref
 
diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py
index 8c565cd..2228885 100644
--- a/stix2/test/test_malware.py
+++ b/stix2/test/test_malware.py
@@ -126,7 +126,7 @@ def test_parse_malware(data):
 
 
 def test_parse_malware_invalid_labels():
-    data = re.compile('\[.+\]', re.DOTALL).sub('1', EXPECTED_MALWARE)
+    data = re.compile('\\[.+\\]', re.DOTALL).sub('1', EXPECTED_MALWARE)
     with pytest.raises(ValueError) as excinfo:
         stix2.parse(data)
     assert "Invalid value for Malware 'labels'" in str(excinfo.value)
diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py
index 363458a..74a7d0f 100644
--- a/stix2/test/test_pattern_expressions.py
+++ b/stix2/test/test_pattern_expressions.py
@@ -319,13 +319,13 @@ def test_invalid_binary_constant():
 
 def test_escape_quotes_and_backslashes():
     exp = stix2.MatchesComparisonExpression("file:name",
-                                            "^Final Report.+\.exe$")
+                                            "^Final Report.+\\.exe$")
     assert str(exp) == "file:name MATCHES '^Final Report.+\\\\.exe$'"
 
 
 def test_like():
     exp = stix2.LikeComparisonExpression("directory:path",
-                                         "C:\Windows\%\\foo")
+                                         "C:\\Windows\\%\\foo")
     assert str(exp) == "directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo'"
 
 

From ba6fa595c6e459303cdc712304bb84b6ff4c9d03 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 11 Apr 2018 20:54:16 -0400
Subject: [PATCH 098/150] WIP - finding more issues with allowing dicts as
 filters

---
 stix2/datastore/filters.py   |  15 +++-
 stix2/environment.py         |   2 +-
 stix2/test/test_datastore.py | 144 ++++++++++++++++++++++++++++++++---
 3 files changed, 147 insertions(+), 14 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 0946694..f0028cb 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -5,11 +5,13 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores
 
 import collections
 
+from stix2.utils import STIXdatetime
+
 """Supported filter operations"""
 FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
 
 """Supported filter value types"""
-FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
+FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple, STIXdatetime]
 try:
     FILTER_VALUE_TYPES.append(unicode)
 except NameError:
@@ -169,19 +171,26 @@ def _check_filter(filter_, stix_obj):
         # Check embedded properties, from e.g. granular_markings or external_references
         sub_property = filter_.property.split(".", 1)[1]
         sub_filter = filter_._replace(property=sub_property)
+
         if isinstance(stix_obj[prop], list):
             for elem in stix_obj[prop]:
                 if _check_filter(sub_filter, elem) is True:
                     return True
             return False
+
+        elif isinstance(stix_obj[prop], dict):
+            return _check_filter(sub_filter, stix_obj[prop])
+
         else:
             return _check_filter(sub_filter, stix_obj[prop])
+
     elif isinstance(stix_obj[prop], list):
         # Check each item in list property to see if it matches
         for elem in stix_obj[prop]:
             if filter_._check_property(elem) is True:
                 return True
         return False
+
     else:
         # Check if property matches
         return filter_._check_property(stix_obj[prop])
@@ -201,6 +210,10 @@ class FilterSet(object):
         for f in self._filters:
             yield f
 
+    def __len__(self):
+        """ """
+        return len(self._filters)
+
     def add(self, filters):
         """ """
         if not isinstance(filters, FilterSet) and not isinstance(filters, list):
diff --git a/stix2/environment.py b/stix2/environment.py
index eb5583e..30f2802 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -115,7 +115,7 @@ class Environment(DataStoreMixin):
 
     def add_filters(self, *args, **kwargs):
         try:
-            return self.source.filters.update(*args, **kwargs)
+            return self.source.filters.add(*args, **kwargs)
         except AttributeError:
             raise AttributeError('Environment has no data source')
 
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index e80e8d8..01304b9 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -2,10 +2,11 @@ import pytest
 from taxii2client import Collection
 
 from stix2 import Filter, MemorySink, MemorySource
+from stix2.core import parse
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
                              make_id, taxii)
 from stix2.datastore.filters import apply_common_filters
-from stix2.utils import deduplicate
+from stix2.utils import deduplicate, parse_into_datetime
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
 
@@ -120,6 +121,9 @@ IND8 = {
 STIX_OBJS2 = [IND6, IND7, IND8]
 STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5]
 
+REAL_STIX_OBJS2 = [parse(IND6), parse(IND7), parse(IND8)]
+REAL_STIX_OBJS1 = [parse(IND1), parse(IND2), parse(IND3), parse(IND4), parse(IND5)]
+
 
 def test_ds_abstract_class_smoke():
     with pytest.raises(TypeError):
@@ -148,12 +152,12 @@ def test_parse_taxii_filters():
         Filter("created_by_ref", "=", "Bane"),
     ]
 
-    taxii_filters_expected = set([
+    taxii_filters_expected = [
         Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
         Filter("id", "=", "taxii stix object ID"),
         Filter("type", "=", "taxii stix object ID"),
         Filter("version", "=", "first")
-    ])
+    ]
 
     ds = taxii.TAXIICollectionSource(collection)
 
@@ -177,7 +181,7 @@ def test_add_get_remove_filter():
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
-    # Addin the same filter again will have no effect since `filters` uses a set
+    # Addin the same filter again will have no effect since `filters` acts like a set
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
@@ -186,14 +190,14 @@ def test_add_get_remove_filter():
     ds.filters.add(valid_filters[2])
     assert len(ds.filters) == 3
 
-    assert set(valid_filters) == ds.filters
+    assert valid_filters == [f for f in ds.filters]
 
     # remove
     ds.filters.remove(valid_filters[0])
 
     assert len(ds.filters) == 2
 
-    ds.filters.update(valid_filters)
+    ds.filters.add(valid_filters)
 
 
 def test_filter_ops_check():
@@ -297,9 +301,32 @@ def test_apply_common_filters():
                 }
             ],
             "labels": ["heartbleed", "has-logo"]
+        },
+        {
+            "type": "observed-data",
+            "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
+            "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+            "created": "2016-04-06T19:58:16.000Z",
+            "modified": "2016-04-06T19:58:16.000Z",
+            "first_observed": "2015-12-21T19:00:00Z",
+            "last_observed": "2015-12-21T19:00:00Z",
+            "number_observed": 1,
+            "objects": {
+                "0": {
+                    "type": "file",
+                    "name": "HAL 9000.exe"
+                }
+            }
+
         }
     ]
 
+    # same as above objects but converted to real Python STIX2 objects
+    # to test filters against true Python STIX2 objects
+    print(stix_objs)
+    real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs]
+    print("after\n\n")
+    print(stix_objs)
     filters = [
         Filter("type", "!=", "relationship"),
         Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
@@ -315,6 +342,7 @@ def test_apply_common_filters():
         Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
         Filter("granular_markings.selectors", "in", "description"),
         Filter("external_references.source_name", "=", "CVE"),
+        Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}})
     ]
 
     # "Return any object whose type is not relationship"
@@ -323,66 +351,125 @@ def test_apply_common_filters():
     assert stix_objs[0]['id'] in ids
     assert stix_objs[1]['id'] in ids
     assert stix_objs[3]['id'] in ids
-    assert len(ids) == 3
+    assert len(ids) == 4
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[0]]))
+    ids = [r.id for r in resp]
+    assert real_stix_objs[0].id in ids
+    assert real_stix_objs[1].id in ids
+    assert real_stix_objs[3].id in ids
+    assert len(ids) == 4
 
     # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
     resp = list(apply_common_filters(stix_objs, [filters[1]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[1]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
     # "Return any object that contains remote-access-trojan in labels"
     resp = list(apply_common_filters(stix_objs, [filters[2]]))
     assert resp[0]['id'] == stix_objs[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[2]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 1
+
     # "Return any object created after 2015-01-01T01:00:00.000Z"
     resp = list(apply_common_filters(stix_objs, [filters[3]]))
     assert resp[0]['id'] == stix_objs[0]['id']
-    assert len(resp) == 2
+    assert len(resp) == 3
 
     # "Return any revoked object"
     resp = list(apply_common_filters(stix_objs, [filters[4]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[4]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
     # "Return any object whose not revoked"
     # Note that if 'revoked' property is not present in object.
     # Currently we can't use such an expression to filter for... :(
     resp = list(apply_common_filters(stix_objs, [filters[5]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
+    assert len(resp) == 0
+
     # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
     resp = list(apply_common_filters(stix_objs, [filters[6]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[6]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
     # "Return any object that contains relationship_type in their selectors AND
     # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
     resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[7], filters[8]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
     # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
     resp = list(apply_common_filters(stix_objs, [filters[9]]))
     assert resp[0]['id'] == stix_objs[3]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[9]]))
+    assert resp[0].id == real_stix_objs[3].id
+    assert len(resp) == 1
+
     # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
     resp = list(apply_common_filters(stix_objs, [filters[10]]))
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[10]]))
+    assert len(resp) == 1
+
     # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
     resp = list(apply_common_filters(stix_objs, [filters[11]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[11]]))
+    assert len(resp) == 0
+
     # "Return any object that contains description in its selectors" (None)
     resp = list(apply_common_filters(stix_objs, [filters[12]]))
     assert len(resp) == 0
 
-    # "Return any object that object that matches CVE in source_name" (None, case sensitive)
+    resp = list(apply_common_filters(real_stix_objs, [filters[12]]))
+    assert len(resp) == 0
+
+    # "Return any object that matches CVE in source_name" (None, case sensitive)
     resp = list(apply_common_filters(stix_objs, [filters[13]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[13]]))
+    assert len(resp) == 0
+
+    # Return any object that matches file object in "objects"
+    # BUG: This test is brokem , weird behavior, the file obj
+    # in stix_objs is being parsed into real python-stix2 obj even though
+    # it never goes through parse() --> BAD <_<
+    print(stix_objs)
+    resp = list(apply_common_filters(stix_objs, [filters[14]]))
+    assert resp[0]["id"] == stix_objs[14]["id"]
+    assert len(resp) == 1
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[14]]))
+    assert resp[0].id == real_stix_objs[14].id
+    assert len(resp) == 1
+
 
 def test_filters0():
     # "Return any object modified before 2017-01-28T13:49:53.935Z"
@@ -390,6 +477,10 @@ def test_filters0():
     assert resp[0]['id'] == STIX_OBJS2[1]['id']
     assert len(resp) == 2
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", "<", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[1].id
+    assert len(resp) == 2
+
 
 def test_filters1():
     # "Return any object modified after 2017-01-28T13:49:53.935Z"
@@ -397,6 +488,10 @@ def test_filters1():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", ">", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 1
+
 
 def test_filters2():
     # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
@@ -404,6 +499,10 @@ def test_filters2():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 3
+
 
 def test_filters3():
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
@@ -411,6 +510,11 @@ def test_filters3():
     assert resp[0]['id'] == STIX_OBJS2[1]['id']
     assert len(resp) == 2
 
+    # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
+    assert resp[0].id == REAL_STIX_OBJS2[1].id
+    assert len(resp) == 2
+
 
 def test_filters4():
     # Assert invalid Filter cannot be created
@@ -426,6 +530,10 @@ def test_filters5():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 1
+
 
 def test_filters6():
     # Test filtering on non-common property
@@ -433,10 +541,14 @@ def test_filters6():
     assert resp[0]['id'] == STIX_OBJS2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")]))
+    assert resp[0].id == REAL_STIX_OBJS2[0].id
+    assert len(resp) == 3
+
 
 def test_filters7():
     # Test filtering on embedded property
-    stix_objects = list(STIX_OBJS2) + [{
+    obsvd_data_obj = {
         "type": "observed-data",
         "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
         "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
@@ -467,11 +579,19 @@ def test_filters7():
                 }
             }
         }
-    }]
+    }
+
+    stix_objects = list(STIX_OBJS2) + [obsvd_data_obj]
+    real_stix_objects = list(REAL_STIX_OBJS2) + [parse(obsvd_data_obj)]
+
     resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
     assert resp[0]['id'] == stix_objects[3]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
+    assert resp[0].id == real_stix_objects[3].id
+    assert len(resp) == 1
+
 
 def test_deduplicate():
     unique = deduplicate(STIX_OBJS1)
@@ -548,7 +668,7 @@ def test_composite_datasource_operations():
         Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")
     ]
 
-    cds1.filters.update(query2)
+    cds1.filters.add(query2)
 
     results = cds1.query(query1)
 

From 31fc1c369a5303c3fdcbbca03b1e155f3da2a8a6 Mon Sep 17 00:00:00 2001
From: = 
Date: Thu, 12 Apr 2018 12:03:07 -0400
Subject: [PATCH 099/150] still WIP

---
 stix2/datastore/filters.py   | 14 ++++++++++++--
 stix2/environment.py         |  2 +-
 stix2/test/test_datastore.py |  1 -
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 12f0cae..c0bed44 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -245,8 +245,13 @@ class FilterSet(object):
         """ """
         return len(self._filters)
 
-    def add(self, filters):
+    def add(self, filters=None):
         """ """
+        if not filters:
+            # so add() can be called blindly, useful for
+            # DataStore/Environment usage of filter operations
+            return
+
         if not isinstance(filters, FilterSet) and not isinstance(filters, list):
             filters = [filters]
 
@@ -256,8 +261,13 @@ class FilterSet(object):
 
         return
 
-    def remove(self, filters):
+    def remove(self, filters=None):
         """ """
+        if not filters:
+            # so remove() can be called blindly, useful for
+            # DataStore/Environemnt usage of filter ops
+            return
+
         if not isinstance(filters, FilterSet) and not isinstance(filters, list):
             filters = [filters]
 
diff --git a/stix2/environment.py b/stix2/environment.py
index a456e95..cc589ae 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -158,7 +158,7 @@ class Environment(DataStoreMixin):
     set_default_object_marking_refs.__doc__ = ObjectFactory.set_default_object_marking_refs.__doc__
 
     def add_filters(self, *args, **kwargs):
-        return self.source.filters.update(*args, **kwargs)
+        return self.source.filters.add(*args, **kwargs)
 
     def add_filter(self, *args, **kwargs):
         return self.source.filters.add(*args, **kwargs)
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index e3f19bf..a989d89 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -461,7 +461,6 @@ def test_apply_common_filters():
     # BUG: This test is brokem , weird behavior, the file obj
     # in stix_objs is being parsed into real python-stix2 obj even though
     # it never goes through parse() --> BAD <_<
-    print(stix_objs)
     resp = list(apply_common_filters(stix_objs, [filters[14]]))
     assert resp[0]["id"] == stix_objs[14]["id"]
     assert len(resp) == 1

From 9ef5b395a8b702eae048f4688e7b8d16eda61795 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 12 Apr 2018 14:20:24 -0400
Subject: [PATCH 100/150] Fix allowing custom observables and extensions

---
 stix2/base.py             | 16 +++++++++++-----
 stix2/test/test_custom.py | 17 +++++++++++++++++
 stix2/v20/observables.py  |  4 ++--
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 898f489..7ca4740 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -10,7 +10,7 @@ from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
                          ExtraPropertiesError, ImmutableError,
                          InvalidObjRefError, InvalidValueError,
                          MissingPropertiesError,
-                         MutuallyExclusivePropertiesError)
+                         MutuallyExclusivePropertiesError, ParseError)
 from .markings.utils import validate
 from .utils import NOW, find_property_index, format_datetime, get_timestamp
 from .utils import new_version as _new_version
@@ -49,7 +49,7 @@ class _STIXBase(collections.Mapping):
 
         return all_properties
 
-    def _check_property(self, prop_name, prop, kwargs):
+    def _check_property(self, prop_name, prop, kwargs, allow_custom=False):
         if prop_name not in kwargs:
             if hasattr(prop, 'default'):
                 value = prop.default()
@@ -61,6 +61,8 @@ class _STIXBase(collections.Mapping):
             try:
                 kwargs[prop_name] = prop.clean(kwargs[prop_name])
             except ValueError as exc:
+                if allow_custom and isinstance(exc, ParseError):
+                    return
                 raise InvalidValueError(self.__class__, prop_name, reason=str(exc))
 
     # interproperty constraint methods
@@ -125,7 +127,11 @@ class _STIXBase(collections.Mapping):
             raise MissingPropertiesError(cls, missing_kwargs)
 
         for prop_name, prop_metadata in cls._properties.items():
-            self._check_property(prop_name, prop_metadata, setting_kwargs)
+            try:
+                self._check_property(prop_name, prop_metadata, setting_kwargs, allow_custom)
+            except ParseError as err:
+                if not allow_custom:
+                    raise err
 
         self._inner = setting_kwargs
 
@@ -244,8 +250,8 @@ class _Observable(_STIXBase):
             if ref_type not in allowed_types:
                 raise InvalidObjRefError(self.__class__, prop_name, "object reference '%s' is of an invalid type '%s'" % (ref, ref_type))
 
-    def _check_property(self, prop_name, prop, kwargs):
-        super(_Observable, self)._check_property(prop_name, prop, kwargs)
+    def _check_property(self, prop_name, prop, kwargs, allow_custom=False):
+        super(_Observable, self)._check_property(prop_name, prop, kwargs, allow_custom)
         if prop_name not in kwargs:
             return
 
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index a50819b..d46acd4 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -617,6 +617,23 @@ def test_parse_observable_with_unregistered_custom_extension():
     assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase)
 
 
+def test_parse_observable_with_unregistered_custom_extension_dict():
+    input_dict = {
+        "type": "domain-name",
+        "value": "example.com",
+        "extensions": {
+            "x-foobar-ext": {
+                "property1": "foo",
+                "property2": 12
+            }
+        }
+    }
+
+    with pytest.raises(ValueError) as excinfo:
+        stix2.v20.observables.DomainName(**input_dict)
+    assert "Can't parse unknown extension type" in str(excinfo.value)
+
+
 def test_register_custom_object():
     # Not the way to register custom object.
     class CustomObject2(object):
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 4449527..7dc7c02 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -67,7 +67,7 @@ class ExtensionsProperty(DictionaryProperty):
                     else:
                         raise ValueError("Cannot determine extension type.")
                 else:
-                    raise ValueError("The key used in the extensions dictionary is not an extension type name")
+                    raise ParseError("Can't parse unknown extension type: {}".format(key))
         else:
             raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type)
         return dictified
@@ -936,7 +936,7 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
                 ext_class = EXT_MAP[obj['type']][name]
             except KeyError:
                 if not allow_custom:
-                    raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
+                    raise ParseError("Can't parse unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
             else:  # extension was found
                 obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
 

From 91376586d4e311f9e24c686b70de7a247772df7d Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 12 Apr 2018 16:33:08 -0400
Subject: [PATCH 101/150] Simplify allowing custom observables/extensions

---
 stix2/base.py             | 15 ++++++---------
 stix2/test/test_custom.py | 19 +------------------
 2 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 7ca4740..05afe3f 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -49,7 +49,7 @@ class _STIXBase(collections.Mapping):
 
         return all_properties
 
-    def _check_property(self, prop_name, prop, kwargs, allow_custom=False):
+    def _check_property(self, prop_name, prop, kwargs):
         if prop_name not in kwargs:
             if hasattr(prop, 'default'):
                 value = prop.default()
@@ -61,7 +61,7 @@ class _STIXBase(collections.Mapping):
             try:
                 kwargs[prop_name] = prop.clean(kwargs[prop_name])
             except ValueError as exc:
-                if allow_custom and isinstance(exc, ParseError):
+                if self.__allow_custom and isinstance(exc, ParseError):
                     return
                 raise InvalidValueError(self.__class__, prop_name, reason=str(exc))
 
@@ -99,6 +99,7 @@ class _STIXBase(collections.Mapping):
 
     def __init__(self, allow_custom=False, **kwargs):
         cls = self.__class__
+        self.__allow_custom = allow_custom
 
         # Use the same timestamp for any auto-generated datetimes
         self.__now = get_timestamp()
@@ -127,11 +128,7 @@ class _STIXBase(collections.Mapping):
             raise MissingPropertiesError(cls, missing_kwargs)
 
         for prop_name, prop_metadata in cls._properties.items():
-            try:
-                self._check_property(prop_name, prop_metadata, setting_kwargs, allow_custom)
-            except ParseError as err:
-                if not allow_custom:
-                    raise err
+            self._check_property(prop_name, prop_metadata, setting_kwargs)
 
         self._inner = setting_kwargs
 
@@ -250,8 +247,8 @@ class _Observable(_STIXBase):
             if ref_type not in allowed_types:
                 raise InvalidObjRefError(self.__class__, prop_name, "object reference '%s' is of an invalid type '%s'" % (ref, ref_type))
 
-    def _check_property(self, prop_name, prop, kwargs, allow_custom=False):
-        super(_Observable, self)._check_property(prop_name, prop, kwargs, allow_custom)
+    def _check_property(self, prop_name, prop, kwargs):
+        super(_Observable, self)._check_property(prop_name, prop, kwargs)
         if prop_name not in kwargs:
             return
 
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index d46acd4..8da2a7a 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -610,30 +610,13 @@ def test_parse_observable_with_unregistered_custom_extension():
 
     with pytest.raises(ValueError) as excinfo:
         stix2.parse_observable(input_str)
-    assert "Can't parse Unknown extension type" in str(excinfo.value)
+    assert "Can't parse unknown extension type" in str(excinfo.value)
 
     parsed_ob = stix2.parse_observable(input_str, allow_custom=True)
     assert parsed_ob['extensions']['x-foobar-ext']['property1'] == 'foo'
     assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase)
 
 
-def test_parse_observable_with_unregistered_custom_extension_dict():
-    input_dict = {
-        "type": "domain-name",
-        "value": "example.com",
-        "extensions": {
-            "x-foobar-ext": {
-                "property1": "foo",
-                "property2": 12
-            }
-        }
-    }
-
-    with pytest.raises(ValueError) as excinfo:
-        stix2.v20.observables.DomainName(**input_dict)
-    assert "Can't parse unknown extension type" in str(excinfo.value)
-
-
 def test_register_custom_object():
     # Not the way to register custom object.
     class CustomObject2(object):

From d08be151f713e020f502a5aea516b0859fd1679e Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 12 Apr 2018 21:26:48 -0400
Subject: [PATCH 102/150] Allow a ListProperty of DictionaryProperties

---
 stix2/properties.py           |  2 ++
 stix2/test/test_properties.py | 13 ++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index ca7f04c..41841b6 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -129,6 +129,8 @@ class ListProperty(Property):
                 # constructor again
                 result.append(valid)
                 continue
+            elif type(self.contained) is DictionaryProperty:
+                obj_type = dict
             else:
                 obj_type = self.contained
 
diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py
index 34edc96..16ff06a 100644
--- a/stix2/test/test_properties.py
+++ b/stix2/test/test_properties.py
@@ -1,6 +1,6 @@
 import pytest
 
-from stix2 import EmailMIMEComponent, ExtensionsProperty, TCPExt
+from stix2 import CustomObject, EmailMIMEComponent, ExtensionsProperty, TCPExt
 from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError
 from stix2.properties import (BinaryProperty, BooleanProperty,
                               DictionaryProperty, EmbeddedObjectProperty,
@@ -266,6 +266,17 @@ def test_dictionary_property_invalid(d):
     assert str(excinfo.value) == d[1]
 
 
+def test_property_list_of_dictionary():
+    @CustomObject('x-new-obj', [
+        ('property1', ListProperty(DictionaryProperty(), required=True)),
+    ])
+    class NewObj():
+        pass
+
+    test_obj = NewObj(property1=[{'foo': 'bar'}])
+    assert test_obj.property1[0]['foo'] == 'bar'
+
+
 @pytest.mark.parametrize("value", [
     {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"},
     [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')],

From a8b7be88d03f4e548b241b89cc56a69b3a3819e2 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 13 Apr 2018 11:04:07 -0400
Subject: [PATCH 103/150] Add support for pretty print case list of
 dictionaries.

---
 stix2/test/test_custom.py | 21 +++++++++++++++++++++
 stix2/utils.py            |  7 ++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index a14503f..b45670f 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -479,6 +479,27 @@ def test_custom_extension_wrong_observable_type():
     assert 'Cannot determine extension type' in excinfo.value.reason
 
 
+@pytest.mark.parametrize("data", [
+    """{
+    "keys": [
+        {
+            "test123": 123,
+            "test345": "aaaa"
+        }
+    ]
+}""",
+])
+def test_custom_extension_with_list_and_dict_properties_observable_type(data):
+    @stix2.observables.CustomExtension(stix2.UserAccount, 'some-extension', [
+        ('keys', stix2.properties.ListProperty(stix2.properties.DictionaryProperty, required=True))
+    ])
+    class SomeCustomExtension:
+        pass
+
+    example = SomeCustomExtension(keys=[{'test123': 123, 'test345': 'aaaa'}])
+    assert data == str(example)
+
+
 def test_custom_extension_invalid_observable():
     # These extensions are being applied to improperly-created Observables.
     # The Observable classes should have been created with the CustomObservable decorator.
diff --git a/stix2/utils.py b/stix2/utils.py
index 9febd78..4ef3d23 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -166,7 +166,7 @@ def get_dict(data):
 def find_property_index(obj, properties, tuple_to_find):
     """Recursively find the property in the object model, return the index
     according to the _properties OrderedDict. If it's a list look for
-    individual objects.
+    individual objects. Returns and integer indicating its location
     """
     from .base import _STIXBase
     try:
@@ -183,6 +183,11 @@ def find_property_index(obj, properties, tuple_to_find):
                                                   tuple_to_find)
                         if val is not None:
                             return val
+                    elif isinstance(item, dict):
+                        for idx, val in enumerate(sorted(item)):
+                            if (tuple_to_find[0] == val and
+                                    item.get(val) == tuple_to_find[1]):
+                                return idx
             elif isinstance(pv, dict):
                 if pv.get(tuple_to_find[0]) is not None:
                     try:

From 1a1e5e161637dc47f1d9e9792fdec5fff2c0b513 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 11:08:03 -0400
Subject: [PATCH 104/150] WIP- getting close though

---
 stix2/__init__.py            |   2 +-
 stix2/core.py                |   6 +-
 stix2/datastore/__init__.py  |   6 +-
 stix2/datastore/filters.py   |  68 ++------
 stix2/properties.py          |   4 +-
 stix2/test/test_custom.py    |   1 +
 stix2/test/test_datastore.py | 293 ++++++++++++++++++++---------------
 stix2/test/test_utils.py     |   4 +-
 stix2/utils.py               |   2 +-
 stix2/v20/common.py          |   4 +-
 stix2/v20/observables.py     |  22 ++-
 stix2/workbench.py           |  38 +++--
 12 files changed, 246 insertions(+), 204 deletions(-)

diff --git a/stix2/__init__.py b/stix2/__init__.py
index 17f2277..449be68 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -50,7 +50,7 @@ from .patterns import (AndBooleanExpression, AndObservationExpression,
                        ReferenceObjectPathComponent, RepeatQualifier,
                        StartStopQualifier, StringConstant, TimestampConstant,
                        WithinQualifier)
-from .utils import get_dict, new_version, revoke
+from .utils import new_version, revoke
 from .v20 import *  # This import will always be the latest STIX 2.X version
 from .version import __version__
 
diff --git a/stix2/core.py b/stix2/core.py
index 7de7984..0b222a9 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -9,7 +9,7 @@ import stix2
 from . import exceptions
 from .base import _STIXBase
 from .properties import IDProperty, ListProperty, Property, TypeProperty
-from .utils import get_class_hierarchy_names, get_dict
+from .utils import _get_dict, get_class_hierarchy_names
 
 
 class STIXObjectProperty(Property):
@@ -25,7 +25,7 @@ class STIXObjectProperty(Property):
                for x in get_class_hierarchy_names(value)):
             return value
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
         except ValueError:
             raise ValueError("This property may only contain a dictionary or object")
         if dictified == {}:
@@ -95,7 +95,7 @@ def parse(data, allow_custom=False, version=None):
 
     """
     # convert STIX object to dict, if not already
-    obj = get_dict(data)
+    obj = _get_dict(data)
 
     # convert dict to full python-stix2 obj
     obj = dict_to_stix2(obj, allow_custom, version)
diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index ff50735..7fdf515 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -16,7 +16,7 @@ import uuid
 
 from six import with_metaclass
 
-from stix2.datastore.filters import Filter, FilterSet, _assemble_filters
+from stix2.datastore.filters import Filter, FilterSet
 from stix2.utils import deduplicate
 
 
@@ -379,10 +379,10 @@ class DataSource(with_metaclass(ABCMeta)):
         ids.discard(obj_id)
 
         # Assemble filters
-        filter_list = _assemble_filters(filters)
+        filter_list = FilterSet(filters)
 
         for i in ids:
-            results.extend(self.query(filter_list + [Filter('id', '=', i)]))
+            results.extend(self.query([f for f in filter_list] + [Filter('id', '=', i)]))
 
         return results
 
diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index c0bed44..3d2c5e8 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -4,14 +4,15 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores
 """
 
 import collections
+from datetime import datetime
 
-from stix2.utils import STIXdatetime
+from stix2.utils import format_datetime
 
 """Supported filter operations"""
 FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
 
 """Supported filter value types"""
-FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple, STIXdatetime]
+FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
 try:
     FILTER_VALUE_TYPES.append(unicode)
 except NameError:
@@ -19,23 +20,6 @@ except NameError:
     pass
 
 
-def deduplicate_filters(filters):
-    """utility for deduplicating list of filters, this
-    is used when 'set()' cannot be used as one of the
-    filter values is a dict (or non-hashable type)
-
-    Args:
-        filters (list): a list of filters
-
-    Returns: list of unique filters
-    """
-    unique_filters = []
-    for filter_ in filters:
-        if filter_ not in unique_filters:
-            unique_filters.append(filter_)
-    return unique_filters
-
-
 def _check_filter_components(prop, op, value):
     """Check that filter meets minimum validity.
 
@@ -63,37 +47,6 @@ def _check_filter_components(prop, op, value):
     return True
 
 
-def _assemble_filters(filters1=None, filters2=None):
-    """Assemble a list of filters.
-
-    This can be used to allow certain functions to work correctly no matter if
-    the user provides a single filter or a list of them.
-
-    Args:
-        filters1 (Filter or list, optional): The single Filter or list of Filters to
-            coerce into a list of Filters.
-        filters2 (Filter or list, optional): The single Filter or list of Filters to
-            append to the list of Filters.
-
-    Returns:
-        List of Filters.
-
-    """
-    if filters1 is None:
-        filter_list = []
-    elif not isinstance(filters1, list):
-        filter_list = [filters1]
-    else:
-        filter_list = filters1
-
-    if isinstance(filters2, list):
-        filter_list.extend(filters2)
-    elif filters2 is not None:
-        filter_list.append(filters2)
-
-    return filter_list
-
-
 class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
     """STIX 2 filters that support the querying functionality of STIX 2
     DataStores and DataSources.
@@ -116,6 +69,11 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
         if isinstance(value, list):
             value = tuple(value)
 
+        if isinstance(value, datetime):
+            # if value is a datetime obj, convert to str
+            dt_str = format_datetime(value)
+            value = dt_str  # use temp variable to avoid deepcopy operation
+
         _check_filter_components(prop, op, value)
 
         self = super(Filter, cls).__new__(cls, prop, op, value)
@@ -131,6 +89,12 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
             True if property matches the filter,
             False otherwise.
         """
+        if isinstance(stix_obj_property, datetime):
+            # if a datetime obj, convert to str before comparison
+            # NOTE: this check seems like it should be done upstream
+            # but will put here for now
+            stix_obj_property = format_datetime(stix_obj_property)
+
         if self.op == "=":
             return stix_obj_property == self.value
         elif self.op == "!=":
@@ -252,7 +216,7 @@ class FilterSet(object):
             # DataStore/Environment usage of filter operations
             return
 
-        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+        if not isinstance(filters, (FilterSet, list)):
             filters = [filters]
 
         for f in filters:
@@ -268,7 +232,7 @@ class FilterSet(object):
             # DataStore/Environemnt usage of filter ops
             return
 
-        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+        if not isinstance(filters, (FilterSet, list)):
             filters = [filters]
 
         for f in filters:
diff --git a/stix2/properties.py b/stix2/properties.py
index ca7f04c..a650d55 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -12,7 +12,7 @@ from stix2patterns.validator import run_validator
 
 from .base import _STIXBase
 from .exceptions import DictionaryKeyError
-from .utils import get_dict, parse_into_datetime
+from .utils import _get_dict, parse_into_datetime
 
 
 class Property(object):
@@ -232,7 +232,7 @@ class DictionaryProperty(Property):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
         if dictified == {}:
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index a14503f..417be00 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -80,6 +80,7 @@ def test_identity_custom_property_allowed():
 def test_parse_identity_custom_property(data):
     with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo:
         identity = stix2.parse(data)
+    assert str(excinfo.value.cls) == str(stix2.Identity)
     assert excinfo.value.cls == stix2.Identity
     assert excinfo.value.properties == ['foo']
     assert "Unexpected properties for" in str(excinfo.value)
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index a989d89..9ffa992 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -5,7 +5,7 @@ from stix2 import Filter, MemorySink, MemorySource
 from stix2.core import parse
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
                              make_id, taxii)
-from stix2.datastore.filters import _assemble_filters, apply_common_filters
+from stix2.datastore.filters import apply_common_filters
 from stix2.utils import deduplicate, parse_into_datetime
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
@@ -21,6 +21,108 @@ def collection():
     return Collection(COLLECTION_URL, MockTAXIIClient())
 
 
+stix_objs = [
+    {
+        "created": "2017-01-27T13:49:53.997Z",
+        "description": "\n\nTITLE:\n\tPoison Ivy",
+        "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "labels": [
+            "remote-access-trojan"
+        ],
+        "modified": "2017-01-27T13:49:53.997Z",
+        "name": "Poison Ivy",
+        "type": "malware"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "labels": [
+            "file-hash-watchlist"
+        ],
+        "modified": "2014-05-08T09:00:00.000Z",
+        "name": "File hash for Poison Ivy variant",
+        "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
+        "type": "indicator",
+        "valid_from": "2014-05-08T09:00:00.000000Z"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "granular_markings": [
+            {
+                "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
+                "selectors": [
+                    "relationship_type"
+                ]
+            }
+        ],
+        "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
+        "modified": "2014-05-08T09:00:00.000Z",
+        "object_marking_refs": [
+            "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
+        ],
+        "relationship_type": "indicates",
+        "revoked": True,
+        "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "type": "relationship"
+    },
+    {
+        "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
+        "created": "2016-02-14T00:00:00.000Z",
+        "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
+        "modified": "2016-02-14T00:00:00.000Z",
+        "type": "vulnerability",
+        "name": "CVE-2014-0160",
+        "description": "The (1) TLS...",
+        "external_references": [
+            {
+                "source_name": "cve",
+                "external_id": "CVE-2014-0160"
+            }
+        ],
+        "labels": ["heartbleed", "has-logo"]
+    },
+    {
+        "type": "observed-data",
+        "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
+        "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "file",
+                "name": "HAL 9000.exe"
+            }
+        }
+
+    }
+]
+
+# same as above objects but converted to real Python STIX2 objects
+# to test filters against true Python STIX2 objects
+real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs]
+
+filters = [
+    Filter("type", "!=", "relationship"),
+    Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
+    Filter("labels", "in", "remote-access-trojan"),
+    Filter("created", ">", "2015-01-01T01:00:00.000Z"),
+    Filter("revoked", "=", True),
+    Filter("revoked", "!=", True),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "relationship_type"),
+    Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
+    Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
+    Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "description"),
+    Filter("external_references.source_name", "=", "CVE"),
+    Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}})
+]
+
 IND1 = {
     "created": "2017-01-27T13:49:53.935Z",
     "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
@@ -240,111 +342,7 @@ def test_filter_type_underscore_check():
     assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
 
 
-def test_apply_common_filters():
-    stix_objs = [
-        {
-            "created": "2017-01-27T13:49:53.997Z",
-            "description": "\n\nTITLE:\n\tPoison Ivy",
-            "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "labels": [
-                "remote-access-trojan"
-            ],
-            "modified": "2017-01-27T13:49:53.997Z",
-            "name": "Poison Ivy",
-            "type": "malware"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "labels": [
-                "file-hash-watchlist"
-            ],
-            "modified": "2014-05-08T09:00:00.000Z",
-            "name": "File hash for Poison Ivy variant",
-            "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
-            "type": "indicator",
-            "valid_from": "2014-05-08T09:00:00.000000Z"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "granular_markings": [
-                {
-                    "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
-                    "selectors": [
-                        "relationship_type"
-                    ]
-                }
-            ],
-            "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
-            "modified": "2014-05-08T09:00:00.000Z",
-            "object_marking_refs": [
-                "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
-            ],
-            "relationship_type": "indicates",
-            "revoked": True,
-            "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "type": "relationship"
-        },
-        {
-            "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
-            "created": "2016-02-14T00:00:00.000Z",
-            "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
-            "modified": "2016-02-14T00:00:00.000Z",
-            "type": "vulnerability",
-            "name": "CVE-2014-0160",
-            "description": "The (1) TLS...",
-            "external_references": [
-                {
-                    "source_name": "cve",
-                    "external_id": "CVE-2014-0160"
-                }
-            ],
-            "labels": ["heartbleed", "has-logo"]
-        },
-        {
-            "type": "observed-data",
-            "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
-            "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
-            "created": "2016-04-06T19:58:16.000Z",
-            "modified": "2016-04-06T19:58:16.000Z",
-            "first_observed": "2015-12-21T19:00:00Z",
-            "last_observed": "2015-12-21T19:00:00Z",
-            "number_observed": 1,
-            "objects": {
-                "0": {
-                    "type": "file",
-                    "name": "HAL 9000.exe"
-                }
-            }
-
-        }
-    ]
-
-    # same as above objects but converted to real Python STIX2 objects
-    # to test filters against true Python STIX2 objects
-    print(stix_objs)
-    real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs]
-    print("after\n\n")
-    print(stix_objs)
-    filters = [
-        Filter("type", "!=", "relationship"),
-        Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
-        Filter("labels", "in", "remote-access-trojan"),
-        Filter("created", ">", "2015-01-01T01:00:00.000Z"),
-        Filter("revoked", "=", True),
-        Filter("revoked", "!=", True),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "relationship_type"),
-        Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
-        Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
-        Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "description"),
-        Filter("external_references.source_name", "=", "CVE"),
-        Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}})
-    ]
-
+def test_apply_common_filters0():
     # "Return any object whose type is not relationship"
     resp = list(apply_common_filters(stix_objs, [filters[0]]))
     ids = [r['id'] for r in resp]
@@ -360,6 +358,8 @@ def test_apply_common_filters():
     assert real_stix_objs[3].id in ids
     assert len(ids) == 4
 
+
+def test_apply_common_filters1():
     # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
     resp = list(apply_common_filters(stix_objs, [filters[1]]))
     assert resp[0]['id'] == stix_objs[2]['id']
@@ -369,6 +369,8 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[2].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters2():
     # "Return any object that contains remote-access-trojan in labels"
     resp = list(apply_common_filters(stix_objs, [filters[2]]))
     assert resp[0]['id'] == stix_objs[0]['id']
@@ -378,11 +380,19 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[0].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters3():
     # "Return any object created after 2015-01-01T01:00:00.000Z"
     resp = list(apply_common_filters(stix_objs, [filters[3]]))
     assert resp[0]['id'] == stix_objs[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[3]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 3
+
+
+def test_apply_common_filters4():
     # "Return any revoked object"
     resp = list(apply_common_filters(stix_objs, [filters[4]]))
     assert resp[0]['id'] == stix_objs[2]['id']
@@ -392,6 +402,8 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[2].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters5():
     # "Return any object whose not revoked"
     # Note that if 'revoked' property is not present in object.
     # Currently we can't use such an expression to filter for... :(
@@ -401,6 +413,8 @@ def test_apply_common_filters():
     resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
     assert len(resp) == 0
 
+
+def test_apply_common_filters6():
     # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
     resp = list(apply_common_filters(stix_objs, [filters[6]]))
     assert resp[0]['id'] == stix_objs[2]['id']
@@ -410,6 +424,8 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[2].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters7():
     # "Return any object that contains relationship_type in their selectors AND
     # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
     resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
@@ -420,6 +436,8 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[2].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters8():
     # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
     resp = list(apply_common_filters(stix_objs, [filters[9]]))
     assert resp[0]['id'] == stix_objs[3]['id']
@@ -429,6 +447,8 @@ def test_apply_common_filters():
     assert resp[0].id == real_stix_objs[3].id
     assert len(resp) == 1
 
+
+def test_apply_common_filters9():
     # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
     resp = list(apply_common_filters(stix_objs, [filters[10]]))
     assert len(resp) == 1
@@ -436,6 +456,8 @@ def test_apply_common_filters():
     resp = list(apply_common_filters(real_stix_objs, [filters[10]]))
     assert len(resp) == 1
 
+
+def test_apply_common_filters10():
     # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
     resp = list(apply_common_filters(stix_objs, [filters[11]]))
     assert len(resp) == 0
@@ -443,6 +465,8 @@ def test_apply_common_filters():
     resp = list(apply_common_filters(real_stix_objs, [filters[11]]))
     assert len(resp) == 0
 
+
+def test_apply_common_filters11():
     # "Return any object that contains description in its selectors" (None)
     resp = list(apply_common_filters(stix_objs, [filters[12]]))
     assert len(resp) == 0
@@ -450,6 +474,8 @@ def test_apply_common_filters():
     resp = list(apply_common_filters(real_stix_objs, [filters[12]]))
     assert len(resp) == 0
 
+
+def test_apply_common_filters12():
     # "Return any object that matches CVE in source_name" (None, case sensitive)
     resp = list(apply_common_filters(stix_objs, [filters[13]]))
     assert len(resp) == 0
@@ -457,19 +483,51 @@ def test_apply_common_filters():
     resp = list(apply_common_filters(real_stix_objs, [filters[13]]))
     assert len(resp) == 0
 
+
+def test_apply_common_filters13():
     # Return any object that matches file object in "objects"
-    # BUG: This test is brokem , weird behavior, the file obj
-    # in stix_objs is being parsed into real python-stix2 obj even though
-    # it never goes through parse() --> BAD <_<
     resp = list(apply_common_filters(stix_objs, [filters[14]]))
-    assert resp[0]["id"] == stix_objs[14]["id"]
+    assert resp[0]["id"] == stix_objs[4]["id"]
     assert len(resp) == 1
 
     resp = list(apply_common_filters(real_stix_objs, [filters[14]]))
-    assert resp[0].id == real_stix_objs[14].id
+    assert resp[0].id == real_stix_objs[4].id
     assert len(resp) == 1
 
 
+def test_datetime_filter_behavior():
+    """if a filter is initialized with its value being a datetime object
+    OR the STIX object property being filtered on is a datetime object, all
+    resulting comparisons executed are done on the string representations
+    of the datetime objects, as the Filter functionality will convert
+    all datetime objects to there string forms using format_datetim()
+
+    This test makes sure all datetime comparisons are carried out correctly
+    """
+    filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
+    filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
+
+    # compare datetime string to filter w/ datetime obj
+    resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ datetime obj
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime string to filter w/ str
+    resp = list(apply_common_filters(stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ str
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+
 def test_filters0():
     # "Return any object modified before 2017-01-28T13:49:53.935Z"
     resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
@@ -510,7 +568,9 @@ def test_filters3():
     assert len(resp) == 2
 
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(REAL_STIX_OBJS2, [Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
+    fv = Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))
+    print(fv)
+    resp = list(apply_common_filters(REAL_STIX_OBJS2, [fv]))
     assert resp[0].id == REAL_STIX_OBJS2[1].id
     assert len(resp) == 2
 
@@ -592,15 +652,6 @@ def test_filters7():
     assert len(resp) == 1
 
 
-def test_assemble_filters():
-    filter1 = Filter("name", "=", "Malicious site hosting downloader")
-    filter2 = Filter("modified", ">", "2017-01-28T13:49:53.935Z")
-    result = _assemble_filters(filter1, filter2)
-    assert len(result) == 2
-    assert result[0].property == 'name'
-    assert result[1].property == 'modified'
-
-
 def test_deduplicate():
     unique = deduplicate(STIX_OBJS1)
 
diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py
index cbe5b0f..84b2476 100644
--- a/stix2/test/test_utils.py
+++ b/stix2/test/test_utils.py
@@ -62,7 +62,7 @@ def test_parse_datetime_invalid(ts):
     [("a", 1,)],
 ])
 def test_get_dict(data):
-    assert stix2.utils.get_dict(data)
+    assert stix2.utils._get_dict(data)
 
 
 @pytest.mark.parametrize('data', [
@@ -73,7 +73,7 @@ def test_get_dict(data):
 ])
 def test_get_dict_invalid(data):
     with pytest.raises(ValueError):
-        stix2.utils.get_dict(data)
+        stix2.utils._get_dict(data)
 
 
 @pytest.mark.parametrize('stix_id, typ', [
diff --git a/stix2/utils.py b/stix2/utils.py
index 9febd78..fdb7679 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -140,7 +140,7 @@ def parse_into_datetime(value, precision=None):
     return STIXdatetime(ts, precision=precision)
 
 
-def get_dict(data):
+def _get_dict(data):
     """Return data as a dictionary.
 
     Input can be a dictionary, string, or file-like object.
diff --git a/stix2/v20/common.py b/stix2/v20/common.py
index e915df6..0bba3b1 100644
--- a/stix2/v20/common.py
+++ b/stix2/v20/common.py
@@ -7,7 +7,7 @@ from ..markings import _MarkingsMixin
 from ..properties import (HashesProperty, IDProperty, ListProperty, Property,
                           ReferenceProperty, SelectorProperty, StringProperty,
                           TimestampProperty, TypeProperty)
-from ..utils import NOW, get_dict
+from ..utils import NOW, _get_dict
 
 
 class ExternalReference(_STIXBase):
@@ -125,7 +125,7 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin):
                 raise ValueError("definition_type must be a valid marking type")
 
             if not isinstance(kwargs['definition'], marking_type):
-                defn = get_dict(kwargs['definition'])
+                defn = _get_dict(kwargs['definition'])
                 kwargs['definition'] = marking_type(**defn)
 
         super(MarkingDefinition, self).__init__(**kwargs)
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 39a8f19..75887ee 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -6,6 +6,7 @@ Observable and do not have a ``_type`` attribute.
 """
 
 from collections import OrderedDict
+import copy
 
 from ..base import _Extension, _Observable, _STIXBase
 from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
@@ -15,7 +16,7 @@ from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           HashesProperty, HexProperty, IntegerProperty,
                           ListProperty, ObjectReferenceProperty, Property,
                           StringProperty, TimestampProperty, TypeProperty)
-from ..utils import get_dict
+from ..utils import _get_dict
 
 
 class ObservableProperty(Property):
@@ -24,7 +25,11 @@ class ObservableProperty(Property):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
+            # get deep copy since we are going modify the dict and might
+            # modify the original dict as _get_dict() does not return new
+            # dict when passed a dict
+            dictified = copy.deepcopy(dictified)
         except ValueError:
             raise ValueError("The observable property must contain a dictionary")
         if dictified == {}:
@@ -49,7 +54,11 @@ class ExtensionsProperty(DictionaryProperty):
 
     def clean(self, value):
         try:
-            dictified = get_dict(value)
+            dictified = _get_dict(value)
+            # get deep copy since we are going modify the dict and might
+            # modify the original dict as _get_dict() does not return new
+            # dict when passed a dict
+            dictified = copy.deepcopy(dictified)
         except ValueError:
             raise ValueError("The extensions property must contain a dictionary")
         if dictified == {}:
@@ -915,7 +924,12 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
         An instantiated Python STIX Cyber Observable object.
     """
 
-    obj = get_dict(data)
+    obj = _get_dict(data)
+    # get deep copy since we are going modify the dict and might
+    # modify the original dict as _get_dict() does not return new
+    # dict when passed a dict
+    obj = copy.deepcopy(obj)
+
     obj['_valid_refs'] = _valid_refs or []
 
     if 'type' not in obj:
diff --git a/stix2/workbench.py b/stix2/workbench.py
index b47968e..6654555 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -48,7 +48,7 @@ from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # n
                WindowsPEOptionalHeaderType, WindowsPESection,
                WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
                WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
-from .datastore.filters import _assemble_filters
+from .datastore.filters import FilterSet
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -156,7 +156,8 @@ def attack_patterns(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'attack-pattern')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'attack-pattern'))
     return query(filter_list)
 
 
@@ -168,7 +169,8 @@ def campaigns(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'campaign')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'campaign'))
     return query(filter_list)
 
 
@@ -180,7 +182,8 @@ def courses_of_action(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'course-of-action')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'course-of-action'))
     return query(filter_list)
 
 
@@ -192,7 +195,8 @@ def identities(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'identity')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'identity'))
     return query(filter_list)
 
 
@@ -204,7 +208,8 @@ def indicators(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'indicator')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'indicator'))
     return query(filter_list)
 
 
@@ -216,7 +221,8 @@ def intrusion_sets(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'intrusion-set')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'intrusion-set'))
     return query(filter_list)
 
 
@@ -228,7 +234,8 @@ def malware(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'malware')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'malware'))
     return query(filter_list)
 
 
@@ -240,7 +247,8 @@ def observed_data(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'observed-data')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'observed-data'))
     return query(filter_list)
 
 
@@ -252,7 +260,8 @@ def reports(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'report')])
+    filter_list = FilterSet(filters)
+    filter_list.add([Filter('type', '=', 'report')])
     return query(filter_list)
 
 
@@ -264,7 +273,8 @@ def threat_actors(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'threat-actor')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'threat-actor'))
     return query(filter_list)
 
 
@@ -276,7 +286,8 @@ def tools(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'tool')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'tool'))
     return query(filter_list)
 
 
@@ -288,5 +299,6 @@ def vulnerabilities(filters=None):
             the query.
 
     """
-    filter_list = _assemble_filters(filters, [Filter('type', '=', 'vulnerability')])
+    filter_list = FilterSet(filters)
+    filter_list.add(Filter('type', '=', 'vulnerability'))
     return query(filter_list)

From fc6a33b23e2a7131249b89eb5a1b8394d328ca26 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 13 Apr 2018 11:18:56 -0400
Subject: [PATCH 105/150] Disallow missing 'type' property with allow_custom

There was a bug where if you allowed custom content the library would parse an
object without the required 'type' property.
---
 stix2/base.py             | 10 +++++-----
 stix2/exceptions.py       |  7 +++++++
 stix2/test/test_custom.py | 12 +++++++++++-
 stix2/v20/observables.py  | 13 +++++++------
 4 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 05afe3f..3219007 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -6,11 +6,11 @@ import datetime as dt
 
 import simplejson as json
 
-from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
-                         ExtraPropertiesError, ImmutableError,
-                         InvalidObjRefError, InvalidValueError,
+from .exceptions import (AtLeastOnePropertyError, CustomContentError,
+                         DependentPropertiesError, ExtraPropertiesError,
+                         ImmutableError, InvalidObjRefError, InvalidValueError,
                          MissingPropertiesError,
-                         MutuallyExclusivePropertiesError, ParseError)
+                         MutuallyExclusivePropertiesError)
 from .markings.utils import validate
 from .utils import NOW, find_property_index, format_datetime, get_timestamp
 from .utils import new_version as _new_version
@@ -61,7 +61,7 @@ class _STIXBase(collections.Mapping):
             try:
                 kwargs[prop_name] = prop.clean(kwargs[prop_name])
             except ValueError as exc:
-                if self.__allow_custom and isinstance(exc, ParseError):
+                if self.__allow_custom and isinstance(exc, CustomContentError):
                     return
                 raise InvalidValueError(self.__class__, prop_name, reason=str(exc))
 
diff --git a/stix2/exceptions.py b/stix2/exceptions.py
index 841a8e9..79c5a81 100644
--- a/stix2/exceptions.py
+++ b/stix2/exceptions.py
@@ -163,6 +163,13 @@ class ParseError(STIXError, ValueError):
         super(ParseError, self).__init__(msg)
 
 
+class CustomContentError(STIXError, ValueError):
+    """Custom STIX Content (SDO, Observable, Extension, etc.) detected."""
+
+    def __init__(self, msg):
+        super(CustomContentError, self).__init__(msg)
+
+
 class InvalidSelectorError(STIXError, AssertionError):
     """Granular Marking selector violation. The selector must resolve into an existing STIX object property."""
 
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 8da2a7a..d655e5d 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -373,7 +373,7 @@ def test_parse_unregistered_custom_observable_object():
         "property1": "something"
     }"""
 
-    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+    with pytest.raises(stix2.exceptions.CustomContentError) as excinfo:
         stix2.parse_observable(nt_string)
     assert "Can't parse unknown observable type" in str(excinfo.value)
 
@@ -384,6 +384,16 @@ def test_parse_unregistered_custom_observable_object():
     assert not isinstance(parsed_custom, stix2.core._STIXBase)
 
 
+def test_parse_unregistered_custom_observable_object_with_no_type():
+    nt_string = """{
+        "property1": "something"
+    }"""
+
+    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+        stix2.parse_observable(nt_string, allow_custom=True)
+    assert "Can't parse observable with no 'type' property" in str(excinfo.value)
+
+
 def test_parse_observed_data_with_custom_observable():
     input_str = """{
         "type": "observed-data",
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 7dc7c02..b5ffe49 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -8,8 +8,8 @@ Observable and do not have a ``_type`` attribute.
 from collections import OrderedDict
 
 from ..base import _Extension, _Observable, _STIXBase
-from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
-                          ParseError)
+from ..exceptions import (AtLeastOnePropertyError, CustomContentError,
+                          DependentPropertiesError, ParseError)
 from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           EmbeddedObjectProperty, EnumProperty, FloatProperty,
                           HashesProperty, HexProperty, IntegerProperty,
@@ -67,7 +67,7 @@ class ExtensionsProperty(DictionaryProperty):
                     else:
                         raise ValueError("Cannot determine extension type.")
                 else:
-                    raise ParseError("Can't parse unknown extension type: {}".format(key))
+                    raise CustomContentError("Can't parse unknown extension type: {}".format(key))
         else:
             raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type)
         return dictified
@@ -927,8 +927,8 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
             # flag allows for unknown custom objects too, but will not
             # be parsed into STIX observable object, just returned as is
             return obj
-        raise ParseError("Can't parse unknown observable type '%s'! For custom observables, "
-                         "use the CustomObservable decorator." % obj['type'])
+        raise CustomContentError("Can't parse unknown observable type '%s'! For custom observables, "
+                                 "use the CustomObservable decorator." % obj['type'])
 
     if 'extensions' in obj and obj['type'] in EXT_MAP:
         for name, ext in obj['extensions'].items():
@@ -936,7 +936,8 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
                 ext_class = EXT_MAP[obj['type']][name]
             except KeyError:
                 if not allow_custom:
-                    raise ParseError("Can't parse unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
+                    raise CustomContentError("Can't parse unknown extension type '%s'"
+                                             "for observable type '%s'!" % (name, obj['type']))
             else:  # extension was found
                 obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
 

From 61e091baf34d029aebf8b0bf3c88449dd87329e7 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 12:25:12 -0400
Subject: [PATCH 106/150] added FilterSet class for internal use; modified
 certain parsing processes to make deepcopies or suppled values(dicts) so as
 to taint original user passed data; added Filter logic to handle datetime
 objects; added/adjusted tests accordingly

---
 stix2/datastore/filters.py   | 37 ++++++++++++++++++++++++++++++------
 stix2/test/test_datastore.py |  1 -
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 3d2c5e8..21a156e 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -192,25 +192,43 @@ def _check_filter(filter_, stix_obj):
 
 
 class FilterSet(object):
-    """ """
+    """Internal STIX2 class to facilitate the grouping of Filters
+    into sets. The primary motivation for this class came from the problem
+    that Filters that had a dict as a value could not be added to a Python
+    set as dicts are not hashable. Thus this class provides set functionality
+    but internally stores filters in a list.
+     """
 
     def __init__(self, filters=None):
-        """ """
+        """
+        Args:
+            filters: see FilterSet.add()
+        """
         self._filters = []
         if filters:
             self.add(filters)
 
     def __iter__(self):
-        """ """
+        """provide iteration functionality of FilterSet"""
         for f in self._filters:
             yield f
 
     def __len__(self):
-        """ """
+        """provide built-in len() utility of FilterSet"""
         return len(self._filters)
 
     def add(self, filters=None):
-        """ """
+        """add a Filter, FilterSet, or list of Filters to the FilterSet
+
+        Operates like set, only adding unique stix2.Filters to the FilterSet
+
+        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
+        as it allows for blind calls (very useful in DataStore)
+
+        Args:
+            filters: stix2.Filter OR list of stix2.Filter OR stix2.FilterSet
+
+        """
         if not filters:
             # so add() can be called blindly, useful for
             # DataStore/Environment usage of filter operations
@@ -226,7 +244,14 @@ class FilterSet(object):
         return
 
     def remove(self, filters=None):
-        """ """
+        """remove a Filter, list of Filters, or FilterSet from the FilterSet
+
+        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
+        as it allows for blind calls (very useful in DataStore)
+
+        Args:
+            filters: stix2.Filter OR list of stix2.Filter or stix2.FilterSet
+        """
         if not filters:
             # so remove() can be called blindly, useful for
             # DataStore/Environemnt usage of filter ops
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 9ffa992..772c72e 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -569,7 +569,6 @@ def test_filters3():
 
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
     fv = Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))
-    print(fv)
     resp = list(apply_common_filters(REAL_STIX_OBJS2, [fv]))
     assert resp[0].id == REAL_STIX_OBJS2[1].id
     assert len(resp) == 2

From eba1844535a8140751dfee6dfcdc81835ba94830 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 14:01:33 -0400
Subject: [PATCH 107/150] tweak to filter property checking to make sure
 original object property is not altered; added tests for this as well

---
 stix2/datastore/filters.py   |  5 +++--
 stix2/test/test_datastore.py | 10 +++++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 21a156e..c260dcc 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -90,10 +90,11 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
             False otherwise.
         """
         if isinstance(stix_obj_property, datetime):
-            # if a datetime obj, convert to str before comparison
+            # if a datetime obj, use str format before comparison
             # NOTE: this check seems like it should be done upstream
             # but will put here for now
-            stix_obj_property = format_datetime(stix_obj_property)
+            tmp = format_datetime(stix_obj_property)
+            stix_obj_property = tmp  # use tmp variable to avoid deepcopy op
 
         if self.op == "=":
             return stix_obj_property == self.value
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 772c72e..7ee4877 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -6,7 +6,7 @@ from stix2.core import parse
 from stix2.datastore import (CompositeDataSource, DataSink, DataSource,
                              make_id, taxii)
 from stix2.datastore.filters import apply_common_filters
-from stix2.utils import deduplicate, parse_into_datetime
+from stix2.utils import STIXdatetime, deduplicate, parse_into_datetime
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
 
@@ -489,6 +489,9 @@ def test_apply_common_filters13():
     resp = list(apply_common_filters(stix_objs, [filters[14]]))
     assert resp[0]["id"] == stix_objs[4]["id"]
     assert len(resp) == 1
+    # important additional check to make sure original File dict was
+    # not converted to File object. (this was a deep bug found)
+    assert isinstance(resp[0]["objects"]["0"], dict)
 
     resp = list(apply_common_filters(real_stix_objs, [filters[14]]))
     assert resp[0].id == real_stix_objs[4].id
@@ -507,6 +510,9 @@ def test_datetime_filter_behavior():
     filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
     filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
 
+    # check taht filter value is converted from datetime to str
+    assert isinstance(filter_with_dt_obj.value, str)
+
     # compare datetime string to filter w/ datetime obj
     resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj]))
     assert len(resp) == 1
@@ -516,6 +522,7 @@ def test_datetime_filter_behavior():
     resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
     assert len(resp) == 1
     assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
 
     # compare datetime string to filter w/ str
     resp = list(apply_common_filters(stix_objs, [filter_with_str]))
@@ -526,6 +533,7 @@ def test_datetime_filter_behavior():
     resp = list(apply_common_filters(real_stix_objs, [filter_with_str]))
     assert len(resp) == 1
     assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
 
 
 def test_filters0():

From a614a78e22b95d1b2f46df2d8432ab8505a68f06 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 14:21:44 -0400
Subject: [PATCH 108/150] rm testing lines

---
 stix2/test/test_custom.py | 1 -
 stix2/workbench.py        | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 5829e29..f9bb875 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -80,7 +80,6 @@ def test_identity_custom_property_allowed():
 def test_parse_identity_custom_property(data):
     with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo:
         identity = stix2.parse(data)
-    assert str(excinfo.value.cls) == str(stix2.Identity)
     assert excinfo.value.cls == stix2.Identity
     assert excinfo.value.properties == ['foo']
     assert "Unexpected properties for" in str(excinfo.value)
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 6654555..3697c63 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -261,7 +261,7 @@ def reports(filters=None):
 
     """
     filter_list = FilterSet(filters)
-    filter_list.add([Filter('type', '=', 'report')])
+    filter_list.add(Filter('type', '=', 'report'))
     return query(filter_list)
 
 

From a475fc6dbd422dc97ece318e2ba089370e03eead Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 13 Apr 2018 14:52:00 -0400
Subject: [PATCH 109/150] Disallow invalid type names in custom classes

---
 stix2/test/test_custom.py | 54 +++++++++++++++++++++++++++++++++++++++
 stix2/utils.py            |  2 ++
 stix2/v20/observables.py  | 15 ++++++++++-
 stix2/v20/sdo.py          | 10 +++++++-
 4 files changed, 79 insertions(+), 2 deletions(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 56e578f..0e004a8 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -197,6 +197,24 @@ def test_custom_object_no_init_2():
     assert no2.property1 == 'something'
 
 
+def test_custom_object_invalid_type_name():
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.sdo.CustomObject('x', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ])
+        class NewObj(object):
+            pass  # pragma: no cover
+    assert "Invalid type name 'x': " in str(excinfo.value)
+
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.sdo.CustomObject('x_new_object', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ])
+        class NewObj2(object):
+            pass  # pragma: no cover
+    assert "Invalid type name 'x_new_object':" in str(excinfo.value)
+
+
 def test_parse_custom_object_type():
     nt_string = """{
         "type": "x-new-type",
@@ -295,6 +313,24 @@ def test_custom_observable_object_no_init_2():
     assert no2.property1 == 'something'
 
 
+def test_custom_observable_object_invalid_type_name():
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.observables.CustomObservable('x', [
+            ('property1', stix2.properties.StringProperty()),
+        ])
+        class NewObs(object):
+            pass  # pragma: no cover
+    assert "Invalid observable type name 'x':" in str(excinfo.value)
+
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.observables.CustomObservable('x_new_obs', [
+            ('property1', stix2.properties.StringProperty()),
+        ])
+        class NewObs2(object):
+            pass  # pragma: no cover
+    assert "Invalid observable type name 'x_new_obs':" in str(excinfo.value)
+
+
 def test_custom_observable_object_invalid_ref_property():
     with pytest.raises(ValueError) as excinfo:
         @stix2.observables.CustomObservable('x-new-obs', [
@@ -573,6 +609,24 @@ def test_custom_extension_invalid_observable():
     assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value)
 
 
+def test_custom_extension_invalid_type_name():
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.observables.CustomExtension(stix2.File, 'x', {
+            'property1': stix2.properties.StringProperty(required=True),
+        })
+        class FooExtension():
+            pass  # pragma: no cover
+    assert "Invalid extension type name 'x':" in str(excinfo.value)
+
+    with pytest.raises(ValueError) as excinfo:
+        @stix2.observables.CustomExtension(stix2.File, 'x_new_ext', {
+            'property1': stix2.properties.StringProperty(required=True),
+        })
+        class BlaExtension():
+            pass  # pragma: no cover
+    assert "Invalid extension type name 'x_new_ext':" in str(excinfo.value)
+
+
 def test_custom_extension_no_properties():
     with pytest.raises(ValueError) as excinfo:
         @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', None)
diff --git a/stix2/utils.py b/stix2/utils.py
index 4ef3d23..e693d08 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -18,6 +18,8 @@ NOW = object()
 # STIX object properties that cannot be modified
 STIX_UNMOD_PROPERTIES = ["created", "created_by_ref", "id", "type"]
 
+TYPE_REGEX = r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$'
+
 
 class STIXdatetime(dt.datetime):
     def __new__(cls, *args, **kwargs):
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index f6bac2b..773611e 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -6,6 +6,7 @@ Observable and do not have a ``_type`` attribute.
 """
 
 from collections import OrderedDict
+import re
 
 from ..base import _Extension, _Observable, _STIXBase
 from ..exceptions import (AtLeastOnePropertyError, CustomContentError,
@@ -15,7 +16,7 @@ from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           HashesProperty, HexProperty, IntegerProperty,
                           ListProperty, ObjectReferenceProperty, Property,
                           StringProperty, TimestampProperty, TypeProperty)
-from ..utils import get_dict
+from ..utils import TYPE_REGEX, get_dict
 
 
 class ObservableProperty(Property):
@@ -967,6 +968,12 @@ def CustomObservable(type='x-custom-observable', properties=None):
 
         class _Custom(cls, _Observable):
 
+            if not re.match(TYPE_REGEX, type):
+                raise ValueError("Invalid observable type name '%s': must only contain the "
+                                 "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type)
+            elif len(type) < 3 or len(type) > 250:
+                raise ValueError("Invalid observable type name '%s': must be between 3 and 250 characters." % type)
+
             _type = type
             _properties = OrderedDict()
             _properties.update([
@@ -1040,6 +1047,12 @@ def CustomExtension(observable=None, type='x-custom-observable', properties=None
 
         class _Custom(cls, _Extension):
 
+            if not re.match(TYPE_REGEX, type):
+                raise ValueError("Invalid extension type name '%s': must only contain the "
+                                 "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type)
+            elif len(type) < 3 or len(type) > 250:
+                raise ValueError("Invalid extension type name '%s': must be between 3 and 250 characters." % type)
+
             _type = type
             _properties = OrderedDict()
 
diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index 060b9f0..ff024f5 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -2,6 +2,7 @@
 """
 
 from collections import OrderedDict
+import re
 
 import stix2
 
@@ -10,7 +11,7 @@ from ..markings import _MarkingsMixin
 from ..properties import (BooleanProperty, IDProperty, IntegerProperty,
                           ListProperty, PatternProperty, ReferenceProperty,
                           StringProperty, TimestampProperty, TypeProperty)
-from ..utils import NOW
+from ..utils import NOW, TYPE_REGEX
 from .common import ExternalReference, GranularMarking, KillChainPhase
 from .observables import ObservableProperty
 
@@ -357,6 +358,13 @@ def CustomObject(type='x-custom-type', properties=None):
     def custom_builder(cls):
 
         class _Custom(cls, STIXDomainObject):
+
+            if not re.match(TYPE_REGEX, type):
+                raise ValueError("Invalid type name '%s': must only contain the "
+                                 "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type)
+            elif len(type) < 3 or len(type) > 250:
+                raise ValueError("Invalid type name '%s': must be between 3 and 250 characters." % type)
+
             _type = type
             _properties = OrderedDict()
             _properties.update([

From 47347111372d31af5050bde387fd8ba9e9d6d62c Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 15:07:49 -0400
Subject: [PATCH 110/150] corresponding doc update

---
 docs/guide/datastore.ipynb | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index 7d40930..c3980a0 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -4,6 +4,7 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
+    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -24,6 +25,7 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
+    "collapsed": true,
     "nbsphinx": "hidden"
    },
    "outputs": [],
@@ -383,8 +385,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "import sys\n",
@@ -415,7 +419,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -428,11 +432,11 @@
     "fs.source.filters.add(f)\n",
     "\n",
     "# attach multiple filters to FileSystemStore\n",
-    "fs.source.filters.update([f1,f2])\n",
+    "fs.source.filters.add([f1,f2])\n",
     "\n",
     "# can also attach filters to a Source\n",
     "# attach multiple filters to FileSystemSource\n",
-    "fs_source.filters.update([f3, f4])\n",
+    "fs_source.filters.add([f3, f4])\n",
     "\n",
     "\n",
     "mem = MemoryStore()\n",
@@ -442,7 +446,7 @@
     "mem.source.filters.add(f)\n",
     "\n",
     "# attach multiple filters to a MemoryStore\n",
-    "mem.source.filters.update([f1,f2])"
+    "mem.source.filters.add([f1,f2])"
    ]
   },
   {
@@ -457,7 +461,9 @@
   {
    "cell_type": "code",
    "execution_count": 10,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from stix2 import Campaign, Identity, Indicator, Malware, Relationship\n",
@@ -719,21 +725,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "cti-python-stix2",
    "language": "python",
-   "name": "python3"
+   "name": "cti-python-stix2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
   }
  },
  "nbformat": 4,

From 6df23e72686b90b93eaf000d2cbb63139361de43 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 13 Apr 2018 16:06:31 -0400
Subject: [PATCH 111/150] removed unecessary checks, (clearly I need to review
 python ref model)

---
 stix2/datastore/filters.py   | 15 +++------------
 stix2/test/test_datastore.py |  2 +-
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index c260dcc..ab94e89 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -71,8 +71,7 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
 
         if isinstance(value, datetime):
             # if value is a datetime obj, convert to str
-            dt_str = format_datetime(value)
-            value = dt_str  # use temp variable to avoid deepcopy operation
+            value = format_datetime(value)
 
         _check_filter_components(prop, op, value)
 
@@ -90,11 +89,10 @@ class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])):
             False otherwise.
         """
         if isinstance(stix_obj_property, datetime):
-            # if a datetime obj, use str format before comparison
+            # if a datetime obj, convert to str format before comparison
             # NOTE: this check seems like it should be done upstream
             # but will put here for now
-            tmp = format_datetime(stix_obj_property)
-            stix_obj_property = tmp  # use tmp variable to avoid deepcopy op
+            stix_obj_property = format_datetime(stix_obj_property)
 
         if self.op == "=":
             return stix_obj_property == self.value
@@ -174,9 +172,6 @@ def _check_filter(filter_, stix_obj):
                     return True
             return False
 
-        elif isinstance(stix_obj[prop], dict):
-            return _check_filter(sub_filter, stix_obj[prop])
-
         else:
             return _check_filter(sub_filter, stix_obj[prop])
 
@@ -242,8 +237,6 @@ class FilterSet(object):
             if f not in self._filters:
                 self._filters.append(f)
 
-        return
-
     def remove(self, filters=None):
         """remove a Filter, list of Filters, or FilterSet from the FilterSet
 
@@ -263,5 +256,3 @@ class FilterSet(object):
 
         for f in filters:
             self._filters.remove(f)
-
-        return
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 7ee4877..1c7fa43 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -510,7 +510,7 @@ def test_datetime_filter_behavior():
     filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
     filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
 
-    # check taht filter value is converted from datetime to str
+    # check that filter value is converted from datetime to str
     assert isinstance(filter_with_dt_obj.value, str)
 
     # compare datetime string to filter w/ datetime obj

From 194672ee2bb4022f78e33a0169c33ba9dc76bc5a Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 13 Apr 2018 17:11:07 -0400
Subject: [PATCH 112/150] Tweak FilterSet docstrings style

---
 stix2/datastore/filters.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index ab94e89..a32b14a 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -193,7 +193,7 @@ class FilterSet(object):
     that Filters that had a dict as a value could not be added to a Python
     set as dicts are not hashable. Thus this class provides set functionality
     but internally stores filters in a list.
-     """
+    """
 
     def __init__(self, filters=None):
         """
@@ -205,16 +205,16 @@ class FilterSet(object):
             self.add(filters)
 
     def __iter__(self):
-        """provide iteration functionality of FilterSet"""
+        """Provide iteration functionality of FilterSet."""
         for f in self._filters:
             yield f
 
     def __len__(self):
-        """provide built-in len() utility of FilterSet"""
+        """Provide built-in len() utility of FilterSet."""
         return len(self._filters)
 
     def add(self, filters=None):
-        """add a Filter, FilterSet, or list of Filters to the FilterSet
+        """Add a Filter, FilterSet, or list of Filters to the FilterSet.
 
         Operates like set, only adding unique stix2.Filters to the FilterSet
 
@@ -238,7 +238,7 @@ class FilterSet(object):
                 self._filters.append(f)
 
     def remove(self, filters=None):
-        """remove a Filter, list of Filters, or FilterSet from the FilterSet
+        """Remove a Filter, list of Filters, or FilterSet from the FilterSet.
 
         NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
         as it allows for blind calls (very useful in DataStore)

From 14dce03616d2d50c902cb3e825f878a8029e10bf Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 16 Apr 2018 14:37:07 -0400
Subject: [PATCH 113/150] Provide default for revoked, sighting:summary.

This allows filter on un-revoked objects. Changes default JSONEncoder to
drop optional properties with default values in the spec if set to the
default value. They can be included by passing
include_optional_defaults=True to serialize().
---
 stix2/base.py                   | 54 ++++++++++++++++++++++++++++++---
 stix2/test/test_bundle.py       | 12 ++++----
 stix2/test/test_datastore.py    |  4 +--
 stix2/test/test_indicator.py    |  1 +
 stix2/test/test_relationship.py |  8 ++---
 stix2/test/test_sighting.py     |  2 +-
 stix2/test/test_tool.py         | 27 +++++++++++++++++
 stix2/v20/sdo.py                | 26 ++++++++--------
 stix2/v20/sro.py                |  6 ++--
 9 files changed, 105 insertions(+), 35 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 3219007..e128d48 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -22,6 +22,33 @@ DEFAULT_ERROR = "{type} must have {property}='{expected}'."
 
 
 class STIXJSONEncoder(json.JSONEncoder):
+    """Custom JSONEncoder subclass for serializing Python ``stix2`` objects.
+
+    If an optional property with a default value specified in the STIX 2 spec
+    is set to that default value, it will be left out of the serialized output.
+
+    An example of this type of property include the ``revoked`` common property.
+    """
+
+    def default(self, obj):
+        if isinstance(obj, (dt.date, dt.datetime)):
+            return format_datetime(obj)
+        elif isinstance(obj, _STIXBase):
+            tmp_obj = dict(copy.deepcopy(obj))
+            for prop_name in obj._defaulted_optional_properties:
+                del tmp_obj[prop_name]
+            return tmp_obj
+        else:
+            return super(STIXJSONEncoder, self).default(obj)
+
+
+class STIXJSONIncludeOptionalDefaultsEncoder(json.JSONEncoder):
+    """Custom JSONEncoder subclass for serializing Python ``stix2`` objects.
+
+    Differs from ``STIXJSONEncoder`` in that if an optional property with a default
+    value specified in the STIX 2 spec is set to that default value, it will be
+    included in the serialized output.
+    """
 
     def default(self, obj):
         if isinstance(obj, (dt.date, dt.datetime)):
@@ -122,14 +149,25 @@ class _STIXBase(collections.Mapping):
                 setting_kwargs[prop_name] = prop_value
 
         # Detect any missing required properties
-        required_properties = get_required_properties(cls._properties)
-        missing_kwargs = set(required_properties) - set(setting_kwargs)
+        required_properties = set(get_required_properties(cls._properties))
+        missing_kwargs = required_properties - set(setting_kwargs)
         if missing_kwargs:
             raise MissingPropertiesError(cls, missing_kwargs)
 
         for prop_name, prop_metadata in cls._properties.items():
             self._check_property(prop_name, prop_metadata, setting_kwargs)
 
+        # Cache defaulted optional properties for serialization
+        defaulted = []
+        for name, prop in cls._properties.items():
+            try:
+                if (not prop.required and not hasattr(prop, '_fixed_value') and
+                        prop.default() == setting_kwargs[name]):
+                    defaulted.append(name)
+            except (AttributeError, KeyError):
+                continue
+        self._defaulted_optional_properties = defaulted
+
         self._inner = setting_kwargs
 
         self._check_object_constraints()
@@ -151,7 +189,7 @@ class _STIXBase(collections.Mapping):
                              (self.__class__.__name__, name))
 
     def __setattr__(self, name, value):
-        if name != '_inner' and not name.startswith("_STIXBase__"):
+        if not name.startswith("_"):
             raise ImmutableError(self.__class__, name)
         super(_STIXBase, self).__setattr__(name, value)
 
@@ -170,6 +208,7 @@ class _STIXBase(collections.Mapping):
         if isinstance(self, _Observable):
             # Assume: valid references in the original object are still valid in the new version
             new_inner['_valid_refs'] = {'*': '*'}
+        new_inner['allow_custom'] = self.__allow_custom
         return cls(**new_inner)
 
     def properties_populated(self):
@@ -183,7 +222,7 @@ class _STIXBase(collections.Mapping):
     def revoke(self):
         return _revoke(self)
 
-    def serialize(self, pretty=False, **kwargs):
+    def serialize(self, pretty=False, include_optional_defaults=False, **kwargs):
         """
         Serialize a STIX object.
 
@@ -191,6 +230,8 @@ class _STIXBase(collections.Mapping):
             pretty (bool): If True, output properties following the STIX specs
                 formatting. This includes indentation. Refer to notes for more
                 details. (Default: ``False``)
+            include_optional_defaults (bool): Determines whether to include
+                optional properties set to the default value defined in the spec.
             **kwargs: The arguments for a json.dumps() call.
 
         Returns:
@@ -213,7 +254,10 @@ class _STIXBase(collections.Mapping):
 
             kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by})
 
-        return json.dumps(self, cls=STIXJSONEncoder, **kwargs)
+        if include_optional_defaults:
+            return json.dumps(self, cls=STIXJSONIncludeOptionalDefaultsEncoder, **kwargs)
+        else:
+            return json.dumps(self, cls=STIXJSONEncoder, **kwargs)
 
 
 class _Observable(_STIXBase):
diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py
index 8b14172..2d14654 100644
--- a/stix2/test/test_bundle.py
+++ b/stix2/test/test_bundle.py
@@ -6,7 +6,7 @@ import stix2
 
 EXPECTED_BUNDLE = """{
     "type": "bundle",
-    "id": "bundle--00000000-0000-0000-0000-000000000004",
+    "id": "bundle--00000000-0000-0000-0000-000000000007",
     "spec_version": "2.0",
     "objects": [
         {
@@ -22,7 +22,7 @@ EXPECTED_BUNDLE = """{
         },
         {
             "type": "malware",
-            "id": "malware--00000000-0000-0000-0000-000000000002",
+            "id": "malware--00000000-0000-0000-0000-000000000003",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "name": "Cryptolocker",
@@ -32,7 +32,7 @@ EXPECTED_BUNDLE = """{
         },
         {
             "type": "relationship",
-            "id": "relationship--00000000-0000-0000-0000-000000000003",
+            "id": "relationship--00000000-0000-0000-0000-000000000005",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "relationship_type": "indicates",
@@ -44,7 +44,7 @@ EXPECTED_BUNDLE = """{
 
 EXPECTED_BUNDLE_DICT = {
     "type": "bundle",
-    "id": "bundle--00000000-0000-0000-0000-000000000004",
+    "id": "bundle--00000000-0000-0000-0000-000000000007",
     "spec_version": "2.0",
     "objects": [
         {
@@ -60,7 +60,7 @@ EXPECTED_BUNDLE_DICT = {
         },
         {
             "type": "malware",
-            "id": "malware--00000000-0000-0000-0000-000000000002",
+            "id": "malware--00000000-0000-0000-0000-000000000003",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "name": "Cryptolocker",
@@ -70,7 +70,7 @@ EXPECTED_BUNDLE_DICT = {
         },
         {
             "type": "relationship",
-            "id": "relationship--00000000-0000-0000-0000-000000000003",
+            "id": "relationship--00000000-0000-0000-0000-000000000005",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "relationship_type": "indicates",
diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 1c7fa43..315aff5 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -405,13 +405,11 @@ def test_apply_common_filters4():
 
 def test_apply_common_filters5():
     # "Return any object whose not revoked"
-    # Note that if 'revoked' property is not present in object.
-    # Currently we can't use such an expression to filter for... :(
     resp = list(apply_common_filters(stix_objs, [filters[5]]))
     assert len(resp) == 0
 
     resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
-    assert len(resp) == 0
+    assert len(resp) == 4
 
 
 def test_apply_common_filters6():
diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py
index 78d1bf2..c9b6e56 100644
--- a/stix2/test/test_indicator.py
+++ b/stix2/test/test_indicator.py
@@ -45,6 +45,7 @@ def test_indicator_with_all_required_properties():
         labels=['malicious-activity'],
     )
 
+    assert ind.revoked is False
     assert str(ind) == EXPECTED_INDICATOR
     rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(ind))
     assert rep == EXPECTED_INDICATOR_REPR
diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py
index c6e2d6f..32ddf91 100644
--- a/stix2/test/test_relationship.py
+++ b/stix2/test/test_relationship.py
@@ -123,8 +123,8 @@ def test_create_relationship_from_objects_rather_than_ids(indicator, malware):
 
     assert rel.relationship_type == 'indicates'
     assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001'
-    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002'
-    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003'
+    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003'
+    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005'
 
 
 def test_create_relationship_with_positional_args(indicator, malware):
@@ -132,8 +132,8 @@ def test_create_relationship_with_positional_args(indicator, malware):
 
     assert rel.relationship_type == 'indicates'
     assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001'
-    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002'
-    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003'
+    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003'
+    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005'
 
 
 @pytest.mark.parametrize("data", [
diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py
index ce1fab9..06f96b8 100644
--- a/stix2/test/test_sighting.py
+++ b/stix2/test/test_sighting.py
@@ -86,7 +86,7 @@ def test_create_sighting_from_objects_rather_than_ids(malware):  # noqa: F811
     rel = stix2.Sighting(sighting_of_ref=malware)
 
     assert rel.sighting_of_ref == 'malware--00000000-0000-0000-0000-000000000001'
-    assert rel.id == 'sighting--00000000-0000-0000-0000-000000000002'
+    assert rel.id == 'sighting--00000000-0000-0000-0000-000000000003'
 
 
 @pytest.mark.parametrize("data", [
diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py
index ce99fb8..9fc2c22 100644
--- a/stix2/test/test_tool.py
+++ b/stix2/test/test_tool.py
@@ -19,6 +19,19 @@ EXPECTED = """{
     ]
 }"""
 
+EXPECTED_WITH_REVOKED = """{
+    "type": "tool",
+    "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+    "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+    "created": "2016-04-06T20:03:48.000Z",
+    "modified": "2016-04-06T20:03:48.000Z",
+    "name": "VNC",
+    "revoked": false,
+    "labels": [
+        "remote-access"
+    ]
+}"""
+
 
 def test_tool_example():
     tool = stix2.Tool(
@@ -64,4 +77,18 @@ def test_tool_no_workbench_wrappers():
     with pytest.raises(AttributeError):
         tool.created_by()
 
+
+def test_tool_serialize_with_defaults():
+    tool = stix2.Tool(
+        id="tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+        created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        created="2016-04-06T20:03:48.000Z",
+        modified="2016-04-06T20:03:48.000Z",
+        name="VNC",
+        labels=["remote-access"],
+    )
+
+    assert tool.serialize(pretty=True, include_optional_defaults=True) == EXPECTED_WITH_REVOKED
+
+
 # TODO: Add other examples
diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index ff024f5..d7e9954 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -36,7 +36,7 @@ class AttackPattern(STIXDomainObject):
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -63,7 +63,7 @@ class Campaign(STIXDomainObject):
         ('first_seen', TimestampProperty()),
         ('last_seen', TimestampProperty()),
         ('objective', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -86,7 +86,7 @@ class CourseOfAction(STIXDomainObject):
         ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')),
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -112,7 +112,7 @@ class Identity(STIXDomainObject):
         ('identity_class', StringProperty(required=True)),
         ('sectors', ListProperty(StringProperty)),
         ('contact_information', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -139,7 +139,7 @@ class Indicator(STIXDomainObject):
         ('valid_from', TimestampProperty(default=lambda: NOW)),
         ('valid_until', TimestampProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -169,7 +169,7 @@ class IntrusionSet(STIXDomainObject):
         ('resource_level', StringProperty()),
         ('primary_motivation', StringProperty()),
         ('secondary_motivations', ListProperty(StringProperty)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -193,7 +193,7 @@ class Malware(STIXDomainObject):
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -218,7 +218,7 @@ class ObservedData(STIXDomainObject):
         ('last_observed', TimestampProperty(required=True)),
         ('number_observed', IntegerProperty(required=True)),
         ('objects', ObservableProperty(required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -243,7 +243,7 @@ class Report(STIXDomainObject):
         ('description', StringProperty()),
         ('published', TimestampProperty(required=True)),
         ('object_refs', ListProperty(ReferenceProperty, required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -274,7 +274,7 @@ class ThreatActor(STIXDomainObject):
         ('primary_motivation', StringProperty()),
         ('secondary_motivations', ListProperty(StringProperty)),
         ('personal_motivations', ListProperty(StringProperty)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -299,7 +299,7 @@ class Tool(STIXDomainObject):
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
         ('tool_version', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -322,7 +322,7 @@ class Vulnerability(STIXDomainObject):
         ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')),
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -382,7 +382,7 @@ def CustomObject(type='x-custom-type', properties=None):
 
             # This is to follow the general properties structure.
             _properties.update([
-                ('revoked', BooleanProperty()),
+                ('revoked', BooleanProperty(default=lambda: False)),
                 ('labels', ListProperty(StringProperty)),
                 ('external_references', ListProperty(ExternalReference)),
                 ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py
index 7d7d3ae..e488229 100644
--- a/stix2/v20/sro.py
+++ b/stix2/v20/sro.py
@@ -32,7 +32,7 @@ class Relationship(STIXRelationshipObject):
         ('description', StringProperty()),
         ('source_ref', ReferenceProperty(required=True)),
         ('target_ref', ReferenceProperty(required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -72,8 +72,8 @@ class Sighting(STIXRelationshipObject):
         ('sighting_of_ref', ReferenceProperty(required=True)),
         ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))),
         ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))),
-        ('summary', BooleanProperty()),
-        ('revoked', BooleanProperty()),
+        ('summary', BooleanProperty(default=lambda: False)),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),

From f66fbb73d9b30b32d153f473c635bea7b79d9687 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 16 Apr 2018 14:56:23 -0400
Subject: [PATCH 114/150] Update CHANGELOG, docs for v1.0.0

---
 CHANGELOG      | 17 +++++++++++++++++
 docs/index.rst |  4 ----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 7f77b25..f63f682 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,23 @@
 CHANGELOG
 =========
 
+1.0.0 - 2018-04-16
+
+* Adds the Workbench layer API.
+* Adds checks to ensure valid type names are provided.
+* Supports parsing generic custom STIX 2 content without needing to create classes for them.
+* Fixes "Not JSON serializable" error in TAXIICollectionStore.
+* Fixes bug with parsing JSON files in FileSystemStore.
+* Fixes bug with Filters in TAXIICollectionStore.
+* Fixes minor bugs in the patterning API.
+* Fixes bug with ListProperty containing DictionaryProperty.
+* Fixes bug with parsing observables.
+* Fixes bug involving optional properties with default values.
+* Changes custom observable extensions to require properties to be defined as a list of tuples rather than a dictionary.
+* Changes Filters to allow passing a dictionary as a filter value.
+* Changes `get_dict` to a private function.
+* `taxii2-client` is now an optional dependency.
+
 0.5.1 - 2018-03-06
 
 * Fixes issue with PyPI.
diff --git a/docs/index.rst b/docs/index.rst
index dd39903..023400e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -6,10 +6,6 @@
 STIX 2 Python API Documentation
 ===============================
 
-.. warning::
-
-    Prior to version 1.0, all APIs are considered unstable and subject to change.
-
 Welcome to the STIX 2 Python API's documentation. This library is designed to
 help you work with STIX 2 content. For more information about STIX 2, see the
 `website `_ of the OASIS Cyber Threat Intelligence

From 161e10ec4068243b6f8eb835bf708d3da9a00bba Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 16 Apr 2018 14:57:35 -0400
Subject: [PATCH 115/150] =?UTF-8?q?Bump=20version:=200.5.1=20=E2=86=92=201?=
 =?UTF-8?q?.0.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/conf.py     | 4 ++--
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 0764454..85c8b23 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,8 +34,8 @@ project = 'stix2'
 copyright = '2017, OASIS Open'
 author = 'OASIS Open'
 
-version = '0.5.1'
-release = '0.5.1'
+version = '1.0.0'
+release = '1.0.0'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints']
diff --git a/setup.cfg b/setup.cfg
index b252746..ac3e5d5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.5.1
+current_version = 1.0.0
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index dd9b22c..5becc17 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "0.5.1"
+__version__ = "1.0.0"

From 3373a66aef9105f6bc7fc1712bcb8b7ac2289573 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 16 Apr 2018 15:47:25 -0400
Subject: [PATCH 116/150] Fix README for PyPI rendering

Title underline too short.
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 46e8164..0212eed 100644
--- a/README.rst
+++ b/README.rst
@@ -179,7 +179,7 @@ repositories/maintainers-guide#additionalMaintainers>`__.
    Corporation `__
 
 About OASIS TC Open Repositories
------------------------------
+--------------------------------
 
 -  `TC Open Repositories: Overview and
    Resources 
Date: Mon, 23 Apr 2018 09:08:29 -0400
Subject: [PATCH 117/150] Rename stix object fixture

---
 stix2/test/conftest.py               |  4 +--
 stix2/test/test_datastore_filters.py | 40 ++++++++++++++--------------
 stix2/test/test_datastore_memory.py  |  9 ++++---
 stix2/test/test_utils.py             |  4 +--
 4 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py
index acbb8b0..5cf63b1 100644
--- a/stix2/test/conftest.py
+++ b/stix2/test/conftest.py
@@ -49,7 +49,7 @@ def relationship(uuid4, clock):
 
 
 @pytest.fixture
-def stix1_objs():
+def stix_objs1():
     ind1 = {
         "created": "2017-01-27T13:49:53.935Z",
         "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
@@ -114,7 +114,7 @@ def stix1_objs():
 
 
 @pytest.fixture
-def stix2_objs():
+def stix_objs2():
     ind6 = {
         "created": "2017-01-27T13:49:53.935Z",
         "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py
index 4f66ff5..32e91ed 100644
--- a/stix2/test/test_datastore_filters.py
+++ b/stix2/test/test_datastore_filters.py
@@ -191,31 +191,31 @@ def test_apply_common_filters():
     assert len(resp) == 0
 
 
-def test_filters0():
+def test_filters0(stix_objs2):
     # "Return any object modified before 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[1]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
+    assert resp[0]['id'] == stix_objs2[1]['id']
     assert len(resp) == 2
 
 
-def test_filters1():
+def test_filters1(stix_objs2):
     # "Return any object modified after 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
+    assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 1
 
 
-def test_filters2():
+def test_filters2(stix_objs2):
     # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
+    assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 3
 
 
-def test_filters3():
+def test_filters3(stix_objs2):
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
-    assert resp[0]['id'] == STIX_OBJS2[1]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
+    assert resp[0]['id'] == stix_objs2[1]['id']
     assert len(resp) == 2
 
 
@@ -227,23 +227,23 @@ def test_filters4():
                                   "for specified property: 'modified'")
 
 
-def test_filters5():
+def test_filters5(stix_objs2):
     # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
+    assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 1
 
 
-def test_filters6():
+def test_filters6(stix_objs2):
     # Test filtering on non-common property
-    resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")]))
-    assert resp[0]['id'] == STIX_OBJS2[0]['id']
+    resp = list(apply_common_filters(stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")]))
+    assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 3
 
 
-def test_filters7():
+def test_filters7(stix_objs2):
     # Test filtering on embedded property
-    stix_objects = list(STIX_OBJS2) + [{
+    stix_objects = list(stix_objs2) + [{
         "type": "observed-data",
         "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
         "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
index b027640..d341fd2 100644
--- a/stix2/test/test_datastore_memory.py
+++ b/stix2/test/test_datastore_memory.py
@@ -1,6 +1,7 @@
 import pytest
 
 from stix2.datastore import CompositeDataSource, make_id
+from stix2.datastore.filters import Filter
 from stix2.datastore.memory import MemorySource, MemorySink
 
 
@@ -24,18 +25,18 @@ def test_add_remove_composite_datasource():
     assert len(cds.get_all_data_sources()) == 0
 
 
-def test_composite_datasource_operations():
+def test_composite_datasource_operations(stix_objs1, stix_objs2):
     BUNDLE1 = dict(id="bundle--%s" % make_id(),
-                   objects=STIX_OBJS1,
+                   objects=stix_objs1,
                    spec_version="2.0",
                    type="bundle")
     cds1 = CompositeDataSource()
     ds1_1 = MemorySource(stix_data=BUNDLE1)
-    ds1_2 = MemorySource(stix_data=STIX_OBJS2)
+    ds1_2 = MemorySource(stix_data=stix_objs2)
 
     cds2 = CompositeDataSource()
     ds2_1 = MemorySource(stix_data=BUNDLE1)
-    ds2_2 = MemorySource(stix_data=STIX_OBJS2)
+    ds2_2 = MemorySource(stix_data=stix_objs2)
 
     cds1.add_data_sources([ds1_1, ds1_2])
     cds2.add_data_sources([ds2_1, ds2_2])
diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py
index 4705ec8..9d25a5e 100644
--- a/stix2/test/test_utils.py
+++ b/stix2/test/test_utils.py
@@ -84,8 +84,8 @@ def test_get_type_from_id(stix_id, typ):
     assert stix2.utils.get_type_from_id(stix_id) == typ
 
 
-def test_deduplicate(stix1_objs):
-    unique = stix2.utils.deduplicate(stix1_objs)
+def test_deduplicate(stix_objs1):
+    unique = stix2.utils.deduplicate(stix_objs1)
 
     # Only 3 objects are unique
     # 2 id's vary

From 988dad79b9b337f00ba3b1d66740fa7d340e65b0 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 09:11:59 -0400
Subject: [PATCH 118/150] WIP: Mock TAXIIClient Collection Endpoint

---
 stix2/test/test_datastore_taxii.py | 97 +++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 10 deletions(-)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index 0a293b7..c20fbb6 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -1,24 +1,101 @@
+from stix2 import Bundle, ThreatActor, TAXIICollectionSource, TAXIICollectionSink
+from stix2.datastore.filters import Filter
+
+import json
+import pytest
+
+from taxii2client import Collection, _filter_kwargs_to_query_params
+from medallion.filters.basic_filter import BasicFilter
+
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
 
 
-class MockTAXIIClient(object):
+class MockTAXIICollectionEndpoint(Collection):
     """Mock for taxii2_client.TAXIIClient"""
-    pass
+
+    def __init__(self, url, **kwargs):
+        super(MockTAXIICollectionEndpoint, self).__init__(url, **kwargs)
+        self.objects = []
+
+    def add_objects(self, bundle):
+        self._verify_can_write()
+        if isinstance(bundle, str):
+            bundle = json.loads(bundle)
+        for object in bundle.get("objects", []):
+            self.objects.append(object)
+
+    def get_objects(self, **filter_kwargs):
+        self._verify_can_read()
+        query_params = _filter_kwargs_to_query_params(filter_kwargs)
+        query_params = json.loads(query_params)
+        full_filter = BasicFilter(query_params or {})
+        objs = full_filter.process_filter(
+            self.objects,
+            ("id", "type", "version"),
+            []
+        )
+        return Bundle(objects=objs)
+
+    def get_object(self, id, version=None):
+        self._verify_can_read()
+        query_params = None
+        if version:
+            query_params = _filter_kwargs_to_query_params({"version": version})
+        if query_params:
+            query_params = json.loads(query_params)
+        full_filter = BasicFilter(query_params or {})
+        objs = full_filter.process_filter(
+            self.objects,
+            ("version",),
+            []
+        )
+        return Bundle(objects=objs)
 
 
 @pytest.fixture
-def collection():
-    return Collection(COLLECTION_URL, MockTAXIIClient())
+def collection(stix_objs1):
+    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{
+        "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116",
+        "title": "Writable Collection",
+        "description": "This collection is a dropbox for submitting indicators",
+        "can_read": True,
+        "can_write": True,
+        "media_types": [
+            "application/vnd.oasis.stix+json; version=2.0"
+        ]
+    })
+
+    mock.objects.extend(stix_objs1)
+    return mock
 
 
 def test_ds_taxii(collection):
-    ds = taxii.TAXIICollectionSource(collection)
+    ds = TAXIICollectionSource(collection)
     assert ds.collection is not None
 
 
-def test_ds_taxii_name(collection):
-    ds = taxii.TAXIICollectionSource(collection)
-    assert ds.collection is not None
+def test_add_stix2_object(collection):
+    tc_sink = TAXIICollectionSink(collection)
+
+    # create new STIX threat-actor
+    ta = ThreatActor(name="Teddy Bear",
+                     labels=["nation-state"],
+                     sophistication="innovator",
+                     resource_level="government",
+                     goals=[
+                         "compromising environment NGOs",
+                         "water-hole attacks geared towards energy sector",
+                     ])
+
+    tc_sink.add(ta)
+
+
+def test_get_stix2_object(collection):
+    tc_sink = TAXIICollectionSource(collection)
+
+    objects = tc_sink.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+
+    assert objects
 
 
 def test_parse_taxii_filters():
@@ -37,7 +114,7 @@ def test_parse_taxii_filters():
         Filter("version", "=", "first")
     ])
 
-    ds = taxii.TAXIICollectionSource(collection)
+    ds = TAXIICollectionSource(collection)
 
     taxii_filters = ds._parse_taxii_filters(query)
 
@@ -45,7 +122,7 @@ def test_parse_taxii_filters():
 
 
 def test_add_get_remove_filter():
-    ds = taxii.TAXIICollectionSource(collection)
+    ds = TAXIICollectionSource(collection)
 
     # First 3 filters are valid, remaining properties are erroneous in some way
     valid_filters = [

From 507e3995dd80888c3bcbc636cb3bd9fe62e6dbd3 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 09:42:33 -0400
Subject: [PATCH 119/150] Update .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 395676e..9e12d7d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,6 +43,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+.pytest_cache/
 
 # Translations
 *.mo

From 8a9acfe487feecaedb92bb8e8a86358a0ad00d53 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:11:08 -0400
Subject: [PATCH 120/150] Create TAXII query correctly. closes #169

---
 stix2/datastore/taxii.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index faa2669..872a510 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -176,8 +176,8 @@ class TAXIICollectionSource(DataSource):
         """
         # make query in TAXII query format since 'id' is TAXII field
         query = [
-            Filter("match[id]", "=", stix_id),
-            Filter("match[version]", "=", "all")
+            Filter("id", "=", stix_id),
+            Filter("version", "=", "all")
         ]
 
         all_data = self.query(query=query, _composite_filters=_composite_filters)
@@ -232,7 +232,8 @@ class TAXIICollectionSource(DataSource):
             all_data = deduplicate(all_data)
 
             # apply local (CompositeDataSource, TAXIICollectionSource and query) filters
-            all_data = list(apply_common_filters(all_data, (query - taxii_filters)))
+            query.remove(taxii_filters)
+            all_data = list(apply_common_filters(all_data, query))
 
         except HTTPError:
             # if resources not found or access is denied from TAXII server, return empty list
@@ -249,7 +250,7 @@ class TAXIICollectionSource(DataSource):
         Does not put in TAXII spec format as the TAXII2Client (that we use)
         does this for us.
 
-        NOTE:
+        Notes:
             Currently, the TAXII2Client can handle TAXII filters where the
             filter value is list, as both a comma-seperated string or python list
 
@@ -265,7 +266,8 @@ class TAXIICollectionSource(DataSource):
             query (list): list of filters to extract which ones are TAXII
                 specific.
 
-        Returns: a list of the TAXII filters
+        Returns:
+            A list of TAXII filters that meet the TAXII filtering parameters.
 
         """
         taxii_filters = []

From fb79e703b8a2b339c45a3e99d3283d314d0258fb Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:12:16 -0400
Subject: [PATCH 121/150] Simplify FilterSet instance in MemorySource

---
 stix2/datastore/memory.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index c43da8b..c1d202d 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -268,10 +268,7 @@ class MemorySource(DataSource):
                 is returned in the same form as it as added.
 
         """
-        if query is None:
-            query = FilterSet()
-        else:
-            query = FilterSet(query)
+        query = FilterSet(query)
 
         # combine all query filters
         if self.filters:

From 488012d39d44fb4f27d3e541a08f06f1bbccea58 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:12:54 -0400
Subject: [PATCH 122/150] Add support for 'real_stix_objs' for test cases

---
 stix2/test/conftest.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py
index 5cf63b1..ddbcb7e 100644
--- a/stix2/test/conftest.py
+++ b/stix2/test/conftest.py
@@ -152,3 +152,13 @@ def stix_objs2():
         "valid_from": "2017-01-27T13:49:53.935382Z"
     }
     return [ind6, ind7, ind8]
+
+
+@pytest.fixture
+def real_stix_objs1(stix_objs1):
+    return [stix2.parse(x) for x in stix_objs1]
+
+
+@pytest.fixture
+def real_stix_objs2(stix_objs2):
+    return [stix2.parse(x) for x in stix_objs2]

From e4a226cae6ae3a5e6c1a507942f7bd8e4c52fc16 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:14:16 -0400
Subject: [PATCH 123/150] Finish merging all the changes from @mbastian1135

---
 stix2/test/test_datastore.py         |   7 +
 stix2/test/test_datastore_filters.py | 376 ++++++++++++++++++++-------
 stix2/test/test_datastore_memory.py  |   2 +-
 3 files changed, 288 insertions(+), 97 deletions(-)

diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py
index 0121a85..323365a 100644
--- a/stix2/test/test_datastore.py
+++ b/stix2/test/test_datastore.py
@@ -108,3 +108,10 @@ def test_composite_datastore_add_data_sources_raises_error():
         ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f"
         CompositeDataSource().add_data_sources(ind)
     assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value)
+
+
+def test_composite_datastore_no_datasource():
+    cds = CompositeDataSource()
+    with pytest.raises(AttributeError) as excinfo:
+        cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert 'CompositeDataSource has no data source' in str(excinfo.value)
diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py
index 32e91ed..8803e4b 100644
--- a/stix2/test/test_datastore_filters.py
+++ b/stix2/test/test_datastore_filters.py
@@ -1,6 +1,112 @@
 import pytest
 
+from stix2 import parse
 from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.utils import STIXdatetime, parse_into_datetime
+
+
+stix_objs = [
+    {
+        "created": "2017-01-27T13:49:53.997Z",
+        "description": "\n\nTITLE:\n\tPoison Ivy",
+        "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "labels": [
+            "remote-access-trojan"
+        ],
+        "modified": "2017-01-27T13:49:53.997Z",
+        "name": "Poison Ivy",
+        "type": "malware"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "labels": [
+            "file-hash-watchlist"
+        ],
+        "modified": "2014-05-08T09:00:00.000Z",
+        "name": "File hash for Poison Ivy variant",
+        "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
+        "type": "indicator",
+        "valid_from": "2014-05-08T09:00:00.000000Z"
+    },
+    {
+        "created": "2014-05-08T09:00:00.000Z",
+        "granular_markings": [
+            {
+                "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
+                "selectors": [
+                    "relationship_type"
+                ]
+            }
+        ],
+        "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
+        "modified": "2014-05-08T09:00:00.000Z",
+        "object_marking_refs": [
+            "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
+        ],
+        "relationship_type": "indicates",
+        "revoked": True,
+        "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
+        "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
+        "type": "relationship"
+    },
+    {
+        "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
+        "created": "2016-02-14T00:00:00.000Z",
+        "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
+        "modified": "2016-02-14T00:00:00.000Z",
+        "type": "vulnerability",
+        "name": "CVE-2014-0160",
+        "description": "The (1) TLS...",
+        "external_references": [
+            {
+                "source_name": "cve",
+                "external_id": "CVE-2014-0160"
+            }
+        ],
+        "labels": ["heartbleed", "has-logo"]
+    },
+    {
+        "type": "observed-data",
+        "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
+        "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "file",
+                "name": "HAL 9000.exe"
+            }
+        }
+
+    }
+]
+
+
+filters = [
+    Filter("type", "!=", "relationship"),
+    Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
+    Filter("labels", "in", "remote-access-trojan"),
+    Filter("created", ">", "2015-01-01T01:00:00.000Z"),
+    Filter("revoked", "=", True),
+    Filter("revoked", "!=", True),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "relationship_type"),
+    Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
+    Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
+    Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
+    Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
+    Filter("granular_markings.selectors", "in", "description"),
+    Filter("external_references.source_name", "=", "CVE"),
+    Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}})
+]
+
+# same as above objects but converted to real Python STIX2 objects
+# to test filters against true Python STIX2 objects
+real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs]
 
 
 def test_filter_ops_check():
@@ -43,181 +149,243 @@ def test_filter_type_underscore_check():
     assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value)
 
 
-def test_apply_common_filters():
-    stix_objs = [
-        {
-            "created": "2017-01-27T13:49:53.997Z",
-            "description": "\n\nTITLE:\n\tPoison Ivy",
-            "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "labels": [
-                "remote-access-trojan"
-            ],
-            "modified": "2017-01-27T13:49:53.997Z",
-            "name": "Poison Ivy",
-            "type": "malware"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "labels": [
-                "file-hash-watchlist"
-            ],
-            "modified": "2014-05-08T09:00:00.000Z",
-            "name": "File hash for Poison Ivy variant",
-            "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']",
-            "type": "indicator",
-            "valid_from": "2014-05-08T09:00:00.000000Z"
-        },
-        {
-            "created": "2014-05-08T09:00:00.000Z",
-            "granular_markings": [
-                {
-                    "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
-                    "selectors": [
-                        "relationship_type"
-                    ]
-                }
-            ],
-            "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463",
-            "modified": "2014-05-08T09:00:00.000Z",
-            "object_marking_refs": [
-                "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
-            ],
-            "relationship_type": "indicates",
-            "revoked": True,
-            "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade",
-            "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111",
-            "type": "relationship"
-        },
-        {
-            "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef",
-            "created": "2016-02-14T00:00:00.000Z",
-            "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9",
-            "modified": "2016-02-14T00:00:00.000Z",
-            "type": "vulnerability",
-            "name": "CVE-2014-0160",
-            "description": "The (1) TLS...",
-            "external_references": [
-                {
-                    "source_name": "cve",
-                    "external_id": "CVE-2014-0160"
-                }
-            ],
-            "labels": ["heartbleed", "has-logo"]
-        }
-    ]
-
-    filters = [
-        Filter("type", "!=", "relationship"),
-        Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"),
-        Filter("labels", "in", "remote-access-trojan"),
-        Filter("created", ">", "2015-01-01T01:00:00.000Z"),
-        Filter("revoked", "=", True),
-        Filter("revoked", "!=", True),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "relationship_type"),
-        Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
-        Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"),
-        Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"),
-        Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"),
-        Filter("granular_markings.selectors", "in", "description"),
-        Filter("external_references.source_name", "=", "CVE"),
-    ]
-
+def test_apply_common_filters0():
     # "Return any object whose type is not relationship"
     resp = list(apply_common_filters(stix_objs, [filters[0]]))
     ids = [r['id'] for r in resp]
     assert stix_objs[0]['id'] in ids
     assert stix_objs[1]['id'] in ids
     assert stix_objs[3]['id'] in ids
-    assert len(ids) == 3
+    assert len(ids) == 4
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[0]]))
+    ids = [r.id for r in resp]
+    assert real_stix_objs[0].id in ids
+    assert real_stix_objs[1].id in ids
+    assert real_stix_objs[3].id in ids
+    assert len(ids) == 4
+
+
+def test_apply_common_filters1():
     # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"
     resp = list(apply_common_filters(stix_objs, [filters[1]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[1]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters2():
     # "Return any object that contains remote-access-trojan in labels"
     resp = list(apply_common_filters(stix_objs, [filters[2]]))
     assert resp[0]['id'] == stix_objs[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[2]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters3():
     # "Return any object created after 2015-01-01T01:00:00.000Z"
     resp = list(apply_common_filters(stix_objs, [filters[3]]))
     assert resp[0]['id'] == stix_objs[0]['id']
-    assert len(resp) == 2
+    assert len(resp) == 3
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[3]]))
+    assert resp[0].id == real_stix_objs[0].id
+    assert len(resp) == 3
+
+
+def test_apply_common_filters4():
     # "Return any revoked object"
     resp = list(apply_common_filters(stix_objs, [filters[4]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[4]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters5():
     # "Return any object whose not revoked"
-    # Note that if 'revoked' property is not present in object.
-    # Currently we can't use such an expression to filter for... :(
     resp = list(apply_common_filters(stix_objs, [filters[5]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
+    assert len(resp) == 4
+
+
+def test_apply_common_filters6():
     # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs"
     resp = list(apply_common_filters(stix_objs, [filters[6]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[6]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters7():
     # "Return any object that contains relationship_type in their selectors AND
     # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref"
     resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]]))
     assert resp[0]['id'] == stix_objs[2]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[7], filters[8]]))
+    assert resp[0].id == real_stix_objs[2].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters8():
     # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id"
     resp = list(apply_common_filters(stix_objs, [filters[9]]))
     assert resp[0]['id'] == stix_objs[3]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[9]]))
+    assert resp[0].id == real_stix_objs[3].id
+    assert len(resp) == 1
+
+
+def test_apply_common_filters9():
     # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9"
     resp = list(apply_common_filters(stix_objs, [filters[10]]))
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[10]]))
+    assert len(resp) == 1
+
+
+def test_apply_common_filters10():
     # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None)
     resp = list(apply_common_filters(stix_objs, [filters[11]]))
     assert len(resp) == 0
 
+    resp = list(apply_common_filters(real_stix_objs, [filters[11]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters11():
     # "Return any object that contains description in its selectors" (None)
     resp = list(apply_common_filters(stix_objs, [filters[12]]))
     assert len(resp) == 0
 
-    # "Return any object that object that matches CVE in source_name" (None, case sensitive)
-    resp = list(apply_common_filters(stix_objs, [filters[13]]))
+    resp = list(apply_common_filters(real_stix_objs, [filters[12]]))
     assert len(resp) == 0
 
 
-def test_filters0(stix_objs2):
+def test_apply_common_filters12():
+    # "Return any object that matches CVE in source_name" (None, case sensitive)
+    resp = list(apply_common_filters(stix_objs, [filters[13]]))
+    assert len(resp) == 0
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[13]]))
+    assert len(resp) == 0
+
+
+def test_apply_common_filters13():
+    # Return any object that matches file object in "objects"
+    resp = list(apply_common_filters(stix_objs, [filters[14]]))
+    assert resp[0]["id"] == stix_objs[4]["id"]
+    assert len(resp) == 1
+    # important additional check to make sure original File dict was
+    # not converted to File object. (this was a deep bug found)
+    assert isinstance(resp[0]["objects"]["0"], dict)
+
+    resp = list(apply_common_filters(real_stix_objs, [filters[14]]))
+    assert resp[0].id == real_stix_objs[4].id
+    assert len(resp) == 1
+
+
+def test_datetime_filter_behavior():
+    """if a filter is initialized with its value being a datetime object
+    OR the STIX object property being filtered on is a datetime object, all
+    resulting comparisons executed are done on the string representations
+    of the datetime objects, as the Filter functionality will convert
+    all datetime objects to there string forms using format_datetim()
+
+    This test makes sure all datetime comparisons are carried out correctly
+    """
+    filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
+    filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
+
+    # check that filter value is converted from datetime to str
+    assert isinstance(filter_with_dt_obj.value, str)
+
+    # compare datetime string to filter w/ datetime obj
+    resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ datetime obj
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
+
+    # compare datetime string to filter w/ str
+    resp = list(apply_common_filters(stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+
+    # compare datetime obj to filter w/ str
+    resp = list(apply_common_filters(real_stix_objs, [filter_with_str]))
+    assert len(resp) == 1
+    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
+    assert isinstance(resp[0].created, STIXdatetime)  # make sure original object not altered
+
+
+def test_filters0(stix_objs2, real_stix_objs2):
     # "Return any object modified before 2017-01-28T13:49:53.935Z"
     resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]))
     assert resp[0]['id'] == stix_objs2[1]['id']
     assert len(resp) == 2
 
+    resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", "<", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == real_stix_objs2[1].id
+    assert len(resp) == 2
 
-def test_filters1(stix_objs2):
+
+def test_filters1(stix_objs2, real_stix_objs2):
     # "Return any object modified after 2017-01-28T13:49:53.935Z"
     resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
     assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">", parse_into_datetime("2017-01-28T13:49:53.935Z"))]))
+    assert resp[0].id == real_stix_objs2[0].id
+    assert len(resp) == 1
 
-def test_filters2(stix_objs2):
+
+def test_filters2(stix_objs2, real_stix_objs2):
     # "Return any object modified after or on 2017-01-28T13:49:53.935Z"
     resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
     assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
+    assert resp[0].id == real_stix_objs2[0].id
+    assert len(resp) == 3
 
-def test_filters3(stix_objs2):
+
+def test_filters3(stix_objs2, real_stix_objs2):
     # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
     resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
     assert resp[0]['id'] == stix_objs2[1]['id']
     assert len(resp) == 2
 
+    # "Return any object modified before or on 2017-01-28T13:49:53.935Z"
+    fv = Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z"))
+    resp = list(apply_common_filters(real_stix_objs2, [fv]))
+    assert resp[0].id == real_stix_objs2[1].id
+    assert len(resp) == 2
+
 
 def test_filters4():
     # Assert invalid Filter cannot be created
@@ -227,23 +395,31 @@ def test_filters4():
                                   "for specified property: 'modified'")
 
 
-def test_filters5(stix_objs2):
+def test_filters5(stix_objs2, real_stix_objs2):
     # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
     resp = list(apply_common_filters(stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
     assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 1
 
+    resp = list(apply_common_filters(real_stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]))
+    assert resp[0].id == real_stix_objs2[0].id
+    assert len(resp) == 1
 
-def test_filters6(stix_objs2):
+
+def test_filters6(stix_objs2, real_stix_objs2):
     # Test filtering on non-common property
     resp = list(apply_common_filters(stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")]))
     assert resp[0]['id'] == stix_objs2[0]['id']
     assert len(resp) == 3
 
+    resp = list(apply_common_filters(real_stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")]))
+    assert resp[0].id == real_stix_objs2[0].id
+    assert len(resp) == 3
 
-def test_filters7(stix_objs2):
+
+def test_filters7(stix_objs2, real_stix_objs2):
     # Test filtering on embedded property
-    stix_objects = list(stix_objs2) + [{
+    obsvd_data_obj = {
         "type": "observed-data",
         "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
         "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
@@ -274,7 +450,15 @@ def test_filters7(stix_objs2):
                 }
             }
         }
-    }]
+    }
+
+    stix_objects = list(stix_objs2) + [obsvd_data_obj]
+    real_stix_objects = list(real_stix_objs2) + [parse(obsvd_data_obj)]
+
     resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
     assert resp[0]['id'] == stix_objects[3]['id']
     assert len(resp) == 1
+
+    resp = list(apply_common_filters(real_stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")]))
+    assert resp[0].id == real_stix_objects[3].id
+    assert len(resp) == 1
diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
index d341fd2..dcec813 100644
--- a/stix2/test/test_datastore_memory.py
+++ b/stix2/test/test_datastore_memory.py
@@ -62,7 +62,7 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2):
         Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")
     ]
 
-    cds1.filters.update(query2)
+    cds1.filters.add(query2)
 
     results = cds1.query(query1)
 

From 2fe9a0f2979cf85159a29d80bb791ce645c1e905 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:15:02 -0400
Subject: [PATCH 124/150] Finish adding new tests for TAXII datastore. closes
 #148

---
 stix2/test/test_datastore_taxii.py | 104 ++++++++++++++++++++++++++---
 1 file changed, 95 insertions(+), 9 deletions(-)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index c20fbb6..dd81cd4 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -1,4 +1,5 @@
-from stix2 import Bundle, ThreatActor, TAXIICollectionSource, TAXIICollectionSink
+from stix2 import (Bundle, ThreatActor, TAXIICollectionSink,
+                   TAXIICollectionSource, TAXIICollectionStore)
 from stix2.datastore.filters import Filter
 
 import json
@@ -27,7 +28,8 @@ class MockTAXIICollectionEndpoint(Collection):
     def get_objects(self, **filter_kwargs):
         self._verify_can_read()
         query_params = _filter_kwargs_to_query_params(filter_kwargs)
-        query_params = json.loads(query_params)
+        if not isinstance(query_params, dict):
+            query_params = json.loads(query_params)
         full_filter = BasicFilter(query_params or {})
         objs = full_filter.process_filter(
             self.objects,
@@ -90,6 +92,80 @@ def test_add_stix2_object(collection):
     tc_sink.add(ta)
 
 
+def test_add_stix2_with_custom_object(collection):
+    tc_sink = TAXIICollectionStore(collection, allow_custom=True)
+
+    # create new STIX threat-actor
+    ta = ThreatActor(name="Teddy Bear",
+                     labels=["nation-state"],
+                     sophistication="innovator",
+                     resource_level="government",
+                     goals=[
+                         "compromising environment NGOs",
+                         "water-hole attacks geared towards energy sector",
+                     ],
+                     foo="bar",
+                     allow_custom=True)
+
+    tc_sink.add(ta)
+
+
+def test_add_list_object(collection, indicator):
+    tc_sink = TAXIICollectionSink(collection)
+
+    # create new STIX threat-actor
+    ta = ThreatActor(name="Teddy Bear",
+                     labels=["nation-state"],
+                     sophistication="innovator",
+                     resource_level="government",
+                     goals=[
+                         "compromising environment NGOs",
+                         "water-hole attacks geared towards energy sector",
+                     ])
+
+    tc_sink.add([ta, indicator])
+
+
+def test_add_stix2_bundle_object(collection):
+    tc_sink = TAXIICollectionSink(collection)
+
+    # create new STIX threat-actor
+    ta = ThreatActor(name="Teddy Bear",
+                     labels=["nation-state"],
+                     sophistication="innovator",
+                     resource_level="government",
+                     goals=[
+                         "compromising environment NGOs",
+                         "water-hole attacks geared towards energy sector",
+                     ])
+
+    tc_sink.add(Bundle(objects=[ta]))
+
+
+def test_add_str_object(collection):
+    tc_sink = TAXIICollectionSink(collection)
+
+    # create new STIX threat-actor
+    ta = """{
+        "type": "threat-actor",
+        "id": "threat-actor--eddff64f-feb1-4469-b07c-499a73c96415",
+        "created": "2018-04-23T16:40:50.847Z",
+        "modified": "2018-04-23T16:40:50.847Z",
+        "name": "Teddy Bear",
+        "goals": [
+            "compromising environment NGOs",
+            "water-hole attacks geared towards energy sector"
+        ],
+        "sophistication": "innovator",
+        "resource_level": "government",
+        "labels": [
+            "nation-state"
+        ]
+    }"""
+
+    tc_sink.add(ta)
+
+
 def test_get_stix2_object(collection):
     tc_sink = TAXIICollectionSource(collection)
 
@@ -98,7 +174,7 @@ def test_get_stix2_object(collection):
     assert objects
 
 
-def test_parse_taxii_filters():
+def test_parse_taxii_filters(collection):
     query = [
         Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
         Filter("id", "=", "taxii stix object ID"),
@@ -107,12 +183,12 @@ def test_parse_taxii_filters():
         Filter("created_by_ref", "=", "Bane"),
     ]
 
-    taxii_filters_expected = set([
+    taxii_filters_expected = [
         Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
         Filter("id", "=", "taxii stix object ID"),
         Filter("type", "=", "taxii stix object ID"),
         Filter("version", "=", "first")
-    ])
+    ]
 
     ds = TAXIICollectionSource(collection)
 
@@ -121,7 +197,7 @@ def test_parse_taxii_filters():
     assert taxii_filters == taxii_filters_expected
 
 
-def test_add_get_remove_filter():
+def test_add_get_remove_filter(collection):
     ds = TAXIICollectionSource(collection)
 
     # First 3 filters are valid, remaining properties are erroneous in some way
@@ -136,20 +212,30 @@ def test_add_get_remove_filter():
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
-    # Addin the same filter again will have no effect since `filters` uses a set
+    # Addin the same filter again will have no effect since `filters` acts
+    # like a set
     ds.filters.add(valid_filters[0])
     assert len(ds.filters) == 1
 
     ds.filters.add(valid_filters[1])
     assert len(ds.filters) == 2
+
     ds.filters.add(valid_filters[2])
     assert len(ds.filters) == 3
 
-    assert set(valid_filters) == ds.filters
+    assert valid_filters == [f for f in ds.filters]
 
     # remove
     ds.filters.remove(valid_filters[0])
 
     assert len(ds.filters) == 2
 
-    ds.filters.update(valid_filters)
+    ds.filters.add(valid_filters)
+
+
+def test_get_all_versions(collection):
+    ds = TAXIICollectionStore(collection)
+
+    indicators = ds.all_versions('indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f')
+    # There are 3 indicators but 2 share the same 'modified' timestamp
+    assert len(indicators) == 2

From 7788ad8583514b62a8f96f37e7103a9f119cbd0f Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:26:26 -0400
Subject: [PATCH 125/150] Add `medallion` dependency to Travis CI tests

---
 .isort.cfg | 1 +
 tox.ini    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.isort.cfg b/.isort.cfg
index cca9d19..56b0448 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -3,6 +3,7 @@ not_skip = __init__.py
 skip = workbench.py
 known_third_party =
     dateutil,
+    medallion,
     ordereddict,
     pytest,
     pytz,
diff --git a/tox.ini b/tox.ini
index 7c2e68c..683e137 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,6 +9,7 @@ deps =
   pytest-cov
   coverage
   taxii2-client
+  medallion
 commands =
   pytest --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
   pytest stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append

From 7e5c8a94096bdf90dc265d314f5fecf7974ab64d Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:27:36 -0400
Subject: [PATCH 126/150] Sort import statements in tests.

---
 stix2/test/test_datastore_filesystem.py |  1 -
 stix2/test/test_datastore_filters.py    |  1 -
 stix2/test/test_datastore_memory.py     |  2 +-
 stix2/test/test_datastore_taxii.py      | 12 ++++++------
 4 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py
index bd1f02c..4139ab1 100644
--- a/stix2/test/test_datastore_filesystem.py
+++ b/stix2/test/test_datastore_filesystem.py
@@ -7,7 +7,6 @@ import pytest
 from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink,
                    FileSystemSource, FileSystemStore, Filter, Identity,
                    Indicator, Malware, Relationship, properties)
-
 from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
                                   IDENTITY_KWARGS, INDICATOR_ID,
                                   INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS,
diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py
index 8803e4b..5ffd051 100644
--- a/stix2/test/test_datastore_filters.py
+++ b/stix2/test/test_datastore_filters.py
@@ -4,7 +4,6 @@ from stix2 import parse
 from stix2.datastore.filters import Filter, apply_common_filters
 from stix2.utils import STIXdatetime, parse_into_datetime
 
-
 stix_objs = [
     {
         "created": "2017-01-27T13:49:53.997Z",
diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
index dcec813..7a5bf10 100644
--- a/stix2/test/test_datastore_memory.py
+++ b/stix2/test/test_datastore_memory.py
@@ -2,7 +2,7 @@ import pytest
 
 from stix2.datastore import CompositeDataSource, make_id
 from stix2.datastore.filters import Filter
-from stix2.datastore.memory import MemorySource, MemorySink
+from stix2.datastore.memory import MemorySink, MemorySource
 
 
 def test_add_remove_composite_datasource():
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index dd81cd4..f3b4f64 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -1,12 +1,12 @@
-from stix2 import (Bundle, ThreatActor, TAXIICollectionSink,
-                   TAXIICollectionSource, TAXIICollectionStore)
-from stix2.datastore.filters import Filter
-
 import json
-import pytest
 
-from taxii2client import Collection, _filter_kwargs_to_query_params
 from medallion.filters.basic_filter import BasicFilter
+import pytest
+from taxii2client import Collection, _filter_kwargs_to_query_params
+
+from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource,
+                   TAXIICollectionStore, ThreatActor)
+from stix2.datastore.filters import Filter
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
 

From 2043a514e14fb77435e5638f2b59d17f3747540b Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:47:32 -0400
Subject: [PATCH 127/150] Add two more tests for the `dict` case in
 TAXIICollectionSink

---
 stix2/test/test_datastore_taxii.py | 53 ++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index f3b4f64..098f944 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -166,6 +166,59 @@ def test_add_str_object(collection):
     tc_sink.add(ta)
 
 
+def test_add_dict_object(collection):
+    tc_sink = TAXIICollectionSink(collection)
+
+    ta = {
+        "type": "threat-actor",
+        "id": "threat-actor--eddff64f-feb1-4469-b07c-499a73c96415",
+        "created": "2018-04-23T16:40:50.847Z",
+        "modified": "2018-04-23T16:40:50.847Z",
+        "name": "Teddy Bear",
+        "goals": [
+            "compromising environment NGOs",
+            "water-hole attacks geared towards energy sector"
+        ],
+        "sophistication": "innovator",
+        "resource_level": "government",
+        "labels": [
+            "nation-state"
+        ]
+    }
+
+    tc_sink.add(ta)
+
+
+def test_add_dict_bundle_object(collection):
+    tc_sink = TAXIICollectionSink(collection)
+
+    ta = {
+        "type": "bundle",
+        "id": "bundle--860ccc8d-56c9-4fda-9384-84276fb52fb1",
+        "spec_version": "2.0",
+        "objects": [
+            {
+                "type": "threat-actor",
+                "id": "threat-actor--dc5a2f41-f76e-425a-81fe-33afc7aabd75",
+                "created": "2018-04-23T18:45:11.390Z",
+                "modified": "2018-04-23T18:45:11.390Z",
+                "name": "Teddy Bear",
+                "goals": [
+                    "compromising environment NGOs",
+                    "water-hole attacks geared towards energy sector"
+                ],
+                "sophistication": "innovator",
+                "resource_level": "government",
+                "labels": [
+                    "nation-state"
+                ]
+            }
+        ]
+    }
+
+    tc_sink.add(ta)
+
+
 def test_get_stix2_object(collection):
     tc_sink = TAXIICollectionSource(collection)
 

From c15267971d7508331f956daddd1576df461663b6 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 14:55:06 -0400
Subject: [PATCH 128/150] Remove 'real_stix_objs1' as it is unused in the tests

---
 stix2/test/conftest.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py
index ddbcb7e..c73eafb 100644
--- a/stix2/test/conftest.py
+++ b/stix2/test/conftest.py
@@ -154,11 +154,6 @@ def stix_objs2():
     return [ind6, ind7, ind8]
 
 
-@pytest.fixture
-def real_stix_objs1(stix_objs1):
-    return [stix2.parse(x) for x in stix_objs1]
-
-
 @pytest.fixture
 def real_stix_objs2(stix_objs2):
     return [stix2.parse(x) for x in stix_objs2]

From 410296e6e1ca617fdb340dba26b2a150af747c9f Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Mon, 23 Apr 2018 15:22:38 -0400
Subject: [PATCH 129/150] Update file taxii_example.py

---
 examples/taxii_example.py | 57 +++++++++++++++------------------------
 1 file changed, 21 insertions(+), 36 deletions(-)

diff --git a/examples/taxii_example.py b/examples/taxii_example.py
index e102b17..978f0f2 100644
--- a/examples/taxii_example.py
+++ b/examples/taxii_example.py
@@ -1,54 +1,39 @@
-import json
+from taxii2client import Collection
 
-from stix2.datastore.taxii import TAXIIDataSource
+import stix2
 
-# Flask TAXII server - developmental
-ROOT = 'http://localhost:5000'
-AUTH = {'user': 'mk', 'pass': 'Pass'}
+# This example is based on the medallion server with default_data.json
+# See https://github.com/oasis-open/cti-taxii-server for more information
 
 
 def main():
+    collection = Collection("http://127.0.0.1:5000/trustgroup1/collections/52892447-4d7e-4f70-b94d-d7f22742ff63/",
+                            user="admin", password="Password0")
 
     # instantiate TAXII data source
-    taxii = TAXIIDataSource(api_root=ROOT, auth=AUTH)
+    taxii = stix2.TAXIICollectionSource(collection)
 
-    # get (file watch indicator)
-    indicator_fw = taxii.get(id_="indicator--a932fcc6-e032-176c-126f-cb970a5a1ade")
+    # get (url watch indicator)
+    indicator_fw = taxii.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
     print("\n\n-------Queried for Indicator - got:")
-    print(json.dumps(indicator_fw, indent=4))
+    print(indicator_fw.serialize(indent=4))
 
-    # all versions (file watch indicator - currently only 1. maybe Emmanuelle can add a version)
-    indicator_fw_versions = taxii.get(id_="indicator--a932fcc6-e032-176c-126f-cb970a5a1ade")
+    # all versions (url watch indicator - currently two)
+    indicator_fw_versions = taxii.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
     print("\n\n------Queried for indicator (all_versions()) - got:")
-    print(json.dumps(indicator_fw_versions, indent=4))
+    for indicator in indicator_fw_versions:
+        print(indicator.serialize(indent=4))
 
     # add TAXII filter (ie filter should be passed to TAXII)
-    taxii_filter_ids, status = taxii.add_filter(
-        [
-            {
-                "field": "type",
-                "op": "in",
-                "value": "malware"
-            }
-        ])
+    query_filter = stix2.Filter("type", "in", "malware")
 
-    print("\n\n-------Added filter:")
-    print("Filter ID: {0}".format(taxii_filter_ids[0]))
-    print("Filter status: \n")
-    print(json.dumps(status, indent=4))
-    print("filters: \n")
-    print(json.dumps(taxii.get_filters(), indent=4))
-
-    # get() - but with filter attached
-    malware = taxii.query()
+    # query() - but with filter attached. There are no malware objects in this collection
+    malwares = taxii.query(query=query_filter)
     print("\n\n\n--------Queried for Malware string (with above filter attached) - got:")
-    print(json.dumps(malware, indent=4))
-
-    # remove TAXII filter
-    taxii.remove_filter(taxii_filter_ids)
-    print("\n\n-------Removed filter(TAXII filter):")
-    print("filters: \n")
-    print(json.dumps(taxii.get_filters(), indent=4))
+    for malware in malwares:
+        print(malware.serialize(indent=4))
+    if not malwares:
+        print(malwares)
 
 
 if __name__ == "__main__":

From 5d96cf11f131a41b0e4398d238767e753290f9ec Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Tue, 24 Apr 2018 15:55:46 -0400
Subject: [PATCH 130/150] Update TAXII2 filter fields in documentation

---
 docs/guide/datastore.ipynb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index c3980a0..2fabf88 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -356,16 +356,16 @@
     "TAXII2 filter fields:\n",
     "\n",
     "* added_after\n",
-    "* match[id]\n",
-    "* match[type]\n",
-    "* match[version]\n",
+    "* id\n",
+    "* type\n",
+    "* version\n",
     "\n",
     "Supported operators:\n",
     "\n",
     "* =\n",
     "* !=\n",
     "* in\n",
-    "* >\n",
+    "* ```>```\n",
     "* < \n",
     "* ```>=```\n",
     "* <=\n",

From f778a45b33f9b74c5a3b62c38db477bb504c2202 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Wed, 25 Apr 2018 14:29:22 -0500
Subject: [PATCH 131/150] Remove dev and nightly Python builds from Travis.

---
 .travis.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7c3a4bb..28728ff 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,13 +5,7 @@ python:
   - "2.7"
   - "3.4"
   - "3.5"
-  - "3.5-dev"
   - "3.6"
-  - "3.6-dev"
-  - "nightly"
-matrix:
-  allow_failures:
-    - python: "nightly"
 install:
   - pip install -U pip setuptools
   - pip install tox-travis pre-commit

From 4229c380a2be444cf617317608007b0cae44a603 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 26 Apr 2018 10:20:46 -0400
Subject: [PATCH 132/150] Fix make_constant when value is already a constant

Fixes #171.
---
 stix2/patterns.py                      | 3 +++
 stix2/test/test_pattern_expressions.py | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/stix2/patterns.py b/stix2/patterns.py
index 23ce71b..3f9cbd9 100644
--- a/stix2/patterns.py
+++ b/stix2/patterns.py
@@ -147,6 +147,9 @@ class ListConstant(_Constant):
 
 
 def make_constant(value):
+    if isinstance(value, _Constant):
+        return value
+
     try:
         return parse_into_datetime(value)
     except ValueError:
diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py
index 74a7d0f..14e3774 100644
--- a/stix2/test/test_pattern_expressions.py
+++ b/stix2/test/test_pattern_expressions.py
@@ -372,3 +372,9 @@ def test_invalid_startstop_qualifier():
         stix2.StartStopQualifier(datetime.date(2016, 6, 1),
                                  'foo')
     assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo)
+
+
+def test_make_constant_already_a_constant():
+    str_const = stix2.StringConstant('Foo')
+    result = stix2.patterns.make_constant(str_const)
+    assert result is str_const

From ab513be21789454644490502ed7d890f99305a45 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 27 Apr 2018 16:45:18 -0400
Subject: [PATCH 133/150] Update CHANGELOG for v1.0.1

---
 CHANGELOG | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index f63f682..8e2be4c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+1.0.1 - 2018-04-27
+
+* Fixes bug with incorrect TAXII parameters (#169).
+* Fixes bug with constructing patterns (#171).
+
 1.0.0 - 2018-04-16
 
 * Adds the Workbench layer API.

From 102623b29af80fdfb159af6cac805e93e0fa1784 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 27 Apr 2018 16:51:47 -0400
Subject: [PATCH 134/150] =?UTF-8?q?Bump=20version:=201.0.0=20=E2=86=92=201?=
 =?UTF-8?q?.0.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/conf.py     | 4 ++--
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 85c8b23..9a2f24e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,8 +34,8 @@ project = 'stix2'
 copyright = '2017, OASIS Open'
 author = 'OASIS Open'
 
-version = '1.0.0'
-release = '1.0.0'
+version = '1.0.1'
+release = '1.0.1'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints']
diff --git a/setup.cfg b/setup.cfg
index ac3e5d5..e9787b8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.0
+current_version = 1.0.1
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index 5becc17..5c4105c 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.0.1"

From 3f80c07342e3d7eebfc6c1e796d4881510a981a2 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Wed, 2 May 2018 12:37:44 -0500
Subject: [PATCH 135/150] Add tox checks for README.

---
 tox.ini | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 683e137..14d379c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,py35,py36,style,isort-check
+envlist = py27,py34,py35,py36,style,isort-check,packaging
 
 [testenv]
 deps =
@@ -31,9 +31,15 @@ commands =
   isort -rc stix2 examples -df
   isort -rc stix2 examples -c
 
+[testenv:packaging]
+deps =
+  readme_renderer
+commands =
+  python setup.py check -r -s
+
 [travis]
 python =
   2.7: py27, style
   3.4: py34, style
   3.5: py35, style
-  3.6: py36, style
+  3.6: py36, style, packaging

From 06e5a33639cd3a9b4880f3845def33564d118a58 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 11 May 2018 17:28:55 -0400
Subject: [PATCH 136/150] Pass allow_custom when adding to ObservedData

---
 stix2/core.py                |  4 ++--
 stix2/test/test_custom.py    | 18 ++++++++++++++++++
 stix2/test/test_workbench.py |  8 +++++++-
 stix2/v20/__init__.py        |  7 ++++---
 stix2/v20/observables.py     |  9 ++++++++-
 stix2/v20/sdo.py             |  6 ++++++
 stix2/workbench.py           | 17 +++++++++++++++--
 7 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/stix2/core.py b/stix2/core.py
index 0b222a9..0c8b7b0 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -14,9 +14,9 @@ from .utils import _get_dict, get_class_hierarchy_names
 
 class STIXObjectProperty(Property):
 
-    def __init__(self, allow_custom=False):
+    def __init__(self, allow_custom=False, *args, **kwargs):
         self.allow_custom = allow_custom
-        super(STIXObjectProperty, self).__init__()
+        super(STIXObjectProperty, self).__init__(*args, **kwargs)
 
     def clean(self, value):
         # Any STIX Object (SDO, SRO, or Marking Definition) can be added to
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 0e004a8..6bf7fb2 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -88,6 +88,24 @@ def test_parse_identity_custom_property(data):
     assert identity.foo == "bar"
 
 
+def test_custom_property_in_observed_data():
+    artifact = stix2.File(
+        allow_custom=True,
+        name='test',
+        x_foo='bar'
+    )
+    observed_data = stix2.ObservedData(
+        allow_custom=True,
+        first_observed="2015-12-21T19:00:00Z",
+        last_observed="2015-12-21T19:00:00Z",
+        number_observed=0,
+        objects={"0": artifact},
+    )
+
+    assert observed_data.objects['0'].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(observed_data)
+
+
 def test_custom_property_in_bundled_object():
     bundle = stix2.Bundle(IDENTITY_CUSTOM_PROP, allow_custom=True)
 
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 7857eb2..501d11c 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -1,7 +1,7 @@
 import os
 
 import stix2
-from stix2.workbench import (AttackPattern, Campaign, CourseOfAction,
+from stix2.workbench import (AttackPattern, Bundle, Campaign, CourseOfAction,
                              ExternalReference, FileSystemSource, Filter,
                              Identity, Indicator, IntrusionSet, Malware,
                              MarkingDefinition, ObservedData, Relationship,
@@ -149,6 +149,12 @@ def test_workbench_get_all_vulnerabilities():
     assert resp[0].id == VULNERABILITY_ID
 
 
+def test_workbench_add_to_bundle():
+    vuln = Vulnerability(**VULNERABILITY_KWARGS)
+    bundle = Bundle(vuln)
+    assert bundle.objects[0].name == 'Heartbleed'
+
+
 def test_workbench_relationships():
     rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID)
     save(rel)
diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py
index 888e1ca..9d7efcc 100644
--- a/stix2/v20/__init__.py
+++ b/stix2/v20/__init__.py
@@ -10,9 +10,10 @@ from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact,
                           Directory, DomainName, EmailAddress, EmailMessage,
                           EmailMIMEComponent, ExtensionsProperty, File,
                           HTTPRequestExt, ICMPExt, IPv4Address, IPv6Address,
-                          MACAddress, Mutex, NetworkTraffic, NTFSExt, PDFExt,
-                          Process, RasterImageExt, SocketExt, Software, TCPExt,
-                          UNIXAccountExt, UserAccount, WindowsPEBinaryExt,
+                          MACAddress, Mutex, NetworkTraffic, NTFSExt,
+                          ObservableProperty, PDFExt, Process, RasterImageExt,
+                          SocketExt, Software, TCPExt, UNIXAccountExt,
+                          UserAccount, WindowsPEBinaryExt,
                           WindowsPEOptionalHeaderType, WindowsPESection,
                           WindowsProcessExt, WindowsRegistryKey,
                           WindowsRegistryValueType, WindowsServiceExt,
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 6c65e8e..7bce53e 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -24,6 +24,10 @@ class ObservableProperty(Property):
     """Property for holding Cyber Observable Objects.
     """
 
+    def __init__(self, allow_custom=False, *args, **kwargs):
+        self.allow_custom = allow_custom
+        super(ObservableProperty, self).__init__(*args, **kwargs)
+
     def clean(self, value):
         try:
             dictified = _get_dict(value)
@@ -39,7 +43,10 @@ class ObservableProperty(Property):
         valid_refs = dict((k, v['type']) for (k, v) in dictified.items())
 
         for key, obj in dictified.items():
-            parsed_obj = parse_observable(obj, valid_refs)
+            if self.allow_custom:
+                parsed_obj = parse_observable(obj, valid_refs, allow_custom=True)
+            else:
+                parsed_obj = parse_observable(obj, valid_refs)
             dictified[key] = parsed_obj
 
         return dictified
diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index d7e9954..9ca1599 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -225,6 +225,12 @@ class ObservedData(STIXDomainObject):
         ('granular_markings', ListProperty(GranularMarking)),
     ])
 
+    def __init__(self, allow_custom=False, *args, **kwargs):
+        if allow_custom:
+            self._properties['objects'] = ObservableProperty(True)
+
+        super(ObservedData, self).__init__(*args, **kwargs)
+
 
 class Report(STIXDomainObject):
     """For more detailed information on this object's properties, see
diff --git a/stix2/workbench.py b/stix2/workbench.py
index 3697c63..c559f40 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -49,6 +49,7 @@ from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # n
                WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
                WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
 from .datastore.filters import FilterSet
+from . import ObservableProperty
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -110,14 +111,26 @@ def _related_wrapper(self, *args, **kwargs):
     return _environ.related_to(self, *args, **kwargs)
 
 
+def _observed_data_init(self, allow_custom=False, *args, **kwargs):
+    if allow_custom:
+        self._properties['objects'] = ObservableProperty(True)
+    super(self.__class__, self).__init__(*args, **kwargs)
+
+
 def _constructor_wrapper(obj_type):
     # Use an intermediate wrapper class so the implicit environment will create objects that have our wrapper functions
-    wrapped_type = type(obj_type.__name__, obj_type.__bases__, dict(
+    class_dict = dict(
         created_by=_created_by_wrapper,
         relationships=_relationships_wrapper,
         related=_related_wrapper,
         **obj_type.__dict__
-    ))
+    )
+
+    # Avoid TypeError about super() in ObservedData
+    if 'ObservedData' in obj_type.__name__:
+        class_dict['__init__'] = _observed_data_init
+
+    wrapped_type = type(obj_type.__name__, obj_type.__bases__, class_dict)
 
     @staticmethod
     def new_constructor(cls, *args, **kwargs):

From 239291253383639594d2df6ebf83c7252b0548b0 Mon Sep 17 00:00:00 2001
From: = 
Date: Tue, 15 May 2018 11:28:34 -0400
Subject: [PATCH 137/150] handle TAXII client/server errors according to
 decided policy

---
 stix2/datastore/taxii.py           | 63 +++++++++++++++++---
 stix2/test/test_datastore_taxii.py | 96 +++++++++++++++++++++++++++++-
 2 files changed, 149 insertions(+), 10 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 872a510..e3f4e45 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -51,7 +51,27 @@ class TAXIICollectionSink(DataSink):
     """
     def __init__(self, collection, allow_custom=False):
         super(TAXIICollectionSink, self).__init__()
-        self.collection = collection
+        try:
+            # we have to execute .can_write first in isolation because the
+            # attribute access could trigger a taxii2client.ValidationError which
+            # we catch here as a ValueError (its parent class). Later, we need to
+            # have the ability to also raise a different ValueError based on the
+            # value of .can_write
+            writeable = collection.can_write
+
+        except (HTTPError, ValueError) as e:
+            e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
+                         " Collection object provided could not be reached. TAXII Collection Error: "
+                         + e.message)
+            raise
+
+        if writeable:
+            # now past taxii2client possible exceptions, check value for local exceptions
+            self.collection = collection
+        else:
+            raise ValueError("The TAXII Collection object provided does not have write access"
+                             " to the underlying linked Collection resource")
+
         self.allow_custom = allow_custom
 
     def add(self, stix_data, version=None):
@@ -111,7 +131,27 @@ class TAXIICollectionSource(DataSource):
     """
     def __init__(self, collection, allow_custom=True):
         super(TAXIICollectionSource, self).__init__()
-        self.collection = collection
+        try:
+            # we have to execute .can_read first in isolation because the
+            # attribute access could trigger a taxii2client.ValidationError which
+            # we catch here as a ValueError (its parent class). Later, we need to
+            # have the ability to also raise a different ValueError based on the
+            # value of .can_read
+            writeable = collection.can_read
+
+        except (HTTPError, ValueError) as e:
+            e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
+                         " Collection object provided could not be reached. TAXII Collection Error: "
+                         + e.message)
+            raise
+
+        if writeable:
+            # now past taxii2client possible exceptions, check value for local exceptions
+            self.collection = collection
+        else:
+            raise ValueError("The TAXII Collection object provided does not have read access"
+                             " to the underlying linked Collection resource")
+
         self.allow_custom = allow_custom
 
     def get(self, stix_id, version=None, _composite_filters=None):
@@ -145,9 +185,12 @@ class TAXIICollectionSource(DataSource):
             stix_objs = self.collection.get_object(stix_id)["objects"]
             stix_obj = list(apply_common_filters(stix_objs, query))
 
-        except HTTPError:
-            # if resource not found or access is denied from TAXII server, return None
-            stix_obj = []
+        except HTTPError as err:
+            if err.response.status_code == 404:
+                # if resource not found or access is denied from TAXII server, return None
+                stix_obj = []
+            else:
+                raise
 
         if len(stix_obj):
             stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version)
@@ -231,13 +274,17 @@ class TAXIICollectionSource(DataSource):
             # deduplicate data (before filtering as reduces wasted filtering)
             all_data = deduplicate(all_data)
 
-            # apply local (CompositeDataSource, TAXIICollectionSource and query) filters
+            # a pply local (CompositeDataSource, TAXIICollectionSource and query) filters
             query.remove(taxii_filters)
             all_data = list(apply_common_filters(all_data, query))
 
-        except HTTPError:
+        except HTTPError as err:
             # if resources not found or access is denied from TAXII server, return empty list
-            all_data = []
+            if err.response.status_code == 404:
+                err.message = ("The requested STIX objects for the TAXII Collection resource defined in"
+                               " the supplied TAXII Collection object is either not found or access is"
+                               " denied. Received error: " + err.message)
+            raise
 
         # parse python STIX objects from the STIX object dicts
         stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index 098f944..c753afc 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -2,6 +2,8 @@ import json
 
 from medallion.filters.basic_filter import BasicFilter
 import pytest
+from requests.exceptions import HTTPError
+from requests.models import Response
 from taxii2client import Collection, _filter_kwargs_to_query_params
 
 from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource,
@@ -36,7 +38,12 @@ class MockTAXIICollectionEndpoint(Collection):
             ("id", "type", "version"),
             []
         )
-        return Bundle(objects=objs)
+        if objs:
+            return Bundle(objects=objs)
+        else:
+            resp = Response()
+            resp.status_code = 404
+            resp.raise_for_status()
 
     def get_object(self, id, version=None):
         self._verify_can_read()
@@ -51,7 +58,12 @@ class MockTAXIICollectionEndpoint(Collection):
             ("version",),
             []
         )
-        return Bundle(objects=objs)
+        if objs:
+            return Bundle(objects=objs)
+        else:
+            resp = Response()
+            resp.status_code = 404
+            resp.raise_for_status()
 
 
 @pytest.fixture
@@ -71,6 +83,23 @@ def collection(stix_objs1):
     return mock
 
 
+@pytest.fixture
+def collection_no_rw_access(stix_objs1):
+    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{
+        "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116",
+        "title": "Not writeable or readable Collection",
+        "description": "This collection is a dropbox for submitting indicators",
+        "can_read": False,
+        "can_write": False,
+        "media_types": [
+            "application/vnd.oasis.stix+json; version=2.0"
+        ]
+    })
+
+    mock.objects.extend(stix_objs1)
+    return mock
+
+
 def test_ds_taxii(collection):
     ds = TAXIICollectionSource(collection)
     assert ds.collection is not None
@@ -292,3 +321,66 @@ def test_get_all_versions(collection):
     indicators = ds.all_versions('indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f')
     # There are 3 indicators but 2 share the same 'modified' timestamp
     assert len(indicators) == 2
+
+
+def test_can_read_error(collection_no_rw_access):
+    """create a TAXIICOllectionSource with a taxii2client.Collection
+    instance that does not have read access, check ValueError exception is raised"""
+    with pytest.raises(ValueError) as excinfo:
+        TAXIICollectionSource(collection_no_rw_access)
+    assert "Collection object provided does not have read access" in str(excinfo.value)
+
+
+def test_can_write_error(collection_no_rw_access):
+    """create a TAXIICOllectionSink with a taxii2client.Collection
+    instance that does not have write access, check ValueError exception is raised"""
+    with pytest.raises(ValueError) as excinfo:
+        TAXIICollectionSink(collection_no_rw_access)
+    assert "Collection object provided does not have write access" in str(excinfo.value)
+
+
+def test_bad_collection():
+    """this triggers a real connectivity issue (HTTPError: 503 ServerError) """
+    with pytest.raises(HTTPError) as excinfo:
+        mock = MockTAXIICollectionEndpoint("http://doenstexist118482.org", verify=False)
+        TAXIICollectionStore(mock)
+    assert "Collection object provided could not be reached. TAXII Collection Error:" in str(excinfo.value.message)
+    assert "HTTPError" in str(excinfo.type)
+
+
+def test_get_404(collection):
+    """a TAXIICollectionSource.get() call that receives an HTTP 404 response
+    code from the taxii2client should be be returned as None.
+
+    TAXII spec states that a TAXII server can return a 404 for
+    nonexistent resources or lack of access. Decided that None is acceptable
+    reponse to imply that state of the TAXII endpoint.
+    """
+    ds = TAXIICollectionStore(collection)
+
+    # this will raise 404 from mock TAXII Client but TAXIICollectionStore
+    # should handle gracefully and return None
+    stix_obj = ds.get("indicator--1")  # this will raise 404 from
+    assert stix_obj is None
+
+
+def test_all_versions_404(collection):
+    """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404
+    response code from the taxii2client should be returned as an exception"""
+    ds = TAXIICollectionStore(collection)
+    with pytest.raises(HTTPError) as excinfo:
+        ds.all_versions("indicator--1")
+    assert "is either not found or access is denied" in str(excinfo.value.message)
+    assert "404" in str(excinfo.value)
+
+
+def test_query_404(collection):
+    """ a TAXIICollectionSource.query() call that recieves an HTTP 404
+    response code from the taxii2client should be returned as an exception"""
+    ds = TAXIICollectionStore(collection)
+    query = [Filter("type", "=", "malware")]
+
+    with pytest.raises(HTTPError) as excinfo:
+        ds.query(query=query)
+    assert "is either not found or access is denied" in str(excinfo.value.message)
+    assert "404" in str(excinfo.value)

From 0d3f80f2fec627ca00e72d031c15d38f2352e1a6 Mon Sep 17 00:00:00 2001
From: = 
Date: Tue, 15 May 2018 15:41:46 -0400
Subject: [PATCH 138/150] removing taxii 503 error test as not reproducible in
 Travis environment

---
 stix2/datastore/taxii.py           | 57 ++++++++++++++----------------
 stix2/test/test_datastore_taxii.py |  9 -----
 2 files changed, 27 insertions(+), 39 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index e3f4e45..776baf7 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -9,6 +9,13 @@ from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate
 
+try:
+    from taxii2client import ValidationError
+    _taxii2_client = True
+except ImportError:
+    _taxii2_client = False
+
+
 TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
 
 
@@ -51,27 +58,22 @@ class TAXIICollectionSink(DataSink):
     """
     def __init__(self, collection, allow_custom=False):
         super(TAXIICollectionSink, self).__init__()
-        try:
-            # we have to execute .can_write first in isolation because the
-            # attribute access could trigger a taxii2client.ValidationError which
-            # we catch here as a ValueError (its parent class). Later, we need to
-            # have the ability to also raise a different ValueError based on the
-            # value of .can_write
-            writeable = collection.can_write
+        if not _taxii2_client:
+            raise ImportError("taxii2client library is required for usage of TAXIICollectionSink")
 
-        except (HTTPError, ValueError) as e:
+        try:
+            if collection.can_write:
+                self.collection = collection
+            else:
+                raise ValueError("The TAXII Collection object provided does not have write access"
+                                 " to the underlying linked Collection resource")
+
+        except (HTTPError, ValidationError) as e:
             e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
                          " Collection object provided could not be reached. TAXII Collection Error: "
                          + e.message)
             raise
 
-        if writeable:
-            # now past taxii2client possible exceptions, check value for local exceptions
-            self.collection = collection
-        else:
-            raise ValueError("The TAXII Collection object provided does not have write access"
-                             " to the underlying linked Collection resource")
-
         self.allow_custom = allow_custom
 
     def add(self, stix_data, version=None):
@@ -131,27 +133,22 @@ class TAXIICollectionSource(DataSource):
     """
     def __init__(self, collection, allow_custom=True):
         super(TAXIICollectionSource, self).__init__()
-        try:
-            # we have to execute .can_read first in isolation because the
-            # attribute access could trigger a taxii2client.ValidationError which
-            # we catch here as a ValueError (its parent class). Later, we need to
-            # have the ability to also raise a different ValueError based on the
-            # value of .can_read
-            writeable = collection.can_read
+        if not _taxii2_client:
+            raise ImportError("taxii2client library is required for usage of TAXIICollectionSource")
 
-        except (HTTPError, ValueError) as e:
+        try:
+            if collection.can_read:
+                self.collection = collection
+            else:
+                raise ValueError("The TAXII Collection object provided does not have read access"
+                                 " to the underlying linked Collection resource")
+
+        except (HTTPError, ValidationError) as e:
             e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
                          " Collection object provided could not be reached. TAXII Collection Error: "
                          + e.message)
             raise
 
-        if writeable:
-            # now past taxii2client possible exceptions, check value for local exceptions
-            self.collection = collection
-        else:
-            raise ValueError("The TAXII Collection object provided does not have read access"
-                             " to the underlying linked Collection resource")
-
         self.allow_custom = allow_custom
 
     def get(self, stix_id, version=None, _composite_filters=None):
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index c753afc..af8fecd 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -339,15 +339,6 @@ def test_can_write_error(collection_no_rw_access):
     assert "Collection object provided does not have write access" in str(excinfo.value)
 
 
-def test_bad_collection():
-    """this triggers a real connectivity issue (HTTPError: 503 ServerError) """
-    with pytest.raises(HTTPError) as excinfo:
-        mock = MockTAXIICollectionEndpoint("http://doenstexist118482.org", verify=False)
-        TAXIICollectionStore(mock)
-    assert "Collection object provided could not be reached. TAXII Collection Error:" in str(excinfo.value.message)
-    assert "HTTPError" in str(excinfo.type)
-
-
 def test_get_404(collection):
     """a TAXIICollectionSource.get() call that receives an HTTP 404 response
     code from the taxii2client should be be returned as None.

From 2b4c5bf264d15ba5dfbc1151fe80a34de00c0a91 Mon Sep 17 00:00:00 2001
From: = 
Date: Tue, 15 May 2018 17:42:19 -0400
Subject: [PATCH 139/150] handling and modifying exception messages in a manner
 acceptable by all python versions

---
 stix2/datastore/taxii.py           | 35 ++++++++++++++++++++----------
 stix2/test/test_datastore_taxii.py |  4 ++--
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 776baf7..4a047a8 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -69,9 +69,13 @@ class TAXIICollectionSink(DataSink):
                                  " to the underlying linked Collection resource")
 
         except (HTTPError, ValidationError) as e:
-            e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
-                         " Collection object provided could not be reached. TAXII Collection Error: "
-                         + e.message)
+            added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII"
+                             " Collection object provided could not be reached.")
+            if not e.args:
+                e.args = (added_context,)
+            else:
+                e.args = (added_context,) + e.args
+
             raise
 
         self.allow_custom = allow_custom
@@ -144,9 +148,13 @@ class TAXIICollectionSource(DataSource):
                                  " to the underlying linked Collection resource")
 
         except (HTTPError, ValidationError) as e:
-            e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII"
-                         " Collection object provided could not be reached. TAXII Collection Error: "
-                         + e.message)
+            added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII"
+                             " Collection object provided could not be reached.")
+            if not e.args:
+                e.args = (added_context,)
+            else:
+                e.args = (added_context,) + e.args
+
             raise
 
         self.allow_custom = allow_custom
@@ -275,12 +283,17 @@ class TAXIICollectionSource(DataSource):
             query.remove(taxii_filters)
             all_data = list(apply_common_filters(all_data, query))
 
-        except HTTPError as err:
+        except HTTPError as e:
             # if resources not found or access is denied from TAXII server, return empty list
-            if err.response.status_code == 404:
-                err.message = ("The requested STIX objects for the TAXII Collection resource defined in"
-                               " the supplied TAXII Collection object is either not found or access is"
-                               " denied. Received error: " + err.message)
+            if e.response.status_code == 404:
+                added_context = ("The requested STIX objects for the TAXII Collection resource defined in"
+                                 " the supplied TAXII Collection object is either not found or access is"
+                                 " denied. Received error: ")
+            if not e.args:
+                e.args = (added_context,)
+            else:
+                e.args = (added_context,) + e.args
+
             raise
 
         # parse python STIX objects from the STIX object dicts
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index af8fecd..a153ce8 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -361,7 +361,7 @@ def test_all_versions_404(collection):
     ds = TAXIICollectionStore(collection)
     with pytest.raises(HTTPError) as excinfo:
         ds.all_versions("indicator--1")
-    assert "is either not found or access is denied" in str(excinfo.value.message)
+    assert "is either not found or access is denied" in str(excinfo.value)
     assert "404" in str(excinfo.value)
 
 
@@ -373,5 +373,5 @@ def test_query_404(collection):
 
     with pytest.raises(HTTPError) as excinfo:
         ds.query(query=query)
-    assert "is either not found or access is denied" in str(excinfo.value.message)
+    assert "is either not found or access is denied" in str(excinfo.value)
     assert "404" in str(excinfo.value)

From 275ca199983d454fbd43417177b8f4cfd953c067 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Tue, 15 May 2018 17:16:23 -0500
Subject: [PATCH 140/150] Clean up isort settings.

---
 .isort.cfg | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.isort.cfg b/.isort.cfg
index 56b0448..0780dcd 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,14 +1,13 @@
 [settings]
-not_skip = __init__.py
 skip = workbench.py
+not_skip = __init__.py
 known_third_party =
     dateutil,
     medallion,
-    ordereddict,
     pytest,
     pytz,
     requests,
-    simplejson
+    simplejson,
     six,
     sphinx,
     stix2patterns,

From 69c31ca3fc5a614c9220fc198b524e531f3d972a Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 16 May 2018 12:14:33 -0400
Subject: [PATCH 141/150] Pass allow_custom to ExtensionsProperty

Also fix bug caused by _properties being a class variable rather than an
instance variable. If you created an object with allow_custom,
allow_custom would be set for all future instances.
---
 stix2/base.py                |  9 ++---
 stix2/core.py                |  5 ++-
 stix2/test/test_custom.py    | 66 +++++++++++++++++++++++++++++++++---
 stix2/test/test_workbench.py | 46 +++++++++++++++++++++++++
 stix2/v20/observables.py     | 10 ++++--
 stix2/v20/sdo.py             |  6 ++--
 stix2/workbench.py           |  7 ++--
 7 files changed, 129 insertions(+), 20 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index e128d48..bb8cd2f 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -264,10 +264,11 @@ class _Observable(_STIXBase):
 
     def __init__(self, **kwargs):
         # the constructor might be called independently of an observed data object
-        if '_valid_refs' in kwargs:
-            self._STIXBase__valid_refs = kwargs.pop('_valid_refs')
-        else:
-            self._STIXBase__valid_refs = []
+        self._STIXBase__valid_refs = kwargs.pop('_valid_refs', [])
+
+        self.__allow_custom = kwargs.get('allow_custom', False)
+        self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False)
+
         super(_Observable, self).__init__(**kwargs)
 
     def _check_ref(self, ref, prop, prop_name):
diff --git a/stix2/core.py b/stix2/core.py
index 0c8b7b0..6c2e23a 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -62,9 +62,8 @@ class Bundle(_STIXBase):
             else:
                 kwargs['objects'] = list(args) + kwargs.get('objects', [])
 
-        allow_custom = kwargs.get('allow_custom', False)
-        if allow_custom:
-            self._properties['objects'] = ListProperty(STIXObjectProperty(True))
+        self.__allow_custom = kwargs.get('allow_custom', False)
+        self._properties['objects'].allow_custom = kwargs.get('allow_custom', False)
 
         super(Bundle, self).__init__(**kwargs)
 
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 6bf7fb2..59cbea8 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -88,6 +88,13 @@ def test_parse_identity_custom_property(data):
     assert identity.foo == "bar"
 
 
+def test_custom_property_in_bundled_object():
+    bundle = stix2.Bundle(IDENTITY_CUSTOM_PROP, allow_custom=True)
+
+    assert bundle.objects[0].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(bundle)
+
+
 def test_custom_property_in_observed_data():
     artifact = stix2.File(
         allow_custom=True,
@@ -106,11 +113,61 @@ def test_custom_property_in_observed_data():
     assert '"x_foo": "bar"' in str(observed_data)
 
 
-def test_custom_property_in_bundled_object():
-    bundle = stix2.Bundle(IDENTITY_CUSTOM_PROP, allow_custom=True)
+def test_custom_property_object_in_observable_extension():
+    ntfs = stix2.NTFSExt(
+        allow_custom=True,
+        sid=1,
+        x_foo='bar',
+    )
+    artifact = stix2.File(
+        name='test',
+        extensions={'ntfs-ext': ntfs},
+    )
+    observed_data = stix2.ObservedData(
+        allow_custom=True,
+        first_observed="2015-12-21T19:00:00Z",
+        last_observed="2015-12-21T19:00:00Z",
+        number_observed=0,
+        objects={"0": artifact},
+    )
 
-    assert bundle.objects[0].x_foo == "bar"
-    assert '"x_foo": "bar"' in str(bundle)
+    assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(observed_data)
+
+
+def test_custom_property_dict_in_observable_extension():
+    with pytest.raises(stix2.exceptions.ExtraPropertiesError):
+        artifact = stix2.File(
+            name='test',
+            extensions={
+                'ntfs-ext': {
+                    'sid': 1,
+                    'x_foo': 'bar',
+                }
+            },
+        )
+
+    artifact = stix2.File(
+        allow_custom=True,
+        name='test',
+        extensions={
+            'ntfs-ext': {
+                'allow_custom': True,
+                'sid': 1,
+                'x_foo': 'bar',
+            }
+        },
+    )
+    observed_data = stix2.ObservedData(
+        allow_custom=True,
+        first_observed="2015-12-21T19:00:00Z",
+        last_observed="2015-12-21T19:00:00Z",
+        number_observed=0,
+        objects={"0": artifact},
+    )
+
+    assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(observed_data)
 
 
 def test_identity_custom_property_revoke():
@@ -560,6 +617,7 @@ def test_custom_extension():
 
 
 def test_custom_extension_wrong_observable_type():
+    # NewExtension is an extension of DomainName, not File
     ext = NewExtension(property1='something')
     with pytest.raises(ValueError) as excinfo:
         stix2.File(name="abc.txt",
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 501d11c..d436261 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -266,3 +266,49 @@ def test_default_object_marking_refs():
     campaign = Campaign(**CAMPAIGN_KWARGS)
 
     assert campaign.object_marking_refs[0] == mark_def.id
+
+
+def test_workbench_custom_property_object_in_observable_extension():
+    ntfs = stix2.NTFSExt(
+        allow_custom=True,
+        sid=1,
+        x_foo='bar',
+    )
+    artifact = stix2.File(
+        name='test',
+        extensions={'ntfs-ext': ntfs},
+    )
+    observed_data = ObservedData(
+        allow_custom=True,
+        first_observed="2015-12-21T19:00:00Z",
+        last_observed="2015-12-21T19:00:00Z",
+        number_observed=0,
+        objects={"0": artifact},
+    )
+
+    assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(observed_data)
+
+
+def test_workbench_custom_property_dict_in_observable_extension():
+    artifact = stix2.File(
+        allow_custom=True,
+        name='test',
+        extensions={
+            'ntfs-ext': {
+                'allow_custom': True,
+                'sid': 1,
+                'x_foo': 'bar',
+            }
+        },
+    )
+    observed_data = ObservedData(
+        allow_custom=True,
+        first_observed="2015-12-21T19:00:00Z",
+        last_observed="2015-12-21T19:00:00Z",
+        number_observed=0,
+        objects={"0": artifact},
+    )
+
+    assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(observed_data)
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 7bce53e..098c7dd 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -56,7 +56,8 @@ class ExtensionsProperty(DictionaryProperty):
     """Property for representing extensions on Observable objects.
     """
 
-    def __init__(self, enclosing_type=None, required=False):
+    def __init__(self, allow_custom=False, enclosing_type=None, required=False):
+        self.allow_custom = allow_custom
         self.enclosing_type = enclosing_type
         super(ExtensionsProperty, self).__init__(required)
 
@@ -78,8 +79,13 @@ class ExtensionsProperty(DictionaryProperty):
                 if key in specific_type_map:
                     cls = specific_type_map[key]
                     if type(subvalue) is dict:
-                        dictified[key] = cls(**subvalue)
+                        if self.allow_custom:
+                            subvalue['allow_custom'] = True
+                            dictified[key] = cls(**subvalue)
+                        else:
+                            dictified[key] = cls(**subvalue)
                     elif type(subvalue) is cls:
+                        # If already an instance of an _Extension class, assume it's valid
                         dictified[key] = subvalue
                     else:
                         raise ValueError("Cannot determine extension type.")
diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index 9ca1599..abe7df4 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -225,9 +225,9 @@ class ObservedData(STIXDomainObject):
         ('granular_markings', ListProperty(GranularMarking)),
     ])
 
-    def __init__(self, allow_custom=False, *args, **kwargs):
-        if allow_custom:
-            self._properties['objects'] = ObservableProperty(True)
+    def __init__(self, *args, **kwargs):
+        self.__allow_custom = kwargs.get('allow_custom', False)
+        self._properties['objects'].allow_custom = kwargs.get('allow_custom', False)
 
         super(ObservedData, self).__init__(*args, **kwargs)
 
diff --git a/stix2/workbench.py b/stix2/workbench.py
index c559f40..e34f028 100644
--- a/stix2/workbench.py
+++ b/stix2/workbench.py
@@ -49,7 +49,6 @@ from . import (AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem,  # n
                WindowsProcessExt, WindowsRegistryKey, WindowsRegistryValueType,
                WindowsServiceExt, X509Certificate, X509V3ExtenstionsType)
 from .datastore.filters import FilterSet
-from . import ObservableProperty
 
 # Use an implicit MemoryStore
 _environ = Environment(store=MemoryStore())
@@ -111,9 +110,9 @@ def _related_wrapper(self, *args, **kwargs):
     return _environ.related_to(self, *args, **kwargs)
 
 
-def _observed_data_init(self, allow_custom=False, *args, **kwargs):
-    if allow_custom:
-        self._properties['objects'] = ObservableProperty(True)
+def _observed_data_init(self, *args, **kwargs):
+    self.__allow_custom = kwargs.get('allow_custom', False)
+    self._properties['objects'].allow_custom = kwargs.get('allow_custom', False)
     super(self.__class__, self).__init__(*args, **kwargs)
 
 

From a3313bc08fb754877bbcab53e9193c835a439e34 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 16 May 2018 13:23:50 -0400
Subject: [PATCH 142/150] creating native DataSourceError exception class to
 wrap taxii client and server errors

---
 stix2/datastore/__init__.py        | 26 ++++++++++++++--
 stix2/datastore/taxii.py           | 49 ++++++++++--------------------
 stix2/test/test_datastore_taxii.py | 14 ++++-----
 3 files changed, 46 insertions(+), 43 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 7fdf515..ac03959 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -24,15 +24,35 @@ def make_id():
     return str(uuid.uuid4())
 
 
+class DataSourceError(Exception):
+    """General DataSource error instance, used primarily for wrapping
+    lower level errors
+
+    Args:
+        message (str): error message
+        root_exception (Exception): Exception instance of root exception
+            in the case that DataSourceError is wrapping a lower level or
+            other exception
+    """
+    def __init__(self, message, root_exception=None):
+        self.message = message
+        self.root_exception = root_exception
+
+    def __str__(self):
+        if self.root_exception:
+            self.message = "{} \"{}\"".format(self.message, self.root_exception)
+        return self.message
+
+
 class DataStoreMixin(object):
     """Provides mechanisms for storing and retrieving STIX data. The specific
     behavior can be customized by subclasses.
 
     Args:
         source (DataSource): An existing DataSource to use
-             as this DataStore's DataSource component
+            as this DataStore's DataSource component
         sink (DataSink): An existing DataSink to use
-             as this DataStore's DataSink component
+            as this DataStore's DataSink component
 
     Attributes:
         id (str): A unique UUIDv4 to identify this DataStore.
@@ -129,7 +149,7 @@ class DataStoreMixin(object):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 relationships will be looked up.
             relationship_type (str): Only retrieve Relationships of this type.
-                If None, all relationships will be returned, regardless of type.
+            If None, all relationships will be returned, regardless of type.
             source_only (bool): Only retrieve Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only retrieve Relationships for which this
diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 4a047a8..88897a9 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -5,7 +5,8 @@ from requests.exceptions import HTTPError
 
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
-from stix2.datastore import DataSink, DataSource, DataStoreMixin
+from stix2.datastore import (DataSink, DataSource, DataSourceError,
+                             DataStoreMixin)
 from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate
 
@@ -65,18 +66,12 @@ class TAXIICollectionSink(DataSink):
             if collection.can_write:
                 self.collection = collection
             else:
-                raise ValueError("The TAXII Collection object provided does not have write access"
-                                 " to the underlying linked Collection resource")
+                raise DataSourceError("The TAXII Collection object provided does not have write access"
+                                      " to the underlying linked Collection resource")
 
         except (HTTPError, ValidationError) as e:
-            added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII"
-                             " Collection object provided could not be reached.")
-            if not e.args:
-                e.args = (added_context,)
-            else:
-                e.args = (added_context,) + e.args
-
-            raise
+            raise DataSourceError("The underlying TAXII Collection resource defined in the supplied TAXII"
+                                  " Collection object provided could not be reached. Receved error:", e)
 
         self.allow_custom = allow_custom
 
@@ -144,18 +139,12 @@ class TAXIICollectionSource(DataSource):
             if collection.can_read:
                 self.collection = collection
             else:
-                raise ValueError("The TAXII Collection object provided does not have read access"
-                                 " to the underlying linked Collection resource")
+                raise DataSourceError("The TAXII Collection object provided does not have read access"
+                                      " to the underlying linked Collection resource")
 
         except (HTTPError, ValidationError) as e:
-            added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII"
-                             " Collection object provided could not be reached.")
-            if not e.args:
-                e.args = (added_context,)
-            else:
-                e.args = (added_context,) + e.args
-
-            raise
+            raise DataSourceError("The underlying TAXII Collection resource defined in the supplied TAXII"
+                                  " Collection object provided could not be reached. Recieved error:", e)
 
         self.allow_custom = allow_custom
 
@@ -190,12 +179,12 @@ class TAXIICollectionSource(DataSource):
             stix_objs = self.collection.get_object(stix_id)["objects"]
             stix_obj = list(apply_common_filters(stix_objs, query))
 
-        except HTTPError as err:
-            if err.response.status_code == 404:
+        except HTTPError as e:
+            if e.response.status_code == 404:
                 # if resource not found or access is denied from TAXII server, return None
                 stix_obj = []
             else:
-                raise
+                raise DataSourceError("TAXII Collection resource returned error", e)
 
         if len(stix_obj):
             stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version)
@@ -286,15 +275,9 @@ class TAXIICollectionSource(DataSource):
         except HTTPError as e:
             # if resources not found or access is denied from TAXII server, return empty list
             if e.response.status_code == 404:
-                added_context = ("The requested STIX objects for the TAXII Collection resource defined in"
-                                 " the supplied TAXII Collection object is either not found or access is"
-                                 " denied. Received error: ")
-            if not e.args:
-                e.args = (added_context,)
-            else:
-                e.args = (added_context,) + e.args
-
-            raise
+                raise DataSourceError("The requested STIX objects for the TAXII Collection resource defined in"
+                                      " the supplied TAXII Collection object are either not found or access is"
+                                      " denied. Received error: ", e)
 
         # parse python STIX objects from the STIX object dicts
         stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index a153ce8..78a3284 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -2,12 +2,12 @@ import json
 
 from medallion.filters.basic_filter import BasicFilter
 import pytest
-from requests.exceptions import HTTPError
 from requests.models import Response
 from taxii2client import Collection, _filter_kwargs_to_query_params
 
 from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource,
                    TAXIICollectionStore, ThreatActor)
+from stix2.datastore import DataSourceError
 from stix2.datastore.filters import Filter
 
 COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
@@ -326,7 +326,7 @@ def test_get_all_versions(collection):
 def test_can_read_error(collection_no_rw_access):
     """create a TAXIICOllectionSource with a taxii2client.Collection
     instance that does not have read access, check ValueError exception is raised"""
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(DataSourceError) as excinfo:
         TAXIICollectionSource(collection_no_rw_access)
     assert "Collection object provided does not have read access" in str(excinfo.value)
 
@@ -334,7 +334,7 @@ def test_can_read_error(collection_no_rw_access):
 def test_can_write_error(collection_no_rw_access):
     """create a TAXIICOllectionSink with a taxii2client.Collection
     instance that does not have write access, check ValueError exception is raised"""
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(DataSourceError) as excinfo:
         TAXIICollectionSink(collection_no_rw_access)
     assert "Collection object provided does not have write access" in str(excinfo.value)
 
@@ -359,9 +359,9 @@ def test_all_versions_404(collection):
     """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404
     response code from the taxii2client should be returned as an exception"""
     ds = TAXIICollectionStore(collection)
-    with pytest.raises(HTTPError) as excinfo:
+    with pytest.raises(DataSourceError) as excinfo:
         ds.all_versions("indicator--1")
-    assert "is either not found or access is denied" in str(excinfo.value)
+    assert "are either not found or access is denied" in str(excinfo.value)
     assert "404" in str(excinfo.value)
 
 
@@ -371,7 +371,7 @@ def test_query_404(collection):
     ds = TAXIICollectionStore(collection)
     query = [Filter("type", "=", "malware")]
 
-    with pytest.raises(HTTPError) as excinfo:
+    with pytest.raises(DataSourceError) as excinfo:
         ds.query(query=query)
-    assert "is either not found or access is denied" in str(excinfo.value)
+    assert "are either not found or access is denied" in str(excinfo.value)
     assert "404" in str(excinfo.value)

From 8fc421e7d43b9004841d109d8809da338c4a7c54 Mon Sep 17 00:00:00 2001
From: = 
Date: Wed, 16 May 2018 15:17:34 -0400
Subject: [PATCH 143/150] think the mock client was erroneous for get_object(),
 couldnt get object by id, just returned random object

---
 stix2/test/test_datastore_taxii.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index 78a3284..dbade77 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -45,17 +45,17 @@ class MockTAXIICollectionEndpoint(Collection):
             resp.status_code = 404
             resp.raise_for_status()
 
-    def get_object(self, id, version=None):
+    def get_object(self, id_, version=None):
         self._verify_can_read()
-        query_params = None
+        kwargs = {"id": id_}
         if version:
-            query_params = _filter_kwargs_to_query_params({"version": version})
-        if query_params:
-            query_params = json.loads(query_params)
+            kwargs.update({"version": version})
+
+        query_params = _filter_kwargs_to_query_params(kwargs)
         full_filter = BasicFilter(query_params or {})
         objs = full_filter.process_filter(
             self.objects,
-            ("version",),
+            ("version", "id"),
             []
         )
         if objs:

From cf972479ed487b0ef58f1275d0bf3ae81b48721a Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 16 May 2018 15:37:30 -0400
Subject: [PATCH 144/150] Pass allow_custom to object dicts in a Bundle

---
 stix2/base.py             |  8 ++++----
 stix2/core.py             |  2 +-
 stix2/test/test_custom.py | 20 +++++++++++++++++++-
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index bb8cd2f..84397f0 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -136,7 +136,7 @@ class _STIXBase(collections.Mapping):
         if custom_props and not isinstance(custom_props, dict):
             raise ValueError("'custom_properties' must be a dictionary")
         if not allow_custom:
-            extra_kwargs = list(set(kwargs) - set(cls._properties))
+            extra_kwargs = list(set(kwargs) - set(self._properties))
             if extra_kwargs:
                 raise ExtraPropertiesError(cls, extra_kwargs)
 
@@ -149,17 +149,17 @@ class _STIXBase(collections.Mapping):
                 setting_kwargs[prop_name] = prop_value
 
         # Detect any missing required properties
-        required_properties = set(get_required_properties(cls._properties))
+        required_properties = set(get_required_properties(self._properties))
         missing_kwargs = required_properties - set(setting_kwargs)
         if missing_kwargs:
             raise MissingPropertiesError(cls, missing_kwargs)
 
-        for prop_name, prop_metadata in cls._properties.items():
+        for prop_name, prop_metadata in self._properties.items():
             self._check_property(prop_name, prop_metadata, setting_kwargs)
 
         # Cache defaulted optional properties for serialization
         defaulted = []
-        for name, prop in cls._properties.items():
+        for name, prop in self._properties.items():
             try:
                 if (not prop.required and not hasattr(prop, '_fixed_value') and
                         prop.default() == setting_kwargs[name]):
diff --git a/stix2/core.py b/stix2/core.py
index 6c2e23a..af74a9e 100644
--- a/stix2/core.py
+++ b/stix2/core.py
@@ -63,7 +63,7 @@ class Bundle(_STIXBase):
                 kwargs['objects'] = list(args) + kwargs.get('objects', [])
 
         self.__allow_custom = kwargs.get('allow_custom', False)
-        self._properties['objects'].allow_custom = kwargs.get('allow_custom', False)
+        self._properties['objects'].contained.allow_custom = kwargs.get('allow_custom', False)
 
         super(Bundle, self).__init__(**kwargs)
 
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 59cbea8..47e8ae1 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -88,13 +88,31 @@ def test_parse_identity_custom_property(data):
     assert identity.foo == "bar"
 
 
-def test_custom_property_in_bundled_object():
+def test_custom_property_object_in_bundled_object():
     bundle = stix2.Bundle(IDENTITY_CUSTOM_PROP, allow_custom=True)
 
     assert bundle.objects[0].x_foo == "bar"
     assert '"x_foo": "bar"' in str(bundle)
 
 
+def test_custom_property_dict_in_bundled_object():
+    custom_identity = {
+        'type': 'identity',
+        'id': 'identity--311b2d2d-f010-5473-83ec-1edf84858f4c',
+        'created': '2015-12-21T19:59:11Z',
+        'name': 'John Smith',
+        'identity_class': 'individual',
+        'x_foo': 'bar',
+    }
+    with pytest.raises(stix2.exceptions.ExtraPropertiesError):
+        bundle = stix2.Bundle(custom_identity)
+
+    bundle = stix2.Bundle(custom_identity, allow_custom=True)
+
+    assert bundle.objects[0].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(bundle)
+
+
 def test_custom_property_in_observed_data():
     artifact = stix2.File(
         allow_custom=True,

From ddc09f70c7df3e9f8128b63fe9907613bef5c600 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 17 May 2018 09:04:44 -0400
Subject: [PATCH 145/150] Set allow_custom if using a custom_properties dict

Fixes #179.
---
 stix2/base.py             |  4 +++-
 stix2/test/test_custom.py | 45 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 84397f0..2afba16 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -135,10 +135,12 @@ class _STIXBase(collections.Mapping):
         custom_props = kwargs.pop('custom_properties', {})
         if custom_props and not isinstance(custom_props, dict):
             raise ValueError("'custom_properties' must be a dictionary")
-        if not allow_custom:
+        if not self.__allow_custom:
             extra_kwargs = list(set(kwargs) - set(self._properties))
             if extra_kwargs:
                 raise ExtraPropertiesError(cls, extra_kwargs)
+        if custom_props:
+            self.__allow_custom = True
 
         # Remove any keyword arguments whose value is None
         setting_kwargs = {}
diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 47e8ae1..7f91d79 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -24,6 +24,20 @@ def test_identity_custom_property():
         )
     assert str(excinfo.value) == "'custom_properties' must be a dictionary"
 
+    with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo:
+        stix2.Identity(
+            id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c",
+            created="2015-12-21T19:59:11Z",
+            modified="2015-12-21T19:59:11Z",
+            name="John Smith",
+            identity_class="individual",
+            custom_properties={
+                "foo": "bar",
+            },
+            foo="bar",
+            )
+    assert "Unexpected properties for Identity" in str(excinfo.value)
+
     identity = stix2.Identity(
         id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c",
         created="2015-12-21T19:59:11Z",
@@ -34,7 +48,6 @@ def test_identity_custom_property():
             "foo": "bar",
         },
     )
-
     assert identity.foo == "bar"
 
 
@@ -95,6 +108,20 @@ def test_custom_property_object_in_bundled_object():
     assert '"x_foo": "bar"' in str(bundle)
 
 
+def test_custom_properties_object_in_bundled_object():
+    obj = stix2.Identity(
+        name="John Smith",
+        identity_class="individual",
+        custom_properties={
+            "x_foo": "bar",
+        }
+    )
+    bundle = stix2.Bundle(obj, allow_custom=True)
+
+    assert bundle.objects[0].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(bundle)
+
+
 def test_custom_property_dict_in_bundled_object():
     custom_identity = {
         'type': 'identity',
@@ -108,6 +135,22 @@ def test_custom_property_dict_in_bundled_object():
         bundle = stix2.Bundle(custom_identity)
 
     bundle = stix2.Bundle(custom_identity, allow_custom=True)
+    assert bundle.objects[0].x_foo == "bar"
+    assert '"x_foo": "bar"' in str(bundle)
+
+
+def test_custom_properties_dict_in_bundled_object():
+    custom_identity = {
+        'type': 'identity',
+        'id': 'identity--311b2d2d-f010-5473-83ec-1edf84858f4c',
+        'created': '2015-12-21T19:59:11Z',
+        'name': 'John Smith',
+        'identity_class': 'individual',
+        'custom_properties': {
+            'x_foo': 'bar',
+        },
+    }
+    bundle = stix2.Bundle(custom_identity)
 
     assert bundle.objects[0].x_foo == "bar"
     assert '"x_foo": "bar"' in str(bundle)

From 6004ec597c432badeb57ca350b2ff22d2cdf1051 Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 18 May 2018 11:19:05 -0400
Subject: [PATCH 146/150] solution to get 404 mocking problem

---
 stix2/test/test_datastore_taxii.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index dbade77..be8adaa 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -45,17 +45,17 @@ class MockTAXIICollectionEndpoint(Collection):
             resp.status_code = 404
             resp.raise_for_status()
 
-    def get_object(self, id_, version=None):
+    def get_object(self, id, version=None):
         self._verify_can_read()
-        kwargs = {"id": id_}
+        query_params = None
         if version:
-            kwargs.update({"version": version})
-
-        query_params = _filter_kwargs_to_query_params(kwargs)
+            query_params = _filter_kwargs_to_query_params({"version": version})
+        if query_params:
+            query_params = json.loads(query_params)
         full_filter = BasicFilter(query_params or {})
         objs = full_filter.process_filter(
             self.objects,
-            ("version", "id"),
+            ("version",),
             []
         )
         if objs:
@@ -339,7 +339,7 @@ def test_can_write_error(collection_no_rw_access):
     assert "Collection object provided does not have write access" in str(excinfo.value)
 
 
-def test_get_404(collection):
+def test_get_404():
     """a TAXIICollectionSource.get() call that receives an HTTP 404 response
     code from the taxii2client should be be returned as None.
 
@@ -347,7 +347,16 @@ def test_get_404(collection):
     nonexistent resources or lack of access. Decided that None is acceptable
     reponse to imply that state of the TAXII endpoint.
     """
-    ds = TAXIICollectionStore(collection)
+
+    class TAXIICollection404():
+        can_read = True
+
+        def get_object(self, id, version=None):
+            resp = Response()
+            resp.status_code = 404
+            resp.raise_for_status()
+
+    ds = TAXIICollectionSource(TAXIICollection404())
 
     # this will raise 404 from mock TAXII Client but TAXIICollectionStore
     # should handle gracefully and return None

From e484b7c25f6fac6209b927a480e3415a1510a3cc Mon Sep 17 00:00:00 2001
From: = 
Date: Fri, 18 May 2018 11:45:40 -0400
Subject: [PATCH 147/150] formatting

---
 stix2/datastore/taxii.py           |  2 +-
 stix2/test/test_datastore_taxii.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py
index 88897a9..c815e12 100644
--- a/stix2/datastore/taxii.py
+++ b/stix2/datastore/taxii.py
@@ -268,7 +268,7 @@ class TAXIICollectionSource(DataSource):
             # deduplicate data (before filtering as reduces wasted filtering)
             all_data = deduplicate(all_data)
 
-            # a pply local (CompositeDataSource, TAXIICollectionSource and query) filters
+            # apply local (CompositeDataSource, TAXIICollectionSource and query) filters
             query.remove(taxii_filters)
             all_data = list(apply_common_filters(all_data, query))
 
diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index be8adaa..8675378 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -326,6 +326,7 @@ def test_get_all_versions(collection):
 def test_can_read_error(collection_no_rw_access):
     """create a TAXIICOllectionSource with a taxii2client.Collection
     instance that does not have read access, check ValueError exception is raised"""
+
     with pytest.raises(DataSourceError) as excinfo:
         TAXIICollectionSource(collection_no_rw_access)
     assert "Collection object provided does not have read access" in str(excinfo.value)
@@ -334,6 +335,7 @@ def test_can_read_error(collection_no_rw_access):
 def test_can_write_error(collection_no_rw_access):
     """create a TAXIICOllectionSink with a taxii2client.Collection
     instance that does not have write access, check ValueError exception is raised"""
+
     with pytest.raises(DataSourceError) as excinfo:
         TAXIICollectionSink(collection_no_rw_access)
     assert "Collection object provided does not have write access" in str(excinfo.value)
@@ -343,9 +345,9 @@ def test_get_404():
     """a TAXIICollectionSource.get() call that receives an HTTP 404 response
     code from the taxii2client should be be returned as None.
 
-    TAXII spec states that a TAXII server can return a 404 for
-    nonexistent resources or lack of access. Decided that None is acceptable
-    reponse to imply that state of the TAXII endpoint.
+    TAXII spec states that a TAXII server can return a 404 for nonexistent
+    resources or lack of access. Decided that None is acceptable reponse
+    to imply that state of the TAXII endpoint.
     """
 
     class TAXIICollection404():
@@ -360,14 +362,16 @@ def test_get_404():
 
     # this will raise 404 from mock TAXII Client but TAXIICollectionStore
     # should handle gracefully and return None
-    stix_obj = ds.get("indicator--1")  # this will raise 404 from
+    stix_obj = ds.get("indicator--1")
     assert stix_obj is None
 
 
 def test_all_versions_404(collection):
     """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404
     response code from the taxii2client should be returned as an exception"""
+
     ds = TAXIICollectionStore(collection)
+
     with pytest.raises(DataSourceError) as excinfo:
         ds.all_versions("indicator--1")
     assert "are either not found or access is denied" in str(excinfo.value)
@@ -377,6 +381,7 @@ def test_all_versions_404(collection):
 def test_query_404(collection):
     """ a TAXIICollectionSource.query() call that recieves an HTTP 404
     response code from the taxii2client should be returned as an exception"""
+
     ds = TAXIICollectionStore(collection)
     query = [Filter("type", "=", "malware")]
 

From 1a62e62c4dc0c37b727f63a66dd335a27d57b4e2 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 18 May 2018 13:05:58 -0400
Subject: [PATCH 148/150] Update CHANGELOG for v1.0.2

---
 CHANGELOG | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 8e2be4c..596f71c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,10 @@
 CHANGELOG
 =========
 
+1.0.2 - 2018-05-18
+
+* Fixes bugs when using allow_custom (#176, #179).
+
 1.0.1 - 2018-04-27
 
 * Fixes bug with incorrect TAXII parameters (#169).

From 636630a5aa097bb30cbc25e098322b4fea9a3d65 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 18 May 2018 13:06:32 -0400
Subject: [PATCH 149/150] =?UTF-8?q?Bump=20version:=201.0.1=20=E2=86=92=201?=
 =?UTF-8?q?.0.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/conf.py     | 4 ++--
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 9a2f24e..9b1ad5d 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,8 +34,8 @@ project = 'stix2'
 copyright = '2017, OASIS Open'
 author = 'OASIS Open'
 
-version = '1.0.1'
-release = '1.0.1'
+version = '1.0.2'
+release = '1.0.2'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints']
diff --git a/setup.cfg b/setup.cfg
index e9787b8..c18a3cf 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.1
+current_version = 1.0.2
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index 5c4105c..7863915 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "1.0.1"
+__version__ = "1.0.2"

From 3fb184028924ac45ae7e8bf269a4282ac6464642 Mon Sep 17 00:00:00 2001
From: = 
Date: Tue, 22 May 2018 10:03:06 -0400
Subject: [PATCH 150/150] safer error class

---
 stix2/datastore/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index ac03959..c2963e2 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -40,8 +40,9 @@ class DataSourceError(Exception):
 
     def __str__(self):
         if self.root_exception:
-            self.message = "{} \"{}\"".format(self.message, self.root_exception)
-        return self.message
+            return "{} \"{}\"".format(self.message, self.root_exception)
+        else:
+            return self.message
 
 
 class DataStoreMixin(object):
@@ -149,7 +150,7 @@ class DataStoreMixin(object):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 relationships will be looked up.
             relationship_type (str): Only retrieve Relationships of this type.
-            If None, all relationships will be returned, regardless of type.
+                If None, all relationships will be returned, regardless of type.
             source_only (bool): Only retrieve Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only retrieve Relationships for which this