From 541c682bf626ed268030bfc6781d968678f63073 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Fri, 7 Sep 2018 17:47:24 -0400
Subject: [PATCH 01/11] Fixed a test fixture to call the cti-taxii-client
 Collection constructor correctly.  It had been recently changed to address
 issue #39 in that library.

---
 stix2/test/test_datastore_taxii.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py
index 86cfb6f..89bf949 100644
--- a/stix2/test/test_datastore_taxii.py
+++ b/stix2/test/test_datastore_taxii.py
@@ -16,8 +16,10 @@ COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c
 class MockTAXIICollectionEndpoint(Collection):
     """Mock for taxii2_client.TAXIIClient"""
 
-    def __init__(self, url, **kwargs):
-        super(MockTAXIICollectionEndpoint, self).__init__(url, **kwargs)
+    def __init__(self, url, collection_info):
+        super(MockTAXIICollectionEndpoint, self).__init__(
+            url, collection_info=collection_info
+        )
         self.objects = []
 
     def add_objects(self, bundle):
@@ -68,7 +70,7 @@ class MockTAXIICollectionEndpoint(Collection):
 
 @pytest.fixture
 def collection(stix_objs1):
-    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{
+    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, {
         "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116",
         "title": "Writable Collection",
         "description": "This collection is a dropbox for submitting indicators",
@@ -85,7 +87,7 @@ def collection(stix_objs1):
 
 @pytest.fixture
 def collection_no_rw_access(stix_objs1):
-    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{
+    mock = MockTAXIICollectionEndpoint(COLLECTION_URL, {
         "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116",
         "title": "Not writeable or readable Collection",
         "description": "This collection is a dropbox for submitting indicators",

From 5a0e10295937a9420ab192e2a99e4d4e7231feb4 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Mon, 15 Oct 2018 17:21:48 -0400
Subject: [PATCH 02/11] Modify docstring for apply_common_filters() to change
 what it says are the required types for its parameters.  It gave specific
 types (list and set), when really it worked with more than just that
 (iterables).  And I certainly didn't only call it with just lists and sets.

---
 stix2/datastore/filters.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py
index 03585ad..0172a50 100644
--- a/stix2/datastore/filters.py
+++ b/stix2/datastore/filters.py
@@ -123,8 +123,11 @@ def apply_common_filters(stix_objs, query):
     Supports only STIX 2.0 common property properties.
 
     Args:
-        stix_objs (list): list of STIX objects to apply the query to
-        query (set): set of filters (combined form complete query)
+        stix_objs (iterable): iterable of STIX objects to apply the query to
+        query (non-iterator iterable): iterable of filters.  Can't be an
+            iterator (e.g. generator iterators won't work), since this is
+            used in an inner loop of a nested loop.  So we require the ability
+            to traverse the filters repeatedly.
 
     Yields:
         STIX objects that successfully evaluate against the query.

From d9f6a213c188c25ab20bb2e91b5e6d8d17ac2545 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Mon, 15 Oct 2018 17:57:57 -0400
Subject: [PATCH 03/11] Fixed Memory source/sink/store so that it supports
 multiple versions of objects.  Fixed several bugs too.

---
 stix2/datastore/memory.py           | 159 ++++++++++++++++++----------
 stix2/test/test_datastore_memory.py |  15 ++-
 stix2/test/test_environment.py      |   2 +-
 stix2/test/test_memory.py           |  30 +++---
 4 files changed, 128 insertions(+), 78 deletions(-)

diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index c1d202d..98304e0 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -12,16 +12,17 @@ Note:
 
 """
 
+import itertools
 import json
 import os
 
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
+from stix2.datastore.filters import FilterSet, apply_common_filters
 
 
-def _add(store, stix_data=None, version=None):
+def _add(store, stix_data=None, allow_custom=True, version=None):
     """Add STIX objects to MemoryStore/Sink.
 
     Adds STIX objects to an in-memory dictionary for fast lookup.
@@ -33,27 +34,55 @@ def _add(store, stix_data=None, version=None):
             None, use latest version.
 
     """
-    if isinstance(stix_data, _STIXBase):
-        # adding a python STIX object
-        store._data[stix_data["id"]] = stix_data
-
-    elif isinstance(stix_data, dict):
-        if stix_data["type"] == "bundle":
-            # adding a json bundle - so just grab STIX objects
-            for stix_obj in stix_data.get("objects", []):
-                _add(store, stix_obj, version=version)
-        else:
-            # adding a json STIX object
-            store._data[stix_data["id"]] = stix_data
-
-    elif isinstance(stix_data, list):
+    if isinstance(stix_data, list):
         # STIX objects are in a list- recurse on each object
         for stix_obj in stix_data:
-            _add(store, stix_obj, version=version)
+            _add(store, stix_obj, allow_custom, version)
+
+    elif stix_data["type"] == "bundle":
+        # adding a json bundle - so just grab STIX objects
+        for stix_obj in stix_data.get("objects", []):
+            _add(store, stix_obj, allow_custom, version)
 
     else:
-        raise TypeError("stix_data expected to be a python-stix2 object (or list of), JSON formatted STIX (or list of),"
-                        " or a JSON formatted STIX bundle. stix_data was of type: " + str(type(stix_data)))
+        # Adding a single non-bundle object
+        if isinstance(stix_data, _STIXBase):
+            stix_obj = stix_data
+        else:
+            stix_obj = parse(stix_data, allow_custom, version)
+
+        if stix_obj.id in store._data:
+            obj_family = store._data[stix_obj.id]
+        else:
+            obj_family = _ObjectFamily()
+            store._data[stix_obj.id] = obj_family
+
+        obj_family.add(stix_obj)
+
+
+class _ObjectFamily(object):
+    """
+    An internal implementation detail of memory sources/sinks/stores.
+    Represents a "family" of STIX objects: all objects with a particular
+    ID.  (I.e. all versions.)  The latest version is also tracked so that it
+    can be obtained quickly.
+    """
+    def __init__(self):
+        self.all_versions = {}
+        self.latest_version = None
+
+    def add(self, obj):
+        self.all_versions[obj.modified] = obj
+        if self.latest_version is None or \
+                obj.modified > self.latest_version.modified:
+            self.latest_version = obj
+
+    def __str__(self):
+        return "<<{}; latest={}>>".format(self.all_versions,
+                                          self.latest_version.modified)
+
+    def __repr__(self):
+        return str(self)
 
 
 class MemoryStore(DataStoreMixin):
@@ -83,7 +112,7 @@ class MemoryStore(DataStoreMixin):
         self._data = {}
 
         if stix_data:
-            _add(self, stix_data, version=version)
+            _add(self, stix_data, allow_custom, version=version)
 
         super(MemoryStore, self).__init__(
             source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True),
@@ -138,25 +167,31 @@ class MemorySink(DataSink):
     """
     def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False):
         super(MemorySink, self).__init__()
-        self._data = {}
         self.allow_custom = allow_custom
 
         if _store:
             self._data = stix_data
-        elif stix_data:
-            _add(self, stix_data, version=version)
+        else:
+            self._data = {}
+            if stix_data:
+                _add(self, stix_data, allow_custom, version=version)
 
     def add(self, stix_data, version=None):
-        _add(self, stix_data, version=version)
+        _add(self, stix_data, self.allow_custom, version)
     add.__doc__ = _add.__doc__
 
     def save_to_file(self, file_path):
         file_path = os.path.abspath(file_path)
 
+        all_objs = itertools.chain.from_iterable(
+            obj_family.all_versions.values()
+            for obj_family in self._data.values()
+        )
+
         if not os.path.exists(os.path.dirname(file_path)):
             os.makedirs(os.path.dirname(file_path))
         with open(file_path, "w") as f:
-            f.write(str(Bundle(list(self._data.values()), allow_custom=self.allow_custom)))
+            f.write(str(Bundle(list(all_objs), allow_custom=self.allow_custom)))
     save_to_file.__doc__ = MemoryStore.save_to_file.__doc__
 
 
@@ -184,13 +219,14 @@ class MemorySource(DataSource):
     """
     def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False):
         super(MemorySource, self).__init__()
-        self._data = {}
         self.allow_custom = allow_custom
 
         if _store:
             self._data = stix_data
-        elif stix_data:
-            _add(self, stix_data, version=version)
+        else:
+            self._data = {}
+            if stix_data:
+                _add(self, stix_data, allow_custom, version=version)
 
     def get(self, stix_id, _composite_filters=None):
         """Retrieve STIX object from in-memory dict via STIX ID.
@@ -207,26 +243,22 @@ class MemorySource(DataSource):
                 is returned in the same form as it as added
 
         """
-        if _composite_filters is None:
-            # if get call is only based on 'id', no need to search, just retrieve from dict
-            try:
-                stix_obj = self._data[stix_id]
-            except KeyError:
-                stix_obj = None
-            return stix_obj
+        stix_obj = None
+        object_family = self._data.get(stix_id)
+        if object_family:
+            stix_obj = object_family.latest_version
 
-        # if there are filters from the composite level, process full query
-        query = [Filter("id", "=", stix_id)]
+        if stix_obj:
+            all_filters = list(
+                itertools.chain(
+                    _composite_filters or [],
+                    self.filters
+                )
+            )
 
-        all_data = self.query(query=query, _composite_filters=_composite_filters)
+            stix_obj = next(apply_common_filters([stix_obj], all_filters), None)
 
-        if all_data:
-            # reduce to most recent version
-            stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
-
-            return stix_obj
-        else:
-            return None
+        return stix_obj
 
     def all_versions(self, stix_id, _composite_filters=None):
         """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it
@@ -246,8 +278,23 @@ class MemorySource(DataSource):
                 is returned in the same form as it as added
 
         """
+        results = []
+        object_family = self._data.get(stix_id)
 
-        return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
+        if object_family:
+            all_filters = list(
+                itertools.chain(
+                    _composite_filters or [],
+                    self.filters
+                )
+            )
+
+            results.extend(
+                apply_common_filters(object_family.all_versions.values(),
+                                     all_filters)
+            )
+
+        return results
 
     def query(self, query=None, _composite_filters=None):
         """Search and retrieve STIX objects based on the complete query.
@@ -265,7 +312,7 @@ class MemorySource(DataSource):
             (list): list of STIX objects that matches the supplied
                 query. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory
                 as they are supplied (either as python dictionary or STIX object), it
-                is returned in the same form as it as added.
+                is returned in the same form as it was added.
 
         """
         query = FilterSet(query)
@@ -276,17 +323,23 @@ class MemorySource(DataSource):
         if _composite_filters:
             query.add(_composite_filters)
 
+        all_objs = itertools.chain.from_iterable(
+            obj_family.all_versions.values()
+            for obj_family in self._data.values()
+        )
+
         # Apply STIX common property filters.
-        all_data = list(apply_common_filters(self._data.values(), query))
+        all_data = list(apply_common_filters(all_objs, query))
 
         return all_data
 
     def load_from_file(self, file_path, version=None):
-        stix_data = json.load(open(os.path.abspath(file_path), "r"))
+        with open(os.path.abspath(file_path), "r") as f:
+            stix_data = json.load(f)
 
+        # Override user version selection if loading a bundle
         if stix_data["type"] == "bundle":
-            for stix_obj in stix_data["objects"]:
-                _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"]))
-        else:
-            _add(self, stix_data=parse(stix_data, allow_custom=self.allow_custom, version=version))
+            version = stix_data["spec_version"]
+
+        _add(self, stix_data, self.allow_custom, version)
     load_from_file.__doc__ = MemoryStore.load_from_file.__doc__
diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
index 3d69953..60ea33b 100644
--- a/stix2/test/test_datastore_memory.py
+++ b/stix2/test/test_datastore_memory.py
@@ -3,6 +3,7 @@ import pytest
 from stix2.datastore import CompositeDataSource, make_id
 from stix2.datastore.filters import Filter
 from stix2.datastore.memory import MemorySink, MemorySource
+from stix2.utils import parse_into_datetime
 
 
 def test_add_remove_composite_datasource():
@@ -44,14 +45,14 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2):
     indicators = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001")
 
     # In STIX_OBJS2 changed the 'modified' property to a later time...
-    assert len(indicators) == 2
+    assert len(indicators) == 3
 
     cds1.add_data_sources([cds2])
 
     indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001")
 
     assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001"
-    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
+    assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z")
     assert indicator["type"] == "indicator"
 
     query1 = [
@@ -68,20 +69,18 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2):
 
     # STIX_OBJS2 has indicator with later time, one with different id, one with
     # original time in STIX_OBJS1
-    assert len(results) == 3
+    assert len(results) == 4
 
     indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001")
 
     assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001"
-    assert indicator["modified"] == "2017-01-31T13:49:53.935Z"
+    assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z")
     assert indicator["type"] == "indicator"
 
-    # There is only one indicator with different ID. Since we use the same data
-    # when deduplicated, only two indicators (one with different modified).
     results = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001")
-    assert len(results) == 2
+    assert len(results) == 3
 
     # Since we have filters already associated with our CompositeSource providing
     # nothing returns the same as cds1.query(query1) (the associated query is query2)
     results = cds1.query([])
-    assert len(results) == 3
+    assert len(results) == 4
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index d179ae9..a3ec469 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -113,7 +113,7 @@ def test_environment_functions():
 
     # Get both versions of the object
     resp = env.all_versions(INDICATOR_ID)
-    assert len(resp) == 1  # should be 2, but MemoryStore only keeps 1 version of objects
+    assert len(resp) == 2
 
     # Get just the most recent version of the object
     resp = env.get(INDICATOR_ID)
diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index 44f90ba..ba70af4 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -11,6 +11,7 @@ from stix2.datastore import make_id
 from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
                         IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
                         MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
+from stix2.utils import parse_into_datetime
 
 IND1 = {
     "created": "2017-01-27T13:49:53.935Z",
@@ -167,7 +168,7 @@ def test_memory_store_all_versions(mem_store):
                   type="bundle"))
 
     resp = mem_store.all_versions("indicator--00000000-0000-4000-8000-000000000001")
-    assert len(resp) == 1  # MemoryStore can only store 1 version of each object
+    assert len(resp) == 3
 
 
 def test_memory_store_query(mem_store):
@@ -179,25 +180,27 @@ def test_memory_store_query(mem_store):
 def test_memory_store_query_single_filter(mem_store):
     query = Filter('id', '=', 'indicator--00000000-0000-4000-8000-000000000001')
     resp = mem_store.query(query)
-    assert len(resp) == 1
+    assert len(resp) == 2
 
 
 def test_memory_store_query_empty_query(mem_store):
     resp = mem_store.query()
     # sort since returned in random order
-    resp = sorted(resp, key=lambda k: k['id'])
-    assert len(resp) == 2
+    resp = sorted(resp, key=lambda k: (k['id'], k['modified']))
+    assert len(resp) == 3
     assert resp[0]['id'] == 'indicator--00000000-0000-4000-8000-000000000001'
-    assert resp[0]['modified'] == '2017-01-27T13:49:53.936Z'
-    assert resp[1]['id'] == 'indicator--00000000-0000-4000-8000-000000000002'
-    assert resp[1]['modified'] == '2017-01-27T13:49:53.935Z'
+    assert resp[0]['modified'] == parse_into_datetime('2017-01-27T13:49:53.935Z')
+    assert resp[1]['id'] == 'indicator--00000000-0000-4000-8000-000000000001'
+    assert resp[1]['modified'] == parse_into_datetime('2017-01-27T13:49:53.936Z')
+    assert resp[2]['id'] == 'indicator--00000000-0000-4000-8000-000000000002'
+    assert resp[2]['modified'] == parse_into_datetime('2017-01-27T13:49:53.935Z')
 
 
 def test_memory_store_query_multiple_filters(mem_store):
     mem_store.source.filters.add(Filter('type', '=', 'indicator'))
     query = Filter('id', '=', 'indicator--00000000-0000-4000-8000-000000000001')
     resp = mem_store.query(query)
-    assert len(resp) == 1
+    assert len(resp) == 2
 
 
 def test_memory_store_save_load_file(mem_store, fs_mem_store):
@@ -218,12 +221,8 @@ def test_memory_store_save_load_file(mem_store, fs_mem_store):
 
 def test_memory_store_add_invalid_object(mem_store):
     ind = ('indicator', IND1)  # tuple isn't valid
-    with pytest.raises(TypeError) as excinfo:
+    with pytest.raises(TypeError):
         mem_store.add(ind)
-    assert 'stix_data expected to be' in str(excinfo.value)
-    assert 'a python-stix2 object' in str(excinfo.value)
-    assert 'JSON formatted STIX' in str(excinfo.value)
-    assert 'JSON formatted STIX bundle' in str(excinfo.value)
 
 
 def test_memory_store_object_with_custom_property(mem_store):
@@ -246,10 +245,9 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store):
                     allow_custom=True)
 
     bundle = Bundle(camp, allow_custom=True)
-    mem_store.add(bundle, True)
+    mem_store.add(bundle)
 
-    bundle_r = mem_store.get(bundle.id)
-    camp_r = bundle_r['objects'][0]
+    camp_r = mem_store.get(camp.id)
     assert camp_r.id == camp.id
     assert camp_r.x_empire == camp.x_empire
 

From 864ba05b71c3d993194f161d08f744a4486cdeef Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Mon, 15 Oct 2018 19:23:28 -0400
Subject: [PATCH 04/11] Fix import order to satisfy isort-check

---
 stix2/test/test_memory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index ba70af4..7499326 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -7,11 +7,11 @@ from stix2 import (Bundle, Campaign, CustomObject, Filter, Identity, Indicator,
                    Malware, MemorySource, MemoryStore, Relationship,
                    properties)
 from stix2.datastore import make_id
+from stix2.utils import parse_into_datetime
 
 from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
                         IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
                         MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
-from stix2.utils import parse_into_datetime
 
 IND1 = {
     "created": "2017-01-27T13:49:53.935Z",

From 2d89cfb0cfc449c72531bf2997e856b28ef3c828 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Wed, 17 Oct 2018 15:49:58 -0400
Subject: [PATCH 05/11] Remove outdated TODO's from the memory datastore
 module-level docstring.

---
 stix2/datastore/memory.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index 98304e0..0f5d647 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -1,15 +1,5 @@
 """
 Python STIX 2.0 Memory Source/Sink
-
-TODO:
-    Use deduplicate() calls only when memory corpus is dirty (been added to)
-    can save a lot of time for successive queries
-
-Note:
-    Not worrying about STIX versioning. The in memory STIX data at anytime
-    will only hold one version of a STIX object. As such, when save() is called,
-    the single versions of all the STIX objects are what is written to file.
-
 """
 
 import itertools

From cbe8d22d0affc515fd27155892d18a459b5eb6b4 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Wed, 17 Oct 2018 20:54:53 -0400
Subject: [PATCH 06/11] Added support to multi-version memory stores for
 markings.  Also added some more unit tests which test storing/retrieving
 markings from the stores.

---
 stix2/datastore/memory.py           | 69 ++++++++++++++++++++++-------
 stix2/test/test_datastore_memory.py | 65 ++++++++++++++++++++++++++-
 2 files changed, 116 insertions(+), 18 deletions(-)

diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index 0f5d647..eb8580e 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -41,13 +41,35 @@ def _add(store, stix_data=None, allow_custom=True, version=None):
         else:
             stix_obj = parse(stix_data, allow_custom, version)
 
-        if stix_obj.id in store._data:
-            obj_family = store._data[stix_obj.id]
-        else:
-            obj_family = _ObjectFamily()
-            store._data[stix_obj.id] = obj_family
+        # Map ID directly to the object, if it is a marking.  Otherwise,
+        # map to a family, so we can track multiple versions.
+        if _is_marking(stix_obj):
+            store._data[stix_obj.id] = stix_obj
 
-        obj_family.add(stix_obj)
+        else:
+            if stix_obj.id in store._data:
+                obj_family = store._data[stix_obj.id]
+            else:
+                obj_family = _ObjectFamily()
+                store._data[stix_obj.id] = obj_family
+
+            obj_family.add(stix_obj)
+
+
+def _is_marking(obj_or_id):
+    """Determines whether the given object or object ID is/is for a marking
+    definition.
+
+    :param obj_or_id: A STIX object or object ID as a string.
+    :return: True if a marking definition, False otherwise.
+    """
+
+    if isinstance(obj_or_id, _STIXBase):
+        id_ = obj_or_id.id
+    else:
+        id_ = obj_or_id
+
+    return id_.startswith("marking-definition--")
 
 
 class _ObjectFamily(object):
@@ -174,8 +196,9 @@ class MemorySink(DataSink):
         file_path = os.path.abspath(file_path)
 
         all_objs = itertools.chain.from_iterable(
-            obj_family.all_versions.values()
-            for obj_family in self._data.values()
+            value.all_versions.values() if isinstance(value, _ObjectFamily)
+            else [value]
+            for value in self._data.values()
         )
 
         if not os.path.exists(os.path.dirname(file_path)):
@@ -234,9 +257,13 @@ class MemorySource(DataSource):
 
         """
         stix_obj = None
-        object_family = self._data.get(stix_id)
-        if object_family:
-            stix_obj = object_family.latest_version
+
+        if _is_marking(stix_id):
+            stix_obj = self._data.get(stix_id)
+        else:
+            object_family = self._data.get(stix_id)
+            if object_family:
+                stix_obj = object_family.latest_version
 
         if stix_obj:
             all_filters = list(
@@ -269,9 +296,17 @@ class MemorySource(DataSource):
 
         """
         results = []
-        object_family = self._data.get(stix_id)
+        stix_objs_to_filter = None
+        if _is_marking(stix_id):
+            stix_obj = self._data.get(stix_id)
+            if stix_obj:
+                stix_objs_to_filter = [stix_obj]
+        else:
+            object_family = self._data.get(stix_id)
+            if object_family:
+                stix_objs_to_filter = object_family.all_versions.values()
 
-        if object_family:
+        if stix_objs_to_filter:
             all_filters = list(
                 itertools.chain(
                     _composite_filters or [],
@@ -280,8 +315,7 @@ class MemorySource(DataSource):
             )
 
             results.extend(
-                apply_common_filters(object_family.all_versions.values(),
-                                     all_filters)
+                apply_common_filters(stix_objs_to_filter, all_filters)
             )
 
         return results
@@ -314,8 +348,9 @@ class MemorySource(DataSource):
             query.add(_composite_filters)
 
         all_objs = itertools.chain.from_iterable(
-            obj_family.all_versions.values()
-            for obj_family in self._data.values()
+            value.all_versions.values() if isinstance(value, _ObjectFamily)
+            else [value]
+            for value in self._data.values()
         )
 
         # Apply STIX common property filters.
diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py
index 60ea33b..cdb3818 100644
--- a/stix2/test/test_datastore_memory.py
+++ b/stix2/test/test_datastore_memory.py
@@ -2,8 +2,9 @@ import pytest
 
 from stix2.datastore import CompositeDataSource, make_id
 from stix2.datastore.filters import Filter
-from stix2.datastore.memory import MemorySink, MemorySource
+from stix2.datastore.memory import MemorySink, MemorySource, MemoryStore
 from stix2.utils import parse_into_datetime
+from stix2.v20.common import TLP_GREEN
 
 
 def test_add_remove_composite_datasource():
@@ -84,3 +85,65 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2):
     # nothing returns the same as cds1.query(query1) (the associated query is query2)
     results = cds1.query([])
     assert len(results) == 4
+
+
+def test_source_markings():
+    msrc = MemorySource(TLP_GREEN)
+
+    assert msrc.get(TLP_GREEN.id) == TLP_GREEN
+    assert msrc.all_versions(TLP_GREEN.id) == [TLP_GREEN]
+    assert msrc.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
+
+
+def test_sink_markings():
+    # just make sure there is no crash
+    msink = MemorySink(TLP_GREEN)
+    msink.add(TLP_GREEN)
+
+
+def test_store_markings():
+    mstore = MemoryStore(TLP_GREEN)
+
+    assert mstore.get(TLP_GREEN.id) == TLP_GREEN
+    assert mstore.all_versions(TLP_GREEN.id) == [TLP_GREEN]
+    assert mstore.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
+
+
+def test_source_mixed(indicator):
+    msrc = MemorySource([TLP_GREEN, indicator])
+
+    assert msrc.get(TLP_GREEN.id) == TLP_GREEN
+    assert msrc.all_versions(TLP_GREEN.id) == [TLP_GREEN]
+    assert msrc.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
+
+    assert msrc.get(indicator.id) == indicator
+    assert msrc.all_versions(indicator.id) == [indicator]
+    assert msrc.query(Filter("id", "=", indicator.id)) == [indicator]
+
+    all_objs = msrc.query()
+    assert TLP_GREEN in all_objs
+    assert indicator in all_objs
+    assert len(all_objs) == 2
+
+
+def test_sink_mixed(indicator):
+    # just make sure there is no crash
+    msink = MemorySink([TLP_GREEN, indicator])
+    msink.add([TLP_GREEN, indicator])
+
+
+def test_store_mixed(indicator):
+    mstore = MemoryStore([TLP_GREEN, indicator])
+
+    assert mstore.get(TLP_GREEN.id) == TLP_GREEN
+    assert mstore.all_versions(TLP_GREEN.id) == [TLP_GREEN]
+    assert mstore.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
+
+    assert mstore.get(indicator.id) == indicator
+    assert mstore.all_versions(indicator.id) == [indicator]
+    assert mstore.query(Filter("id", "=", indicator.id)) == [indicator]
+
+    all_objs = mstore.query()
+    assert TLP_GREEN in all_objs
+    assert indicator in all_objs
+    assert len(all_objs) == 2

From 6613b55a433d4278c46f51327569fce0f125724c Mon Sep 17 00:00:00 2001
From: Chris Lenk <clenk@mitre.org>
Date: Tue, 23 Oct 2018 14:29:56 -0400
Subject: [PATCH 07/11] Update MemoryStore documentation

...now that MemorySource normalizes its data to python-stix2 objects.
---
 docs/guide/memory.ipynb   |  4 ++--
 stix2/datastore/memory.py | 15 +++------------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb
index 6b6d5cb..491187b 100644
--- a/docs/guide/memory.ipynb
+++ b/docs/guide/memory.ipynb
@@ -62,11 +62,11 @@
     "\n",
     "\n",
     "### Memory API\n",
-    "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n",
+    "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) actually stores and retrieves STIX content as python-stix2 objects.\n",
     "\n",
     "A note on [load_from_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.load_from_file): For [load_from_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n",
     "\n",
-    "A note on [save_to_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore). \n",
+    "A note on [save_to_file()](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/datastore/stix2.datastore.memory.rst#stix2.datastore.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle.\n",
     "\n",
     "### Memory Examples\n",
     "\n",
diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py
index eb8580e..4b82c34 100644
--- a/stix2/datastore/memory.py
+++ b/stix2/datastore/memory.py
@@ -250,10 +250,7 @@ class MemorySource(DataSource):
                 CompositeDataSource, not user supplied
 
         Returns:
-            (dict OR STIX object): STIX object that has the supplied
-                ID. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory
-                as they are supplied (either as python dictionary or STIX object), it
-                is returned in the same form as it as added
+            (STIX object): STIX object that has the supplied ID.
 
         """
         stix_obj = None
@@ -289,10 +286,7 @@ class MemorySource(DataSource):
                 CompositeDataSource, not user supplied
 
         Returns:
-            (list): list of STIX objects that has the supplied ID. As the
-                MemoryStore(i.e. MemorySink) adds STIX objects to memory as they
-                are supplied (either as python dictionary or STIX object), it
-                is returned in the same form as it as added
+            (list): list of STIX objects that have the supplied ID.
 
         """
         results = []
@@ -333,10 +327,7 @@ class MemorySource(DataSource):
                 CompositeDataSource, not user supplied
 
         Returns:
-            (list): list of STIX objects that matches the supplied
-                query. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory
-                as they are supplied (either as python dictionary or STIX object), it
-                is returned in the same form as it was added.
+            (list): list of STIX objects that match the supplied query.
 
         """
         query = FilterSet(query)

From db300d1f21e33eb5fd0e267e07b5422eca72466f Mon Sep 17 00:00:00 2001
From: Chris Lenk <clenk@mitre.org>
Date: Thu, 25 Oct 2018 13:58:21 -0400
Subject: [PATCH 08/11] Fix `created` millisecond precision in TLPs

A marking definition's `created` property doesn't require millisecond
preprecision, but for TLP markings the TLP instances provided in the
spec must be used and they all use millisecond precision.
---
 stix2/test/test_markings.py | 2 +-
 stix2/v20/common.py         | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py
index ec8b664..49803f3 100644
--- a/stix2/test/test_markings.py
+++ b/stix2/test/test_markings.py
@@ -11,7 +11,7 @@ from .constants import MARKING_DEFINITION_ID
 EXPECTED_TLP_MARKING_DEFINITION = """{
     "type": "marking-definition",
     "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
-    "created": "2017-01-20T00:00:00Z",
+    "created": "2017-01-20T00:00:00.000Z",
     "definition_type": "tlp",
     "definition": {
         "tlp": "white"
diff --git a/stix2/v20/common.py b/stix2/v20/common.py
index c66b8f6..d574c92 100644
--- a/stix2/v20/common.py
+++ b/stix2/v20/common.py
@@ -1,6 +1,7 @@
 """STIX 2 Common Data Types and Properties."""
 
 from collections import OrderedDict
+import copy
 
 from ..base import _cls_init, _STIXBase
 from ..markings import _MarkingsMixin
@@ -124,6 +125,13 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin):
             except KeyError:
                 raise ValueError("definition_type must be a valid marking type")
 
+            if marking_type == TLPMarking:
+                # TLP instances in the spec have millisecond precision unlike other markings
+                self._properties = copy.deepcopy(self._properties)
+                self._properties.update([
+                    ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')),
+                ])
+
             if not isinstance(kwargs['definition'], marking_type):
                 defn = _get_dict(kwargs['definition'])
                 kwargs['definition'] = marking_type(**defn)

From e521e24387c3ef68baa76dbd833b8ed60ecc2d16 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez <emmanuelle@mitre.org>
Date: Wed, 31 Oct 2018 13:39:09 -0400
Subject: [PATCH 09/11] Update the details on technical specification support

---
 docs/guide/ts_support.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb
index 445e263..8c89e93 100644
--- a/docs/guide/ts_support.ipynb
+++ b/docs/guide/ts_support.ipynb
@@ -230,7 +230,7 @@
    "metadata": {},
    "source": [
     "### How parsing works\n",
-    "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n",
+    "If the ``version`` positional argument is not provided. The library will make the best attempt using the \"spec_version\" property found on a Bundle, SDOs, and SROs.\n",
     "\n",
     "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:"
    ]

From ee7e997a14c6bce43055cc75a6c5495071f5aed8 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez <emmanuelle@mitre.org>
Date: Wed, 31 Oct 2018 13:42:50 -0400
Subject: [PATCH 10/11] Update CHANGELOG for v1.0.3

---
 CHANGELOG | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 596f71c..20f10ff 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,18 @@
 CHANGELOG
 =========
 
+1.0.3 - 2018-10-31
+
+* #187 Pickle proof objects
+* #181 Improvements to error handling for datastores
+* #184, #192 Fix "pretty" JSON serialization problems
+* #195 Fix wrong property name for raster-image-ext
+* #201 UUIDv4 enforcement on identifiers
+* #203 New filter option "contains" for datastores
+* #207 Add documentation on patterning
+* #213 Python 3.7 support
+* #214 Support for multiple object versions in MemoryStore
+
 1.0.2 - 2018-05-18
 
 * Fixes bugs when using allow_custom (#176, #179).

From f85f4e566bf87ae8e7238c4ceb972e24c6aeb443 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez <emmanuelle@mitre.org>
Date: Wed, 31 Oct 2018 13:46:47 -0400
Subject: [PATCH 11/11] =?UTF-8?q?Bump=20version:=201.0.2=20=E2=86=92=201.0?=
 =?UTF-8?q?.3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/conf.py     | 4 ++--
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 9b1ad5d..fb3efe3 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,8 +34,8 @@ project = 'stix2'
 copyright = '2017, OASIS Open'
 author = 'OASIS Open'
 
-version = '1.0.2'
-release = '1.0.2'
+version = '1.0.3'
+release = '1.0.3'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints']
diff --git a/setup.cfg b/setup.cfg
index 62da89c..97f5047 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.2
+current_version = 1.0.3
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index 7863915..976498a 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "1.0.2"
+__version__ = "1.0.3"