Merge branch 'master' of github.com:oasis-open/cti-python-stix2 into 1.1.0-release

docs/Makefile

@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api/datastore/stix2.datastore.filesystem.rst

@@ -2,4 +2,4 @@ filesystem
 ==========================
 
 .. automodule:: stix2.datastore.filesystem
-   :members:
+   :members:

docs/api/datastore/stix2.datastore.filters.rst

@@ -2,4 +2,4 @@ filters
 =======================
 
 .. automodule:: stix2.datastore.filters
-   :members:
+   :members:

docs/api/datastore/stix2.datastore.memory.rst

@@ -2,4 +2,4 @@ memory
 ======================
 
 .. automodule:: stix2.datastore.memory
-   :members:
+   :members:

docs/api/datastore/stix2.datastore.taxii.rst

@@ -2,4 +2,4 @@ taxii
 =====================
 
 .. automodule:: stix2.datastore.taxii
-   :members:
+   :members:

docs/api/markings/stix2.markings.granular_markings.rst

@@ -2,4 +2,4 @@ granular_markings
 ================================
 
 .. automodule:: stix2.markings.granular_markings
-   :members:
+   :members:

docs/api/markings/stix2.markings.object_markings.rst

@@ -2,4 +2,4 @@ object_markings
 ==============================
 
 .. automodule:: stix2.markings.object_markings
-   :members:
+   :members:

docs/api/markings/stix2.markings.utils.rst

@@ -2,4 +2,4 @@ utils
 ====================
 
 .. automodule:: stix2.markings.utils
-   :members:
+   :members:

docs/api/stix2.core.rst

@@ -2,4 +2,4 @@ core
 ==========
 
 .. automodule:: stix2.core
-   :members:
+   :members:

docs/api/stix2.datastore.rst

@@ -2,4 +2,4 @@ datastore
 ===============
 
 .. automodule:: stix2.datastore
-   :members:
+   :members:

docs/api/stix2.environment.rst

@@ -2,4 +2,4 @@ environment
 =================
 
 .. automodule:: stix2.environment
-   :members:
+   :members:

docs/api/stix2.exceptions.rst

@@ -2,4 +2,4 @@ exceptions
 ================
 
 .. automodule:: stix2.exceptions
-   :members:
+   :members:

docs/api/stix2.markings.rst

@@ -2,4 +2,4 @@ markings
 ==============
 
 .. automodule:: stix2.markings
-   :members:
+   :members:

docs/api/stix2.patterns.rst

@@ -2,4 +2,4 @@ patterns
 ==============
 
 .. automodule:: stix2.patterns
-   :members:
+   :members:

docs/api/stix2.properties.rst

@@ -2,4 +2,4 @@ properties
 ================
 
 .. automodule:: stix2.properties
-   :members:
+   :members:

docs/api/stix2.utils.rst

@@ -2,4 +2,4 @@ utils
 ===========
 
 .. automodule:: stix2.utils
-   :members:
+   :members:

docs/api/stix2.v20.common.rst

@@ -2,4 +2,4 @@ common
 ================
 
 .. automodule:: stix2.v20.common
-   :members:
+   :members:

docs/api/stix2.v20.observables.rst

@@ -2,4 +2,4 @@ observables
 =====================
 
 .. automodule:: stix2.v20.observables
-   :members:
+   :members:

docs/api/stix2.v20.sdo.rst

@@ -2,4 +2,4 @@ sdo
 =============
 
 .. automodule:: stix2.v20.sdo
-   :members:
+   :members:

docs/api/stix2.v20.sro.rst

@@ -2,4 +2,4 @@ sro
 =============
 
 .. automodule:: stix2.v20.sro
-   :members:
+   :members:

docs/api/stix2.workbench.rst

@@ -2,4 +2,4 @@ workbench
 ===============
 
 .. automodule:: stix2.workbench
-   :members:
+   :members:

stix2/datastore/filesystem.py

@@ -1,14 +1,442 @@
 """Python STIX2 FileSystem Source/Sink"""
 
+import errno
 import io
 import json
 import os
+import stat
+
+import pytz
+import six
 
 from stix2 import v20, v21
+from stix2.base import _STIXBase
 from stix2.core import parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
-from stix2.utils import deduplicate, get_class_hierarchy_names
+from stix2.utils import get_type_from_id, is_marking
+
+
+def _timestamp2filename(timestamp):
+    """
+    Encapsulates a way to create unique filenames based on an object's
+    "modified" property value.  This should not include an extension.
+
+    Args:
+        timestamp: A timestamp, as a datetime.datetime object.
+
+    """
+    # Different times will only produce different file names if all timestamps
+    # are in the same time zone!  So if timestamp is timezone-aware convert
+    # to UTC just to be safe.  If naive, just use as-is.
+    if timestamp.tzinfo is not None:
+        timestamp = timestamp.astimezone(pytz.utc)
+
+    return timestamp.strftime("%Y%m%d%H%M%S%f")
+
+
+class AuthSet(object):
+    """
+    Represents either a whitelist or blacklist of values, where/what we
+    must/must not search to find objects which match a query.  (Maybe "AuthSet"
+    isn't the right name, but determining authorization is a typical context in
+    which black/white lists are used.)
+
+    The set may be empty.  For a whitelist, this means you mustn't search
+    anywhere, which means the query was impossible to match, so you can skip
+    searching altogether.  For a blacklist, this means nothing is excluded
+    and you must search everywhere.
+
+    """
+    BLACK = 0
+    WHITE = 1
+
+    def __init__(self, allowed, prohibited):
+        """
+        Initialize this AuthSet from the given sets of allowed and/or
+        prohibited values.  The type of set (black or white) is determined
+        from the allowed and/or prohibited values given.
+
+        Args:
+            allowed: A set of allowed values (or None if no allow filters
+                were found in the query)
+            prohibited: A set of prohibited values (not None)
+
+        """
+        if allowed is None:
+            self.__values = prohibited
+            self.__type = AuthSet.BLACK
+
+        else:
+            # There was at least one allow filter, so create a whitelist.  But
+            # any matching prohibited values create a combination of conditions
+            # which can never match.  So exclude those.
+            self.__values = allowed - prohibited
+            self.__type = AuthSet.WHITE
+
+    @property
+    def values(self):
+        """
+        Get the values in this white/blacklist, as a set.
+        """
+        return self.__values
+
+    @property
+    def auth_type(self):
+        """
+        Get the type of set: AuthSet.WHITE or AuthSet.BLACK.
+        """
+        return self.__type
+
+    def __repr__(self):
+        return "{}list: {}".format(
+            "white" if self.auth_type == AuthSet.WHITE else "black",
+            self.values
+        )
+
+
+# A fixed, reusable AuthSet which accepts anything.  It came in handy.
+_AUTHSET_ANY = AuthSet(None, set())
+
+
+def _update_allow(allow_set, value):
+    """
+    Updates the given set of "allow" values.  The first time an update to the
+    set occurs, the value(s) are added.  Thereafter, since all filters are
+    implicitly AND'd, the given values are intersected with the existing allow
+    set, which may remove values.  At the end, it may even wind up empty.
+
+    Args:
+        allow_set: The allow set, or None
+        value: The value(s) to add (single value, or iterable of values)
+
+    Returns:
+        The updated allow set (not None)
+
+    """
+    adding_seq = hasattr(value, "__iter__") and \
+        not isinstance(value, six.string_types)
+
+    if allow_set is None:
+        allow_set = set()
+        if adding_seq:
+            allow_set.update(value)
+        else:
+            allow_set.add(value)
+    else:
+        # strangely, the "&=" operator requires a set on the RHS
+        # whereas the method allows any iterable.
+        if adding_seq:
+            allow_set.intersection_update(value)
+        else:
+            allow_set.intersection_update({value})
+
+    return allow_set
+
+
+def _find_search_optimizations(filters):
+    """
+    Searches through all the filters, and creates white/blacklists of types and
+    IDs, which can be used to optimize the filesystem search.
+
+    Args:
+        filters: An iterable of filter objects representing a query
+
+    Returns:
+        A 2-tuple of AuthSet objects: the first is for object types, and
+        the second is for object IDs.
+
+    """
+    # The basic approach to this is to determine what is allowed and
+    # prohibited, independently, and then combine them to create the final
+    # white/blacklists.
+
+    allowed_types = allowed_ids = None
+    prohibited_types = set()
+    prohibited_ids = set()
+
+    for filter_ in filters:
+        if filter_.property == "type":
+            if filter_.op in ("=", "in"):
+                allowed_types = _update_allow(allowed_types, filter_.value)
+            elif filter_.op == "!=":
+                prohibited_types.add(filter_.value)
+
+        elif filter_.property == "id":
+            if filter_.op == "=":
+                # An "allow" ID filter implies a type filter too, since IDs
+                # contain types within them.
+                allowed_ids = _update_allow(allowed_ids, filter_.value)
+                allowed_types = _update_allow(allowed_types,
+                                              get_type_from_id(filter_.value))
+            elif filter_.op == "!=":
+                prohibited_ids.add(filter_.value)
+            elif filter_.op == "in":
+                allowed_ids = _update_allow(allowed_ids, filter_.value)
+                allowed_types = _update_allow(allowed_types, (
+                    get_type_from_id(id_) for id_ in filter_.value
+                ))
+
+    opt_types = AuthSet(allowed_types, prohibited_types)
+    opt_ids = AuthSet(allowed_ids, prohibited_ids)
+
+    # If we have both type and ID whitelists, perform a type-based intersection
+    # on them, to further optimize.  (Some of the cross-property constraints
+    # occur above; this is essentially a second pass which operates on the
+    # final whitelists, which among other things, incorporates any of the
+    # prohibitions found above.)
+    if opt_types.auth_type == AuthSet.WHITE and \
+            opt_ids.auth_type == AuthSet.WHITE:
+
+        opt_types.values.intersection_update(
+            get_type_from_id(id_) for id_ in opt_ids.values
+        )
+
+        opt_ids.values.intersection_update(
+            id_ for id_ in opt_ids.values
+            if get_type_from_id(id_) in opt_types.values
+        )
+
+    return opt_types, opt_ids
+
+
+def _get_matching_dir_entries(parent_dir, auth_set, st_mode_test=None, ext=""):
+    """
+    Search a directory (non-recursively), and find entries which match the
+    given criteria.
+
+    Args:
+        parent_dir: The directory to search
+        auth_set: an AuthSet instance, which represents a black/whitelist
+            filter on filenames
+        st_mode_test: A callable allowing filtering based on the type of
+            directory entry.  E.g. just get directories, or just get files.  It
+            will be passed the st_mode field of a stat() structure and should
+            return True to include the file, or False to exclude it.  Easy thing to
+            do is pass one of the stat module functions, e.g. stat.S_ISREG.  If
+            None, don't filter based on entry type.
+        ext: Determines how names from auth_set match up to directory
+            entries, and allows filtering by extension.  The extension is added
+            to auth_set values to obtain directory entries; it is removed from
+            directory entries to obtain auth_set values.  In this way, auth_set
+            may be treated as having only "basenames" of the entries.  Only entries
+            having the given extension will be included in the results.  If not
+            empty, the extension MUST include a leading ".".  The default is the
+            empty string, which will result in direct comparisons, and no
+            extension-based filtering.
+
+    Returns:
+        (list): A list of directory entries matching the criteria.  These will not
+            have any path info included; they will just be bare names.
+
+    Raises:
+        OSError: If there are errors accessing directory contents or stat()'ing
+            files
+
+    """
+    results = []
+    if auth_set.auth_type == AuthSet.WHITE:
+        for value in auth_set.values:
+            filename = value + ext
+            try:
+                if st_mode_test:
+                    s = os.stat(os.path.join(parent_dir, filename))
+                    type_pass = st_mode_test(s.st_mode)
+                else:
+                    type_pass = True
+
+                if type_pass:
+                    results.append(filename)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                # else, file-not-found is ok, just skip
+    else:  # auth_set is a blacklist
+        for entry in os.listdir(parent_dir):
+            if ext:
+                auth_name, this_ext = os.path.splitext(entry)
+                if this_ext != ext:
+                    continue
+            else:
+                auth_name = entry
+
+            if auth_name in auth_set.values:
+                continue
+
+            try:
+                if st_mode_test:
+                    s = os.stat(os.path.join(parent_dir, entry))
+                    type_pass = st_mode_test(s.st_mode)
+                else:
+                    type_pass = True
+
+                if type_pass:
+                    results.append(entry)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                # else, file-not-found is ok, just skip
+
+    return results
+
+
+def _check_object_from_file(query, filepath, allow_custom, version):
+    """
+    Read a STIX object from the given file, and check it against the given
+    filters.
+
+    Args:
+        query: Iterable of filters
+        filepath: Path to file to read
+        allow_custom: Whether to allow custom properties as well unknown
+        custom objects.
+        version (str): If present, it forces the parser to use the version
+            provided. Otherwise, the library will make the best effort based
+            on checking the "spec_version" property.
+
+    Returns:
+        The (parsed) STIX object, if the object passes the filters.  If
+        not, None is returned.
+
+    Raises:
+        TypeError: If the file had invalid JSON
+        IOError: If there are problems opening/reading the file
+        stix2.exceptions.STIXError: If there were problems creating a STIX
+            object from the JSON
+
+    """
+    try:
+        with io.open(filepath, "r") as f:
+            stix_json = json.load(f)
+    except ValueError:  # not a JSON file
+        raise TypeError(
+            "STIX JSON object at '{0}' could either not be parsed "
+            "to JSON or was not valid STIX JSON".format(filepath)
+        )
+
+    stix_obj = parse(stix_json, allow_custom, version)
+
+    if stix_obj["type"] == "bundle":
+        stix_obj = stix_obj["objects"][0]
+
+    # check against other filters, add if match
+    result = next(apply_common_filters([stix_obj], query), None)
+
+    return result
+
+
+def _search_versioned(query, type_path, auth_ids, allow_custom, version):
+    """
+    Searches the given directory, which contains data for STIX objects of a
+    particular versioned type (i.e. not markings), and return any which match
+    the query.
+
+    Args:
+        query: The query to match against
+        type_path: The directory with type-specific STIX object files
+        auth_ids: Search optimization based on object ID
+        allow_custom: Whether to allow custom properties as well unknown
+            custom objects.
+        version (str): If present, it forces the parser to use the version
+            provided. Otherwise, the library will make the best effort based
+            on checking the "spec_version" property.
+
+    Returns:
+        A list of all matching objects
+
+    Raises:
+        stix2.exceptions.STIXError: If any objects had invalid content
+        TypeError: If any objects had invalid content
+        IOError: If there were any problems opening/reading files
+        OSError: If there were any problems opening/reading files
+
+    """
+    results = []
+    id_dirs = _get_matching_dir_entries(type_path, auth_ids,
+                                        stat.S_ISDIR)
+    for id_dir in id_dirs:
+        id_path = os.path.join(type_path, id_dir)
+
+        # This leverages a more sophisticated function to do a simple thing:
+        # get all the JSON files from a directory.  I guess it does give us
+        # file type checking, ensuring we only get regular files.
+        version_files = _get_matching_dir_entries(id_path, _AUTHSET_ANY,
+                                                  stat.S_ISREG, ".json")
+        for version_file in version_files:
+            version_path = os.path.join(id_path, version_file)
+
+            try:
+                stix_obj = _check_object_from_file(query, version_path,
+                                                   allow_custom, version)
+                if stix_obj:
+                    results.append(stix_obj)
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                # else, file-not-found is ok, just skip
+
+    # For backward-compatibility, also search for plain files named after
+    # object IDs, in the type directory.
+    id_files = _get_matching_dir_entries(type_path, auth_ids, stat.S_ISREG,
+                                         ".json")
+    for id_file in id_files:
+        id_path = os.path.join(type_path, id_file)
+
+        try:
+            stix_obj = _check_object_from_file(query, id_path, allow_custom,
+                                               version)
+            if stix_obj:
+                results.append(stix_obj)
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            # else, file-not-found is ok, just skip
+
+    return results
+
+
+def _search_markings(query, markings_path, auth_ids, allow_custom, version):
+    """
+    Searches the given directory, which contains markings data, and return any
+    which match the query.
+
+    Args:
+        query: The query to match against
+        markings_path: The directory with STIX markings files
+        auth_ids: Search optimization based on object ID
+        allow_custom: Whether to allow custom properties as well unknown
+            custom objects.
+        version (str): If present, it forces the parser to use the version
+            provided. Otherwise, the library will make the best effort based
+            on checking the "spec_version" property.
+
+    Returns:
+        A list of all matching objects
+
+    Raises:
+        stix2.exceptions.STIXError: If any objects had invalid content
+        TypeError: If any objects had invalid content
+        IOError: If there were any problems opening/reading files
+        OSError: If there were any problems opening/reading files
+
+    """
+    results = []
+    id_files = _get_matching_dir_entries(markings_path, auth_ids, stat.S_ISREG,
+                                         ".json")
+    for id_file in id_files:
+        id_path = os.path.join(markings_path, id_file)
+
+        try:
+            stix_obj = _check_object_from_file(query, id_path, allow_custom,
+                                               version)
+            if stix_obj:
+                results.append(stix_obj)
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            # else, file-not-found is ok, just skip
+
+    return results
 
 
 class FileSystemStore(DataStoreMixin):
@@ -20,12 +448,12 @@ class FileSystemStore(DataStoreMixin):
     Args:
         stix_dir (str): path to directory of STIX objects
         allow_custom (bool): whether to allow custom STIX content to be
-            pushed/retrieved. Defaults to True for FileSystemSource side(retrieving data)
-            and False for FileSystemSink side(pushing data). However, when
-            parameter is supplied, it will be applied to both FileSystemSource
-            and FileSystemSink.
-        bundlify (bool): whether to wrap objects in bundles when saving them.
-            Default: False.
+            pushed/retrieved. Defaults to True for FileSystemSource side
+            (retrieving data) and False for FileSystemSink
+            side(pushing data). However, when parameter is supplied, it
+            will be applied to both FileSystemSource and FileSystemSink.
+        bundlify (bool): whether to wrap objects in bundles when saving
+            them. Default: False.
 
     Attributes:
         source (FileSystemSource): FileSystemSource
@@ -76,10 +504,18 @@ class FileSystemSink(DataSink):
     def _check_path_and_write(self, stix_obj, encoding='utf-8'):
         """Write the given STIX object to a file in the STIX file directory.
         """
-        path = os.path.join(self._stix_dir, stix_obj['type'], stix_obj['id'] + '.json')
+        type_dir = os.path.join(self._stix_dir, stix_obj["type"])
+        if is_marking(stix_obj):
+            filename = stix_obj["id"]
+            obj_dir = type_dir
+        else:
+            filename = _timestamp2filename(stix_obj["modified"])
+            obj_dir = os.path.join(type_dir, stix_obj["id"])
 
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
+        file_path = os.path.join(obj_dir, filename + ".json")
+
+        if not os.path.exists(obj_dir):
+            os.makedirs(obj_dir)
 
         if self.bundlify:
             if 'spec_version' in stix_obj:
@@ -90,17 +526,20 @@ class FileSystemSink(DataSink):
             else:
                 stix_obj = v20.Bundle(stix_obj, allow_custom=self.allow_custom)
 
-        with io.open(path, 'w', encoding=encoding) as f:
+        with io.open(file_path, 'w', encoding=encoding) as f:
             stix_obj = stix_obj.serialize(pretty=True, encoding=encoding, ensure_ascii=False)
             f.write(stix_obj)
 
-    def add(self, stix_data=None):
+    def add(self, stix_data=None, version=None):
         """Add STIX objects to file directory.
 
         Args:
             stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content
                 in a STIX object (or list of), dict (or list of), or a STIX 2.0
                 json encoded string.
+            version (str): If present, it forces the parser to use the version
+                provided. Otherwise, the library will make the best effort based
+                on checking the "spec_version" property.
 
         Note:
             ``stix_data`` can be a Bundle object, but each object in it will be
@@ -108,25 +547,18 @@ class FileSystemSink(DataSink):
             the Bundle contained, but not the Bundle itself.
 
         """
-        if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition')
-               for x in get_class_hierarchy_names(stix_data)):
+        if isinstance(stix_data, (v20.Bundle, v21.Bundle)):
+            # recursively add individual STIX objects
+            for stix_obj in stix_data.get("objects", []):
+                self.add(stix_obj, version=version)
+
+        elif isinstance(stix_data, _STIXBase):
             # adding python STIX object
             self._check_path_and_write(stix_data)
 
         elif isinstance(stix_data, (str, dict)):
-            stix_data = parse(stix_data, allow_custom=self.allow_custom)
-            if stix_data['type'] == 'bundle':
-                # extract STIX objects
-                for stix_obj in stix_data.get('objects', []):
-                    self.add(stix_obj)
-            else:
-                # adding json-formatted STIX
-                self._check_path_and_write(stix_data)
-
-        elif 'Bundle' in get_class_hierarchy_names(stix_data):
-            # recursively add individual STIX objects
-            for stix_obj in stix_data.get('objects', []):
-                self.add(stix_obj)
+            stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version)
+            self.add(stix_data, version=version)
 
         elif isinstance(stix_data, list):
             # recursively add individual STIX objects
@@ -137,7 +569,7 @@ class FileSystemSink(DataSink):
             raise TypeError(
                 "stix_data must be a STIX object (or list of), "
                 "JSON formatted STIX (or list of), "
-                "or a JSON formatted STIX bundle",
+                "or a JSON formatted STIX bundle"
             )
 
 
@@ -166,15 +598,16 @@ class FileSystemSource(DataSource):
     def stix_dir(self):
         return self._stix_dir
 
-    def get(self, stix_id, _composite_filters=None):
+    def get(self, stix_id, version=None, _composite_filters=None):
         """Retrieve STIX object from file directory via STIX ID.
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
             _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
-            version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
-                None, use latest version.
+            version (str): If present, it forces the parser to use the version
+                provided. Otherwise, the library will make the best effort based
+                on checking the "spec_version" property.
 
         Returns:
             (STIX object): STIX object that has the supplied STIX ID.
@@ -182,18 +615,21 @@ class FileSystemSource(DataSource):
                 a python STIX object and then returned
 
         """
-        query = [Filter('id', '=', stix_id)]
-
-        all_data = self.query(query=query, _composite_filters=_composite_filters)
+        all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters)
 
         if all_data:
-            stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
+            if is_marking(stix_id):
+                # Markings are unversioned; there shouldn't be more than one
+                # result.
+                stix_obj = all_data[0]
+            else:
+                stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1]
         else:
             stix_obj = None
 
         return stix_obj
 
-    def all_versions(self, stix_id, _composite_filters=None):
+    def all_versions(self, stix_id, version=None, _composite_filters=None):
         """Retrieve STIX object from file directory via STIX ID, all versions.
 
         Note: Since FileSystem sources/sinks don't handle multiple versions
@@ -202,7 +638,10 @@ class FileSystemSource(DataSource):
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
             _composite_filters (FilterSet): collection of filters passed from
-                the parent CompositeDataSource, not user supplied.
+                the parent CompositeDataSource, not user supplied
+            version (str): If present, it forces the parser to use the version
+                provided. Otherwise, the library will make the best effort based
+                on checking the "spec_version" property.
 
         Returns:
             (list): of STIX objects that has the supplied STIX ID.
@@ -210,9 +649,10 @@ class FileSystemSource(DataSource):
                 a python STIX objects and then returned
 
         """
-        return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
+        query = [Filter("id", "=", stix_id)]
+        return self.query(query, version=version, _composite_filters=_composite_filters)
 
-    def query(self, query=None, _composite_filters=None):
+    def query(self, query=None, version=None, _composite_filters=None):
         """Search and retrieve STIX objects based on the complete query.
 
         A "complete query" includes the filters from the query, the filters
@@ -223,15 +663,17 @@ class FileSystemSource(DataSource):
             query (list): list of filters to search on
             _composite_filters (FilterSet): collection of filters passed from
                 the CompositeDataSource, not user supplied
+            version (str): If present, it forces the parser to use the version
+                provided. Otherwise, the library will make the best effort based
+                on checking the "spec_version" property.
 
         Returns:
-            stix_objs (list): list of STIX objects that matches the supplied
+            (list): list of STIX objects that matches the supplied
                 query. The STIX objects are loaded from their json files,
                 parsed into a python STIX objects and then returned.
 
         """
         all_data = []
-
         query = FilterSet(query)
 
         # combine all query filters
@@ -240,105 +682,17 @@ class FileSystemSource(DataSource):
         if _composite_filters:
             query.add(_composite_filters)
 
-        # extract any filters that are for "type" or "id" , as we can then do
-        # filtering before reading in the STIX objects. A STIX 'type' filter
-        # can reduce the query to a single sub-directory. A STIX 'id' filter
-        # allows for the fast checking of the file names versus loading it.
-        file_filters = self._parse_file_filters(query)
-
-        # establish which subdirectories can be avoided in query
-        # by decluding as many as possible. A filter with "type" as the property
-        # means that certain STIX object types can be ruled out, and thus
-        # the corresponding subdirectories as well
-        include_paths = []
-        declude_paths = []
-        if 'type' in [filter.property for filter in file_filters]:
-            for filter in file_filters:
-                if filter.property == 'type':
-                    if filter.op == '=':
-                        include_paths.append(os.path.join(self._stix_dir, filter.value))
-                    elif filter.op == '!=':
-                        declude_paths.append(os.path.join(self._stix_dir, filter.value))
-        else:
-            # have to walk entire STIX directory
-            include_paths.append(self._stix_dir)
-
-        # if a user specifies a "type" filter like "type = <stix-object_type>",
-        # the filter is reducing the search space to single stix object types
-        # (and thus single directories). This makes such a filter more powerful
-        # than "type != <stix-object_type>" bc the latter is substracting
-        # only one type of stix object type (and thus only one directory),
-        # As such the former type of filters are given preference over the latter;
-        # i.e. if both exist in a query, that latter type will be ignored
-
-        if not include_paths:
-            # user has specified types that are not wanted (i.e. "!=")
-            # so query will look in all STIX directories that are not
-            # the specified type. Compile correct dir paths
-            for dir in os.listdir(self._stix_dir):
-                if os.path.abspath(os.path.join(self._stix_dir, dir)) not in declude_paths:
-                    include_paths.append(os.path.abspath(os.path.join(self._stix_dir, dir)))
-
-        # grab stix object ID as well - if present in filters, as
-        # may forgo the loading of STIX content into memory
-        if 'id' in [filter.property for filter in file_filters]:
-            for filter in file_filters:
-                if filter.property == 'id' and filter.op == '=':
-                    id_ = filter.value
-                    break
+        auth_types, auth_ids = _find_search_optimizations(query)
+        type_dirs = _get_matching_dir_entries(self._stix_dir, auth_types,
+                                              stat.S_ISDIR)
+        for type_dir in type_dirs:
+            type_path = os.path.join(self._stix_dir, type_dir)
+            if type_dir == "marking-definition":
+                type_results = _search_markings(query, type_path, auth_ids,
+                                                self.allow_custom, version)
             else:
-                id_ = None
-        else:
-            id_ = None
+                type_results = _search_versioned(query, type_path, auth_ids,
+                                                 self.allow_custom, version)
+            all_data.extend(type_results)
 
-        # now iterate through all STIX objs
-        for path in include_paths:
-            for root, dirs, files in os.walk(path):
-                for file_ in files:
-                    if not file_.endswith('.json'):
-                        # skip non '.json' files as more likely to be random non-STIX files
-                        continue
-
-                    if not id_ or id_ == file_.split('.')[0]:
-                        # have to load into memory regardless to evaluate other filters
-                        try:
-                            stix_obj = json.load(open(os.path.join(root, file_)))
-
-                            if stix_obj['type'] == 'bundle':
-                                stix_obj = stix_obj['objects'][0]
-
-                            # naive STIX type checking
-                            stix_obj['type']
-                            stix_obj['id']
-
-                        except (ValueError, KeyError):  # likely not a JSON file
-                            raise TypeError("STIX JSON object at '{0}' could either not be parsed to "
-                                            "JSON or was not valid STIX JSON".format(os.path.join(root, file_)))
-
-                        # check against other filters, add if match
-                        all_data.extend(apply_common_filters([stix_obj], query))
-
-        all_data = deduplicate(all_data)
-
-        # parse python STIX objects from the STIX object dicts
-        stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom) for stix_obj_dict in all_data]
-
-        return stix_objs
-
-    def _parse_file_filters(self, query):
-        """Extract STIX common filters.
-
-        Possibly speeds up querying STIX objects from the file system.
-
-        Extracts filters that are for the "id" and "type" property of
-        a STIX object. As the file directory is organized by STIX
-        object type with filenames that are equivalent to the STIX
-        object ID, these filters can be used first to reduce the
-        search space of a FileSystemStore (or FileSystemSink).
-
-        """
-        file_filters = []
-        for filter_ in query:
-            if filter_.property == 'id' or filter_.property == 'type':
-                file_filters.append(filter_)
-        return file_filters
+        return all_data

stix2/datastore/filters.py

@@ -3,18 +3,16 @@
 import collections
 from datetime import datetime
 
-from stix2.utils import format_datetime
+import six
+
+import stix2.utils
 
 """Supported filter operations"""
 FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=', 'contains']
 
 """Supported filter value types"""
-FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
-try:
-    FILTER_VALUE_TYPES.append(unicode)
-except NameError:
-    # Python 3 doesn't need to worry about unicode
-    pass
+FILTER_VALUE_TYPES = (bool, dict, float, int, list, tuple, six.string_types,
+                      datetime)
 
 
 def _check_filter_components(prop, op, value):
@@ -33,7 +31,7 @@ def _check_filter_components(prop, op, value):
         # check filter operator is supported
         raise ValueError("Filter operator '%s' not supported for specified property: '%s'" % (op, prop))
 
-    if type(value) not in FILTER_VALUE_TYPES:
+    if not isinstance(value, FILTER_VALUE_TYPES):
         # check filter value type is supported
         raise TypeError("Filter value of '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value))
 
@@ -66,10 +64,6 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])):
         if isinstance(value, list):
             value = tuple(value)
 
-        if isinstance(value, datetime):
-            # if value is a datetime obj, convert to str
-            value = format_datetime(value)
-
         _check_filter_components(prop, op, value)
 
         self = super(Filter, cls).__new__(cls, prop, op, value)
@@ -85,31 +79,33 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])):
             True if property matches the filter,
             False otherwise.
         """
-        if isinstance(stix_obj_property, datetime):
-            # if a datetime obj, convert to str format before comparison
-            # NOTE: this check seems like it should be done upstream
-            # but will put here for now
-            stix_obj_property = format_datetime(stix_obj_property)
+        # If filtering on a timestamp property and the filter value is a string,
+        # try to convert the filter value to a datetime instance.
+        if isinstance(stix_obj_property, datetime) and \
+                isinstance(self.value, six.string_types):
+            filter_value = stix2.utils.parse_into_datetime(self.value)
+        else:
+            filter_value = self.value
 
-        if self.op == '=':
-            return stix_obj_property == self.value
-        elif self.op == '!=':
-            return stix_obj_property != self.value
-        elif self.op == 'in':
-            return stix_obj_property in self.value
-        elif self.op == 'contains':
-            if isinstance(self.value, dict):
-                return self.value in stix_obj_property.values()
+        if self.op == "=":
+            return stix_obj_property == filter_value
+        elif self.op == "!=":
+            return stix_obj_property != filter_value
+        elif self.op == "in":
+            return stix_obj_property in filter_value
+        elif self.op == "contains":
+            if isinstance(filter_value, dict):
+                return filter_value in stix_obj_property.values()
             else:
-                return self.value in stix_obj_property
-        elif self.op == '>':
-            return stix_obj_property > self.value
-        elif self.op == '<':
-            return stix_obj_property < self.value
-        elif self.op == '>=':
-            return stix_obj_property >= self.value
-        elif self.op == '<=':
-            return stix_obj_property <= self.value
+                return filter_value in stix_obj_property
+        elif self.op == ">":
+            return stix_obj_property > filter_value
+        elif self.op == "<":
+            return stix_obj_property < filter_value
+        elif self.op == ">=":
+            return stix_obj_property >= filter_value
+        elif self.op == "<=":
+            return stix_obj_property <= filter_value
         else:
             raise ValueError("Filter operator: {0} not supported for specified property: {1}".format(self.op, self.property))
 
@@ -223,8 +219,9 @@ class FilterSet(object):
 
         Operates like set, only adding unique stix2.Filters to the FilterSet
 
-        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
-        as it allows for blind calls (very useful in DataStore)
+        Note:
+            method designed to be very accomodating (i.e. even accepting filters=None)
+            as it allows for blind calls (very useful in DataStore)
 
         Args:
             filters: stix2.Filter OR list of stix2.Filter OR stix2.FilterSet
@@ -245,11 +242,13 @@ class FilterSet(object):
     def remove(self, filters=None):
         """Remove a Filter, list of Filters, or FilterSet from the FilterSet.
 
-        NOTE: method designed to be very accomodating (i.e. even accepting filters=None)
-        as it allows for blind calls (very useful in DataStore)
+        Note:
+            method designed to be very accomodating (i.e. even accepting filters=None)
+            as it allows for blind calls (very useful in DataStore)
 
         Args:
             filters: stix2.Filter OR list of stix2.Filter or stix2.FilterSet
+
         """
         if not filters:
             # so remove() can be called blindly, useful for

stix2/datastore/memory.py

@@ -10,6 +10,7 @@ from stix2.base import _STIXBase
 from stix2.core import parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.filters import FilterSet, apply_common_filters
+from stix2.utils import is_marking
 
 
 def _add(store, stix_data, allow_custom=True, version=None):
@@ -48,7 +49,7 @@ def _add(store, stix_data, allow_custom=True, version=None):
 
         # Map ID directly to the object, if it is a marking.  Otherwise,
         # map to a family, so we can track multiple versions.
-        if _is_marking(stix_obj):
+        if is_marking(stix_obj):
             store._data[stix_obj["id"]] = stix_obj
 
         else:
@@ -61,22 +62,6 @@ def _add(store, stix_data, allow_custom=True, version=None):
             obj_family.add(stix_obj)
 
 
-def _is_marking(obj_or_id):
-    """Determines whether the given object or object ID is/is for a marking
-    definition.
-
-    :param obj_or_id: A STIX object or object ID as a string.
-    :return: True if a marking definition, False otherwise.
-    """
-
-    if isinstance(obj_or_id, (_STIXBase, dict)):
-        id_ = obj_or_id["id"]
-    else:
-        id_ = obj_or_id
-
-    return id_.startswith("marking-definition--")
-
-
 class _ObjectFamily(object):
     """
     An internal implementation detail of memory sources/sinks/stores.
@@ -276,7 +261,7 @@ class MemorySource(DataSource):
         """
         stix_obj = None
 
-        if _is_marking(stix_id):
+        if is_marking(stix_id):
             stix_obj = self._data.get(stix_id)
         else:
             object_family = self._data.get(stix_id)
@@ -313,7 +298,7 @@ class MemorySource(DataSource):
         """
         results = []
         stix_objs_to_filter = None
-        if _is_marking(stix_id):
+        if is_marking(stix_id):
             stix_obj = self._data.get(stix_id)
             if stix_obj:
                 stix_objs_to_filter = [stix_obj]

stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20181101232448446000.json

@@ -0,0 +1,11 @@
+{
+    "type": "identity",
+    "id": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
+    "created": "2017-06-01T00:00:00.000Z",
+    "modified": "2018-11-01T23:24:48.446Z",
+    "name": "The MITRE Corporation",
+    "identity_class": "organization",
+    "labels": [
+        "version two"
+    ]
+}

stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json

@@ -0,0 +1,34 @@
+{
+    "id": "bundle--f64de948-7067-4534-8018-85f03d470625",
+    "objects": [
+        {
+            "created": "2017-05-31T21:32:58.226477Z",
+            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
+            "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]",
+            "external_references": [
+                {
+                    "external_id": "S0090",
+                    "source_name": "mitre-attack",
+                    "url": "https://attack.mitre.org/wiki/Software/S0090"
+                },
+                {
+                    "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.",
+                    "source_name": "Palo Alto Rover",
+                    "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/"
+                }
+            ],
+            "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
+            "labels": [
+                "malware"
+            ],
+            "modified": "2017-05-31T21:32:58.226477Z",
+            "name": "Rover",
+            "object_marking_refs": [
+                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
+            ],
+            "type": "malware"
+        }
+    ],
+    "spec_version": "2.0",
+    "type": "bundle"
+}

stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json

@@ -0,0 +1,27 @@
+{
+    "type": "malware",
+    "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
+    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
+    "created": "2017-05-31T21:32:58.226Z",
+    "modified": "2018-11-01T23:24:48.456Z",
+    "name": "Rover",
+    "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]",
+    "labels": [
+        "version two"
+    ],
+    "external_references": [
+        {
+            "source_name": "mitre-attack",
+            "url": "https://attack.mitre.org/wiki/Software/S0090",
+            "external_id": "S0090"
+        },
+        {
+            "source_name": "Palo Alto Rover",
+            "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.",
+            "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/"
+        }
+    ],
+    "object_marking_refs": [
+        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
+    ]
+}

stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json

@@ -0,0 +1,27 @@
+{
+    "type": "malware",
+    "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
+    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
+    "created": "2017-05-31T21:32:58.226Z",
+    "modified": "2018-11-01T23:24:48.457Z",
+    "name": "Rover",
+    "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]",
+    "labels": [
+        "version three"
+    ],
+    "external_references": [
+        {
+            "source_name": "mitre-attack",
+            "url": "https://attack.mitre.org/wiki/Software/S0090",
+            "external_id": "S0090"
+        },
+        {
+            "source_name": "Palo Alto Rover",
+            "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.",
+            "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/"
+        }
+    ],
+    "object_marking_refs": [
+        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
+    ]
+}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json

@@ -1,42 +0,0 @@
-{
-    "id": "bundle--f68640b4-0cdc-42ae-b176-def1754a1ea0",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:19.73501Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Credential dumping is the process of obtaining account login and password information from the operating system and software. Credentials can be used to perform Windows Credential Editor, Mimikatz, and gsecdump. These tools are in use by both professional security testers and adversaries.\n\nPlaintext passwords can be obtained using tools such as Mimikatz to extract passwords stored by the Local Security Authority (LSA). If smart cards are used to authenticate to a domain using a personal identification number (PIN), then that PIN is also cached as a result and may be dumped.Mimikatz access the LSA Subsystem Service (LSASS) process by opening the process, locating the LSA secrets key, and decrypting the sections in memory where credential details are stored. Credential dumpers may also use methods for reflective DLL Injection to reduce potential indicators of malicious activity.\n\nNTLM hash dumpers open the Security Accounts Manager (SAM) on the local file system (%SystemRoot%/system32/config/SAM) or create a dump of the Registry SAM key to access stored account password hashes. Some hash dumpers will open the local file system as a device and parse to the SAM table to avoid file access defenses. Others will make an in-memory copy of the SAM table before reading hashes. Detection of compromised Legitimate Credentials in-use by adversaries may help as well. \n\nOn Windows 8.1 and Windows Server 2012 R2, monitor Windows Logs for LSASS.exe creation to verify that LSASS started as a protected process.\n\nMonitor processes and command-line arguments for program execution that may be indicative of credential dumping. Remote access tools may contain built-in features or incorporate existing tools like Mimikatz. PowerShell scripts also exist that contain credential dumping functionality, such as PowerSploit's Invoke-Mimikatz module,[[Citation: Powersploit]] which may require additional logging features to be configured in the operating system to collect necessary information for analysis.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: API monitoring, Process command-line parameters, Process monitoring, PowerShell logs",
-            "external_references": [
-                {
-                    "external_id": "T1003",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1003"
-                },
-                {
-                    "description": "Delpy, B. (2014, September 14). Mimikatz module ~ sekurlsa. Retrieved January 10, 2016.",
-                    "source_name": "Github Mimikatz Module sekurlsa",
-                    "url": "https://github.com/gentilkiwi/mimikatz/wiki/module-~-sekurlsa"
-                },
-                {
-                    "description": "PowerSploit. (n.d.).  Retrieved December 4, 2014.",
-                    "source_name": "Powersploit",
-                    "url": "https://github.com/mattifestation/PowerSploit"
-                }
-            ],
-            "id": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "credential-access"
-                }
-            ],
-            "modified": "2017-05-31T21:30:19.73501Z",
-            "name": "Credential Dumping",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json

@@ -1,37 +0,0 @@
-{
-    "id": "bundle--b07d6fd6-7cc5-492d-a1eb-9ba956b329d5",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:26.496201Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Rootkits are programs that hide the existence of malware by intercepting and modifying operating system API calls that supply system information. Rootkits or rootkit enabling functionality may reside at the user or kernel level in the operating system or lower, to include a Hypervisor, Master Boot Record, or the Basic Input/Output System.[[Citation: Wikipedia Rootkit]]\n\nAdversaries may use rootkits to hide the presence of programs, files, network connections, services, drivers, and other system components.\n\nDetection: Some rootkit protections may be built into anti-virus or operating system software. There are dedicated rootkit detection tools that look for specific types of rootkit behavior. Monitor for the existence of unrecognized DLLs, devices, services, and changes to the MBR.[[Citation: Wikipedia Rootkit]]\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: BIOS, MBR, System calls",
-            "external_references": [
-                {
-                    "external_id": "T1014",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1014"
-                },
-                {
-                    "description": "Wikipedia. (2016, June 1). Rootkit. Retrieved June 2, 2016.",
-                    "source_name": "Wikipedia Rootkit",
-                    "url": "https://en.wikipedia.org/wiki/Rootkit"
-                }
-            ],
-            "id": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "defense-evasion"
-                }
-            ],
-            "modified": "2017-05-31T21:30:26.496201Z",
-            "name": "Rootkit",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json

@@ -1,32 +0,0 @@
-{
-    "id": "bundle--1a854c96-639e-4771-befb-e7b960a65974",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:29.45894Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Data, such as sensitive documents, may be exfiltrated through the use of automated processing or Scripting after being gathered during Exfiltration Over Command and Control Channel and Exfiltration Over Alternative Protocol.\n\nDetection: Monitor process file access patterns and network behavior. Unrecognized processes or scripts that appear to be traversing file systems and sending network traffic may be suspicious.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process use of network",
-            "external_references": [
-                {
-                    "external_id": "T1020",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1020"
-                }
-            ],
-            "id": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "exfiltration"
-                }
-            ],
-            "modified": "2017-05-31T21:30:29.45894Z",
-            "name": "Automated Exfiltration",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json

@@ -1,32 +0,0 @@
-{
-    "id": "bundle--33e3e33a-38b8-4a37-9455-5b8c82d3b10a",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:45.139269Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Adversaries may attempt to get a listing of network connections to or from the compromised system.\nUtilities and commands that acquire this information include netstat, \"net use,\" and \"net session\" with Net.\n\nDetection: System and network discovery techniques normally occur throughout an operation as an adversary learns the environment. Data and events should not be viewed in isolation, but as part of a chain of behavior that could lead to other activities, such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Process command-line parameters, Process monitoring",
-            "external_references": [
-                {
-                    "external_id": "T1049",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1049"
-                }
-            ],
-            "id": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "discovery"
-                }
-            ],
-            "modified": "2017-05-31T21:30:45.139269Z",
-            "name": "Local Network Connections Discovery",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json

@@ -1,32 +0,0 @@
-{
-    "id": "bundle--a87938c5-cc1e-4e06-a8a3-b10243ae397d",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:41.022897Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Sensitive data can be collected from remote systems via shared network drives (host shared directory, network file server, etc.) that are accessible from the current system prior to cmd may be used to gather information.\n\nDetection: Monitor processes and command-line arguments for actions that could be taken to collect files from a network share. Remote access tools with built-in features may interact directly with the Windows API to gather data. Data may also be acquired through Windows system management tools such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process command-line parameters",
-            "external_references": [
-                {
-                    "external_id": "T1039",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1039"
-                }
-            ],
-            "id": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "collection"
-                }
-            ],
-            "modified": "2017-05-31T21:30:41.022897Z",
-            "name": "Data from Network Shared Drive",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json

@@ -1,32 +0,0 @@
-{
-    "id": "bundle--5ddaeff9-eca7-4094-9e65-4f53da21a444",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:32.662702Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Adversaries may attempt to make an executable or file difficult to discover or analyze by encrypting, encoding, or otherwise obfuscating its contents on the system.\n\nDetection: Detection of file obfuscation is difficult unless artifacts are left behind by the obfuscation process that are uniquely detectable with a signature. If detection of the obfuscation itself is not possible, it may be possible to detect the malicious activity that caused the obfuscated file (for example, the method that was used to write, read, or modify the file on the file system).\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Network protocol analysis, Process use of network, Binary file metadata, File monitoring, Malware reverse engineering",
-            "external_references": [
-                {
-                    "external_id": "T1027",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Technique/T1027"
-                }
-            ],
-            "id": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a",
-            "kill_chain_phases": [
-                {
-                    "kill_chain_name": "mitre-attack",
-                    "phase_name": "defense-evasion"
-                }
-            ],
-            "modified": "2017-05-31T21:30:32.662702Z",
-            "name": "Obfuscated Files or Information",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "attack-pattern"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json

@@ -1,16 +0,0 @@
-{
-    "id": "bundle--a42d26fe-c938-4074-a1b3-50d852e6f0bd",
-    "objects": [
-        {
-            "created": "2017-05-31T21:30:26.495974Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Identify potentially malicious software that may contain rootkit functionality, and audit and/or block it by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]",
-            "id": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f",
-            "modified": "2017-05-31T21:30:26.495974Z",
-            "name": "Rootkit Mitigation",
-            "type": "course-of-action"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json

@@ -1,9 +0,0 @@
-{
-    "created": "2017-05-31T21:30:41.022744Z",
-    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-    "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]",
-    "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd",
-    "modified": "2017-05-31T21:30:41.022744Z",
-    "name": "Data from Network Shared Drive Mitigation",
-    "type": "course-of-action"
-}

stix2/test/v20/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json

@@ -1,15 +0,0 @@
-{
-    "id": "bundle--81884287-2548-47fc-a997-39489ddd5462",
-    "objects": [
-        {
-            "created": "2017-06-01T00:00:00Z",
-            "id": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "identity_class": "organization",
-            "modified": "2017-06-01T00:00:00Z",
-            "name": "The MITRE Corporation",
-            "type": "identity"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json

@@ -1,54 +0,0 @@
-{
-    "id": "bundle--7790ee4c-2d57-419a-bc9d-8805b5bb4118",
-    "objects": [
-        {
-            "aliases": [
-                "Deep Panda",
-                "Shell Crew",
-                "WebMasters",
-                "KungFu Kittens",
-                "PinkPanther",
-                "Black Vine"
-            ],
-            "created": "2017-05-31T21:31:49.412497Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Deep Panda is a suspected Chinese threat group known to target many industries, including government, defense, financial, and telecommunications.Deep Panda.Deep Panda also appears to be known as Black Vine based on the attribution of both group names to the Anthem intrusion.[[Citation: Symantec Black Vine]]",
-            "external_references": [
-                {
-                    "external_id": "G0009",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Group/G0009"
-                },
-                {
-                    "description": "Alperovitch, D. (2014, July 7). Deep in Thought: Chinese Targeting of National Security Think Tanks. Retrieved November 12, 2014.",
-                    "source_name": "Alperovitch 2014",
-                    "url": "http://blog.crowdstrike.com/deep-thought-chinese-targeting-national-security-think-tanks/"
-                },
-                {
-                    "description": "DiMaggio, J.. (2015, August 6). The Black Vine cyberespionage group. Retrieved January 26, 2016.",
-                    "source_name": "Symantec Black Vine",
-                    "url": "http://www.symantec.com/content/en/us/enterprise/media/security%20response/whitepapers/the-black-vine-cyberespionage-group.pdf"
-                },
-                {
-                    "description": "RSA Incident Response. (2014, January). RSA Incident Response Emerging Threat Profile: Shell Crew. Retrieved January 14, 2016.",
-                    "source_name": "RSA Shell Crew",
-                    "url": "https://www.emc.com/collateral/white-papers/h12756-wp-shell-crew.pdf"
-                },
-                {
-                    "description": "ThreatConnect Research Team. (2015, February 27). The Anthem Hack: All Roads Lead to China. Retrieved January 26, 2016.",
-                    "source_name": "ThreatConnect Anthem",
-                    "url": "https://www.threatconnect.com/the-anthem-hack-all-roads-lead-to-china/"
-                }
-            ],
-            "id": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064",
-            "modified": "2017-05-31T21:31:49.412497Z",
-            "name": "Deep Panda",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "intrusion-set"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json

@@ -1,44 +0,0 @@
-{
-    "id": "bundle--96a6ea7a-fcff-4aab-925b-a494bcdf0480",
-    "objects": [
-        {
-            "aliases": [
-                "DragonOK"
-            ],
-            "created": "2017-05-31T21:31:53.197755Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]",
-            "external_references": [
-                {
-                    "external_id": "G0017",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Group/G0017"
-                },
-                {
-                    "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.",
-                    "source_name": "Operation Quantum Entanglement",
-                    "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf"
-                },
-                {
-                    "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.",
-                    "source_name": "Symbiotic APT Groups",
-                    "url": "https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&D%20Track%20Insight%20into%20Symbiotic%20APT.pdf"
-                },
-                {
-                    "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.",
-                    "source_name": "New DragonOK",
-                    "url": "http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/"
-                }
-            ],
-            "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",
-            "modified": "2017-05-31T21:31:53.197755Z",
-            "name": "DragonOK",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "intrusion-set"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json

@@ -1,35 +1,27 @@
 {
-    "id": "bundle--f64de948-7067-4534-8018-85f03d470625",
-    "objects": [
+    "type": "malware",
+    "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
+    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
+    "created": "2017-05-31T21:32:58.226Z",
+    "modified": "2018-11-16T22:54:20.390Z",
+    "name": "Rover",
+    "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]",
+    "labels": [
+        "version four"
+    ],
+    "external_references": [
         {
-            "created": "2017-05-31T21:32:58.226477Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]",
-            "external_references": [
-                {
-                    "external_id": "S0090",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0090"
-                },
-                {
-                    "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.",
-                    "source_name": "Palo Alto Rover",
-                    "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/"
-                }
-            ],
-            "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
-            "labels": [
-                "malware"
-            ],
-            "modified": "2017-05-31T21:32:58.226477Z",
-            "name": "Rover",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "malware",
-            "is_family": false
+            "source_name": "mitre-attack",
+            "url": "https://attack.mitre.org/wiki/Software/S0090",
+            "external_id": "S0090"
+        },
+        {
+            "source_name": "Palo Alto Rover",
+            "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.",
+            "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/"
         }
     ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}
+    "object_marking_refs": [
+        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
+    ]
+}

stix2/test/v20/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json

@@ -1,35 +0,0 @@
-{
-    "id": "bundle--c633942b-545c-4c87-91b7-9fe5740365e0",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:26.565056Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "RTM is custom malware written in Delphi. It is used by the group of the same name (RTM).[[Citation: ESET RTM Feb 2017]]",
-            "external_references": [
-                {
-                    "external_id": "S0148",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0148"
-                },
-                {
-                    "description": "Faou, M. and Boutin, J.. (2017, February). Read The Manual: A Guide to the RTM Banking Trojan. Retrieved March 9, 2017.",
-                    "source_name": "ESET RTM Feb 2017",
-                    "url": "https://www.welivesecurity.com/wp-content/uploads/2017/02/Read-The-Manual.pdf"
-                }
-            ],
-            "id": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841",
-            "labels": [
-                "malware"
-            ],
-            "modified": "2017-05-31T21:33:26.565056Z",
-            "name": "RTM",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "malware",
-            "is_family": false
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json

@@ -1,35 +0,0 @@
-{
-    "id": "bundle--09ce4338-8741-4fcf-9738-d216c8e40974",
-    "objects": [
-        {
-            "created": "2017-05-31T21:32:48.482655Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Sakula is a remote access tool (RAT) that first surfaced in 2012 and was used in intrusions throughout 2015.[[Citation: Dell Sakula]]\n\nAliases: Sakula, Sakurel, VIPER",
-            "external_references": [
-                {
-                    "external_id": "S0074",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0074"
-                },
-                {
-                    "description": "Dell SecureWorks Counter Threat Unit Threat Intelligence. (2015, July 30). Sakula Malware Family. Retrieved January 26, 2016.",
-                    "source_name": "Dell Sakula",
-                    "url": "http://www.secureworks.com/cyber-threat-intelligence/threats/sakula-malware-family/"
-                }
-            ],
-            "id": "malware--96b08451-b27a-4ff6-893f-790e26393a8e",
-            "labels": [
-                "malware"
-            ],
-            "modified": "2017-05-31T21:32:48.482655Z",
-            "name": "Sakula",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "malware",
-            "is_family": false
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json

@@ -1,35 +0,0 @@
-{
-    "id": "bundle--611947ce-ae3b-4fdb-b297-aed8eab22e4f",
-    "objects": [
-        {
-            "created": "2017-05-31T21:32:15.263882Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "PoisonIvy is a popular remote access tool (RAT) that has been used by many groups.[[Citation: FireEye Poison Ivy]]\n\nAliases: PoisonIvy, Poison Ivy",
-            "external_references": [
-                {
-                    "external_id": "S0012",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0012"
-                },
-                {
-                    "description": "FireEye. (2014). POISON IVY: Assessing Damage and Extracting Intelligence. Retrieved November 12, 2014.",
-                    "source_name": "FireEye Poison Ivy",
-                    "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-poison-ivy.pdf"
-                }
-            ],
-            "id": "malware--b42378e0-f147-496f-992a-26a49705395b",
-            "labels": [
-                "malware"
-            ],
-            "modified": "2017-05-31T21:32:15.263882Z",
-            "name": "PoisonIvy",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "malware",
-            "is_family": false
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json

@@ -13,4 +13,4 @@
     ],
     "spec_version": "2.0",
     "type": "bundle"
-}
+}

stix2/test/v20/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--7e715462-dd9d-40b9-968a-10ef0ecf126d",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.182784Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--0d4a7788-7f3b-4df8-a498-31a38003c883",
-            "modified": "2017-05-31T21:33:27.182784Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a",
-            "target_ref": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--a53eef35-abfc-4bcd-b84e-a048f7b4a9bf",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.082801Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227",
-            "modified": "2017-05-31T21:33:27.082801Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22",
-            "target_ref": "tool--242f3da3-4425-4d11-8f5c-b842886da966",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--0b9f6412-314f-44e3-8779-9738c9578ef5",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.018782Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--1e91cd45-a725-4965-abe3-700694374432",
-            "modified": "2017-05-31T21:33:27.018782Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "mitigates",
-            "source_ref": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f",
-            "target_ref": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--6d5b04a8-efb2-4179-990e-74f1dcc76e0c",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.100701Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e",
-            "modified": "2017-05-31T21:33:27.100701Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475",
-            "target_ref": "tool--03342581-f790-4f03-ba41-e82e67392e23",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--a7efc025-040d-49c7-bf97-e5a1120ecacc",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.143973Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1",
-            "modified": "2017-05-31T21:33:27.143973Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9",
-            "target_ref": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--9f013d47-7704-41c2-9749-23d0d94af94d",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.021562Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--592d0c31-e61f-495e-a60e-70d7be59a719",
-            "modified": "2017-05-31T21:33:27.021562Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "mitigates",
-            "source_ref": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd",
-            "target_ref": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--15167b24-4cee-4c96-a140-32a6c37df4b4",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.044387Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1",
-            "modified": "2017-05-31T21:33:27.044387Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064",
-            "target_ref": "malware--96b08451-b27a-4ff6-893f-790e26393a8e",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json

@@ -1,20 +0,0 @@
-{
-    "id": "bundle--ff845dca-7036-416f-aae0-95030994c49f",
-    "objects": [
-        {
-            "created": "2017-05-31T21:33:27.051532Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "id": "relationship--8797579b-e3be-4209-a71b-255a4d08243d",
-            "modified": "2017-05-31T21:33:27.051532Z",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "relationship_type": "uses",
-            "source_ref": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",
-            "target_ref": "malware--b42378e0-f147-496f-992a-26a49705395b",
-            "type": "relationship"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json

@@ -1,39 +0,0 @@
-{
-    "id": "bundle--d8826afc-1561-4362-a4e3-05a4c2c3ac3c",
-    "objects": [
-        {
-            "created": "2017-05-31T21:32:31.601148Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "The Net utility is a component of the Windows operating system. It is used in command-line operations for control of users, groups, services, and network connections.Net has a great deal of functionality,[[Citation: Savill 1999]] much of which is useful for an adversary, such as gathering system and network information for [[Discovery]], moving laterally through [[Windows admin shares]] using <code>net use</code> commands, and interacting with services.\n\nAliases: Net, net.exe",
-            "external_references": [
-                {
-                    "external_id": "S0039",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0039"
-                },
-                {
-                    "description": "Microsoft. (2006, October 18). Net.exe Utility. Retrieved September 22, 2015.",
-                    "source_name": "Microsoft Net Utility",
-                    "url": "https://msdn.microsoft.com/en-us/library/aa939914"
-                },
-                {
-                    "description": "Savill, J. (1999, March 4). Net.exe reference. Retrieved September 22, 2015.",
-                    "source_name": "Savill 1999",
-                    "url": "http://windowsitpro.com/windows/netexe-reference"
-                }
-            ],
-            "id": "tool--03342581-f790-4f03-ba41-e82e67392e23",
-            "labels": [
-                "tool"
-            ],
-            "modified": "2017-05-31T21:32:31.601148Z",
-            "name": "Net",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "tool"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json

@@ -1,34 +0,0 @@
-{
-    "id": "bundle--7dbde18f-6f14-4bf0-8389-505c89d6d5a6",
-    "objects": [
-        {
-            "created": "2017-05-31T21:32:12.684914Z",
-            "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
-            "description": "Windows Credential Editor is a password dumping tool.[[Citation: Amplia WCE]]\n\nAliases: Windows Credential Editor, WCE",
-            "external_references": [
-                {
-                    "external_id": "S0005",
-                    "source_name": "mitre-attack",
-                    "url": "https://attack.mitre.org/wiki/Software/S0005"
-                },
-                {
-                    "description": "Amplia Security. (n.d.). Windows Credentials Editor (WCE) F.A.Q.. Retrieved December 17, 2015.",
-                    "source_name": "Amplia WCE",
-                    "url": "http://www.ampliasecurity.com/research/wcefaq.html"
-                }
-            ],
-            "id": "tool--242f3da3-4425-4d11-8f5c-b842886da966",
-            "labels": [
-                "tool"
-            ],
-            "modified": "2017-05-31T21:32:12.684914Z",
-            "name": "Windows Credential Editor",
-            "object_marking_refs": [
-                "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
-            ],
-            "type": "tool"
-        }
-    ],
-    "spec_version": "2.0",
-    "type": "bundle"
-}

stix2/test/v20/test_datastore_filesystem.py

@@ -1,10 +1,18 @@
+import datetime
+import errno
 import json
 import os
 import shutil
+import stat
 
 import pytest
+import pytz
 
 import stix2
+from stix2.datastore.filesystem import (AuthSet, _find_search_optimizations,
+                                        _get_matching_dir_entries,
+                                        _timestamp2filename)
+from stix2.exceptions import STIXError
 
 from .constants import (
     CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID,
@@ -96,7 +104,20 @@ def rel_fs_store():
     yield fs
 
     for o in stix_objs:
-        os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
+        filepath = os.path.join(FS_PATH, o.type, o.id,
+                                _timestamp2filename(o.modified) + '.json')
+
+        # Some test-scoped fixtures (e.g. fs_store) delete all campaigns, so by
+        # the time this module-scoped fixture tears itself down, it may find
+        # its campaigns already gone, which causes not-found errors.
+        try:
+            os.remove(filepath)
+        except OSError as e:
+            # 3 is the ERROR_PATH_NOT_FOUND windows error code.  Which has an
+            # errno symbolic value, but not the windows meaning...
+            if e.errno in (errno.ENOENT, 3):
+                continue
+            raise
 
 
 def test_filesystem_source_nonexistent_folder():
@@ -126,32 +147,36 @@ def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files):
     # this tests handling of bad STIX json object
     try:
         fs_source.get("intrusion-set--test-non-stix")
-    except TypeError as e:
-        assert "intrusion-set--test-non-stix" in str(e)
-        assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e)
+    except STIXError as e:
+        assert "Can't parse object with no 'type' property" in str(e)
 
 
-def test_filesytem_source_get_object(fs_source):
-    # get object
+def test_filesystem_source_get_object(fs_source):
+    # get (latest) object
     mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38")
     assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"
     assert mal.name == "Rover"
+    assert mal.modified == datetime.datetime(2018, 11, 16, 22, 54, 20, 390000,
+                                             pytz.utc)
 
 
-def test_filesytem_source_get_nonexistent_object(fs_source):
+def test_filesystem_source_get_nonexistent_object(fs_source):
     ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38")
     assert ind is None
 
 
-def test_filesytem_source_all_versions(fs_source):
-    # all versions - (currently not a true all versions call as FileSystem cant have multiple versions)
-    id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5")
-    assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5"
-    assert id_.name == "The MITRE Corporation"
-    assert id_.type == "identity"
+def test_filesystem_source_all_versions(fs_source):
+    ids = fs_source.all_versions(
+        "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5"
+    )
+    assert len(ids) == 2
+    assert all(id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5"
+               for id_ in ids)
+    assert all(id_.name == "The MITRE Corporation" for id_ in ids)
+    assert all(id_.type == "identity" for id_ in ids)
 
 
-def test_filesytem_source_query_single(fs_source):
+def test_filesystem_source_query_single(fs_source):
     # query2
     is_2 = fs_source.query([stix2.Filter("external_references.external_id", '=', "T1027")])
     assert len(is_2) == 1
@@ -161,7 +186,7 @@ def test_filesytem_source_query_single(fs_source):
     assert is_2.type == "attack-pattern"
 
 
-def test_filesytem_source_query_multiple(fs_source):
+def test_filesystem_source_query_multiple(fs_source):
     # query
     intrusion_sets = fs_source.query([stix2.Filter("type", '=', "intrusion-set")])
     assert len(intrusion_sets) == 2
@@ -173,6 +198,24 @@ def test_filesytem_source_query_multiple(fs_source):
     assert len(is_1.external_references) == 4
 
 
+def test_filesystem_source_backward_compatible(fs_source):
+    # this specific object is outside an "ID" directory; make sure we can get
+    # it.
+    modified = datetime.datetime(2018, 11, 16, 22, 54, 20, 390000, pytz.utc)
+    results = fs_source.query([
+        stix2.Filter("type", "=", "malware"),
+        stix2.Filter("id", "=", "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"),
+        stix2.Filter("modified", "=", modified)
+    ])
+
+    assert len(results) == 1
+    result = results[0]
+    assert result.type == "malware"
+    assert result.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"
+    assert result.modified == modified
+    assert result.labels == ["version four"]
+
+
 def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source):
     # add python stix object
     camp1 = stix2.v20.Campaign(
@@ -183,14 +226,16 @@ def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source):
 
     fs_sink.add(camp1)
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json"))
+    filepath = os.path.join(FS_PATH, "campaign", camp1.id,
+                            _timestamp2filename(camp1.modified) + ".json")
+    assert os.path.exists(filepath)
 
     camp1_r = fs_source.get(camp1.id)
     assert camp1_r.id == camp1.id
     assert camp1_r.name == "Hannibal"
     assert "War Elephant" in camp1_r.aliases
 
-    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source):
@@ -202,18 +247,29 @@ def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source):
         "aliases": ["Purple Robes"],
         "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
         "created": "2017-05-31T21:31:53.197755Z",
+        "modified": "2017-05-31T21:31:53.197755Z"
     }
 
     fs_sink.add(camp2)
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json"))
+    # Need to get the exact "modified" timestamp which would have been
+    # in effect at the time the object was saved to the sink, which determines
+    # the filename it would have been saved as.  It may not be exactly the same
+    # as what's in the dict, since the parsing process can enforce a precision
+    # constraint (e.g. truncate to milliseconds), which results in a slightly
+    # different name.
+    camp2obj = stix2.parse(camp2)
+    filepath = os.path.join(FS_PATH, "campaign", camp2obj["id"],
+                            _timestamp2filename(camp2obj["modified"]) + ".json")
+
+    assert os.path.exists(filepath)
 
     camp2_r = fs_source.get(camp2["id"])
     assert camp2_r.id == camp2["id"]
     assert camp2_r.name == camp2["name"]
     assert "Purple Robes" in camp2_r.aliases
 
-    os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source):
@@ -230,52 +286,73 @@ def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source):
                 "aliases": ["Huns"],
                 "id": "campaign--b8f86161-ccae-49de-973a-4ca320c62478",
                 "created": "2017-05-31T21:31:53.197755Z",
-            },
-        ],
+                "modified": "2017-05-31T21:31:53.197755Z"
+            }
+        ]
     }
 
     fs_sink.add(bund)
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json"))
+    camp_obj = stix2.parse(bund["objects"][0])
+    filepath = os.path.join(FS_PATH, "campaign", camp_obj["id"],
+                            _timestamp2filename(camp_obj["modified"]) + ".json")
+
+    assert os.path.exists(filepath)
 
     camp3_r = fs_source.get(bund["objects"][0]["id"])
     assert camp3_r.id == bund["objects"][0]["id"]
     assert camp3_r.name == bund["objects"][0]["name"]
     assert "Huns" in camp3_r.aliases
 
-    os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source):
     # add json-encoded stix obj
     camp4 = '{"type": "campaign", "id":"campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d",'\
-            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}'
+            ' "created":"2017-05-31T21:31:53.197755Z",'\
+            ' "modified":"2017-05-31T21:31:53.197755Z",'\
+            ' "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}'
 
     fs_sink.add(camp4)
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d" + ".json"))
+    camp4obj = stix2.parse(camp4)
+    filepath = os.path.join(FS_PATH, "campaign",
+                            "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d",
+                            _timestamp2filename(camp4obj["modified"]) + ".json")
+
+    assert os.path.exists(filepath)
 
     camp4_r = fs_source.get("campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d")
     assert camp4_r.id == "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d"
     assert camp4_r.name == "Ghengis Khan"
 
-    os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source):
     # add json-encoded stix bundle
     bund2 = '{"type": "bundle", "id": "bundle--3d267103-8475-4d8f-b321-35ec6eccfa37",' \
             ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",' \
-            ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}'
+            ' "created":"2017-05-31T21:31:53.197755Z",'\
+            ' "modified":"2017-05-31T21:31:53.197755Z",'\
+            ' "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}'
     fs_sink.add(bund2)
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b" + ".json"))
+    bund2obj = stix2.parse(bund2)
+    camp_obj = bund2obj["objects"][0]
+
+    filepath = os.path.join(FS_PATH, "campaign",
+                            "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",
+                            _timestamp2filename(camp_obj["modified"]) + ".json")
+
+    assert os.path.exists(filepath)
 
     camp5_r = fs_source.get("campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b")
     assert camp5_r.id == "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b"
     assert camp5_r.name == "Spartacus"
 
-    os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_sink_add_objects_list(fs_sink, fs_source):
@@ -293,12 +370,22 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source):
         "aliases": ["The Frenchmen"],
         "id": "campaign--122818b6-1112-4fb0-b11b-b111107ca70a",
         "created": "2017-05-31T21:31:53.197755Z",
+        "modified": "2017-05-31T21:31:53.197755Z"
     }
 
     fs_sink.add([camp6, camp7])
 
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json"))
-    assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a" + ".json"))
+    camp7obj = stix2.parse(camp7)
+
+    camp6filepath = os.path.join(FS_PATH, "campaign", camp6.id,
+                                 _timestamp2filename(camp6["modified"]) +
+                                 ".json")
+    camp7filepath = os.path.join(
+        FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a",
+        _timestamp2filename(camp7obj["modified"]) + ".json")
+
+    assert os.path.exists(camp6filepath)
+    assert os.path.exists(camp7filepath)
 
     camp6_r = fs_source.get(camp6.id)
     assert camp6_r.id == camp6.id
@@ -309,8 +396,24 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source):
     assert "The Frenchmen" in camp7_r.aliases
 
     # remove all added objects
-    os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json"))
-    os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json"))
+    os.remove(camp6filepath)
+    os.remove(camp7filepath)
+
+
+def test_filesystem_sink_marking(fs_sink):
+    marking = stix2.v20.MarkingDefinition(
+        definition_type="tlp",
+        definition=stix2.v20.TLPMarking(tlp="green")
+    )
+
+    fs_sink.add(marking)
+    marking_filepath = os.path.join(
+        FS_PATH, "marking-definition", marking["id"] + ".json"
+    )
+
+    assert os.path.exists(marking_filepath)
+
+    os.remove(marking_filepath)
 
 
 def test_filesystem_store_get_stored_as_bundle(fs_store):
@@ -326,8 +429,9 @@ def test_filesystem_store_get_stored_as_object(fs_store):
 
 
 def test_filesystem_store_all_versions(fs_store):
-    # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored)
-    rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0]
+    rels = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")
+    assert len(rels) == 1
+    rel = rels[0]
     assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1"
     assert rel.type == "relationship"
 
@@ -350,9 +454,9 @@ def test_filesystem_store_query_single_filter(fs_store):
 
 def test_filesystem_store_empty_query(fs_store):
     results = fs_store.query()  # returns all
-    assert len(results) == 26
-    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj["id"] for obj in results]
-    assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj["id"] for obj in results]
+    assert len(results) == 30
+    assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results]
+    assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results]
 
 
 def test_filesystem_store_query_multiple_filters(fs_store):
@@ -364,7 +468,7 @@ def test_filesystem_store_query_multiple_filters(fs_store):
 
 def test_filesystem_store_query_dont_include_type_folder(fs_store):
     results = fs_store.query(stix2.Filter("type", "!=", "tool"))
-    assert len(results) == 24
+    assert len(results) == 28
 
 
 def test_filesystem_store_add(fs_store):
@@ -380,8 +484,11 @@ def test_filesystem_store_add(fs_store):
     assert camp1_r.id == camp1.id
     assert camp1_r.name == camp1.name
 
+    filepath = os.path.join(FS_PATH, "campaign", camp1_r.id,
+                            _timestamp2filename(camp1_r.modified) + ".json")
+
     # remove
-    os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json"))
+    os.remove(filepath)
 
 
 def test_filesystem_store_add_as_bundle():
@@ -394,7 +501,10 @@ def test_filesystem_store_add_as_bundle():
     )
     fs_store.add(camp1)
 
-    with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file:
+    filepath = os.path.join(FS_PATH, "campaign", camp1.id,
+                            _timestamp2filename(camp1.modified) + ".json")
+
+    with open(filepath) as bundle_file:
         assert '"type": "bundle"' in bundle_file.read()
 
     camp1_r = fs_store.get(camp1.id)
@@ -419,6 +529,26 @@ def test_filesystem_store_add_invalid_object(fs_store):
     assert 'JSON formatted STIX bundle' in str(excinfo.value)
 
 
+def test_filesystem_store_add_marking(fs_store):
+    marking = stix2.v20.MarkingDefinition(
+        definition_type="tlp",
+        definition=stix2.v20.TLPMarking(tlp="green")
+    )
+
+    fs_store.add(marking)
+    marking_filepath = os.path.join(
+        FS_PATH, "marking-definition", marking["id"] + ".json"
+    )
+
+    assert os.path.exists(marking_filepath)
+
+    marking_r = fs_store.get(marking["id"])
+    assert marking_r["id"] == marking["id"]
+    assert marking_r["definition"]["tlp"] == "green"
+
+    os.remove(marking_filepath)
+
+
 def test_filesystem_object_with_custom_property(fs_store):
     camp = stix2.v20.Campaign(
         name="Scipio Africanus",
@@ -540,3 +670,357 @@ def test_related_to_by_target(rel_fs_store):
     assert len(resp) == 2
     assert any(x['id'] == CAMPAIGN_ID for x in resp)
     assert any(x['id'] == INDICATOR_ID for x in resp)
+
+
+def test_auth_set_white1():
+    auth_set = AuthSet({"A"}, set())
+
+    assert auth_set.auth_type == AuthSet.WHITE
+    assert auth_set.values == {"A"}
+
+
+def test_auth_set_white2():
+    auth_set = AuthSet(set(), set())
+
+    assert auth_set.auth_type == AuthSet.WHITE
+    assert len(auth_set.values) == 0
+
+
+def test_auth_set_white3():
+    auth_set = AuthSet({"A", "B"}, {"B", "C"})
+
+    assert auth_set.auth_type == AuthSet.WHITE
+    assert auth_set.values == {"A"}
+
+
+def test_auth_set_black1():
+    auth_set = AuthSet(None, {"B", "C"})
+
+    assert auth_set.auth_type == AuthSet.BLACK
+    assert auth_set.values == {"B", "C"}
+
+
+def test_optimize_types1():
+    filters = [
+        stix2.Filter("type", "=", "foo")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"foo"}
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types2():
+    filters = [
+        stix2.Filter("type", "=", "foo"),
+        stix2.Filter("type", "=", "bar")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert len(auth_types.values) == 0
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types3():
+    filters = [
+        stix2.Filter("type", "in", ["A", "B", "C"]),
+        stix2.Filter("type", "in", ["B", "C", "D"])
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"B", "C"}
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types4():
+    filters = [
+        stix2.Filter("type", "in", ["A", "B", "C"]),
+        stix2.Filter("type", "in", ["D", "E", "F"])
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert len(auth_types.values) == 0
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types5():
+    filters = [
+        stix2.Filter("type", "in", ["foo", "bar"]),
+        stix2.Filter("type", "!=", "bar")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"foo"}
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types6():
+    filters = [
+        stix2.Filter("type", "!=", "foo"),
+        stix2.Filter("type", "!=", "bar")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.BLACK
+    assert auth_types.values == {"foo", "bar"}
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types7():
+    filters = [
+        stix2.Filter("type", "=", "foo"),
+        stix2.Filter("type", "!=", "foo")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert len(auth_types.values) == 0
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types8():
+    filters = []
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.BLACK
+    assert len(auth_types.values) == 0
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types_ids1():
+    filters = [
+        stix2.Filter("type", "in", ["foo", "bar"]),
+        stix2.Filter("id", "=", "foo--00000000-0000-0000-0000-000000000000")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"foo"}
+    assert auth_ids.auth_type == AuthSet.WHITE
+    assert auth_ids.values == {"foo--00000000-0000-0000-0000-000000000000"}
+
+
+def test_optimize_types_ids2():
+    filters = [
+        stix2.Filter("type", "=", "foo"),
+        stix2.Filter("id", "=", "bar--00000000-0000-0000-0000-000000000000")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert len(auth_types.values) == 0
+    assert auth_ids.auth_type == AuthSet.WHITE
+    assert len(auth_ids.values) == 0
+
+
+def test_optimize_types_ids3():
+    filters = [
+        stix2.Filter("type", "in", ["foo", "bar"]),
+        stix2.Filter("id", "!=", "bar--00000000-0000-0000-0000-000000000000")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"foo", "bar"}
+    assert auth_ids.auth_type == AuthSet.BLACK
+    assert auth_ids.values == {"bar--00000000-0000-0000-0000-000000000000"}
+
+
+def test_optimize_types_ids4():
+    filters = [
+        stix2.Filter("type", "in", ["A", "B", "C"]),
+        stix2.Filter("id", "in", [
+            "B--00000000-0000-0000-0000-000000000000",
+            "C--00000000-0000-0000-0000-000000000000",
+            "D--00000000-0000-0000-0000-000000000000",
+        ])
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"B", "C"}
+    assert auth_ids.auth_type == AuthSet.WHITE
+    assert auth_ids.values == {
+        "B--00000000-0000-0000-0000-000000000000",
+        "C--00000000-0000-0000-0000-000000000000"
+    }
+
+
+def test_optimize_types_ids5():
+    filters = [
+        stix2.Filter("type", "in", ["A", "B", "C"]),
+        stix2.Filter("type", "!=", "C"),
+        stix2.Filter("id", "in", [
+            "B--00000000-0000-0000-0000-000000000000",
+            "C--00000000-0000-0000-0000-000000000000",
+            "D--00000000-0000-0000-0000-000000000000"
+        ]),
+        stix2.Filter("id", "!=", "D--00000000-0000-0000-0000-000000000000")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"B"}
+    assert auth_ids.auth_type == AuthSet.WHITE
+    assert auth_ids.values == {"B--00000000-0000-0000-0000-000000000000"}
+
+
+def test_optimize_types_ids6():
+    filters = [
+        stix2.Filter("id", "=", "A--00000000-0000-0000-0000-000000000000")
+    ]
+
+    auth_types, auth_ids = _find_search_optimizations(filters)
+
+    assert auth_types.auth_type == AuthSet.WHITE
+    assert auth_types.values == {"A"}
+    assert auth_ids.auth_type == AuthSet.WHITE
+    assert auth_ids.values == {"A--00000000-0000-0000-0000-000000000000"}
+
+
+def test_search_auth_set_white1():
+    auth_set = AuthSet(
+        {"attack-pattern", "doesntexist"},
+        set()
+    )
+
+    results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR)
+    assert results == ["attack-pattern"]
+
+    results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISREG)
+    assert len(results) == 0
+
+
+def test_search_auth_set_white2():
+    auth_set = AuthSet(
+        {
+            "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38",
+            "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841"
+
+        },
+        {
+            "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841",
+            "malware--96b08451-b27a-4ff6-893f-790e26393a8e",
+            "doesntexist"
+        }
+    )
+
+    results = _get_matching_dir_entries(
+        os.path.join(FS_PATH, "malware"),
+        auth_set, stat.S_ISDIR
+    )
+
+    assert results == ["malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"]
+
+
+def test_search_auth_set_white3():
+    auth_set = AuthSet({"20170531213258226477", "doesntexist"}, set())
+
+    results = _get_matching_dir_entries(
+        os.path.join(FS_PATH, "malware",
+                     "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"),
+        auth_set, stat.S_ISREG, ".json"
+    )
+
+    assert results == ["20170531213258226477.json"]
+
+
+def test_search_auth_set_black1():
+    auth_set = AuthSet(
+        None,
+        {"tool--242f3da3-4425-4d11-8f5c-b842886da966", "doesntexist"}
+    )
+
+    results = _get_matching_dir_entries(
+        os.path.join(FS_PATH, "tool"),
+        auth_set, stat.S_ISDIR
+    )
+
+    assert set(results) == {
+        "tool--03342581-f790-4f03-ba41-e82e67392e23"
+    }
+
+
+def test_search_auth_set_white_empty():
+    auth_set = AuthSet(
+        set(),
+        set()
+    )
+
+    results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR)
+
+    assert len(results) == 0
+
+
+def test_search_auth_set_black_empty(rel_fs_store):
+    # Ensure rel_fs_store fixture has run so that the type directories are
+    # predictable (it adds "campaign").
+    auth_set = AuthSet(
+        None,
+        set()
+    )
+
+    results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR)
+
+    # Should get all dirs
+    assert set(results) == {
+        "attack-pattern",
+        "campaign",
+        "course-of-action",
+        "identity",
+        "indicator",
+        "intrusion-set",
+        "malware",
+        "marking-definition",
+        "relationship",
+        "tool"
+    }
+
+
+def test_timestamp2filename_naive():
+    dt = datetime.datetime(
+        2010, 6, 15,
+        8, 30, 10, 1234
+    )
+
+    filename = _timestamp2filename(dt)
+    assert filename == "20100615083010001234"
+
+
+def test_timestamp2filename_tz():
+    # one hour west of UTC (i.e. an hour earlier)
+    tz = pytz.FixedOffset(-60)
+    dt = datetime.datetime(
+        2010, 6, 15,
+        7, 30, 10, 1234,
+        tz
+    )
+
+    filename = _timestamp2filename(dt)
+    assert filename == "20100615083010001234"

stix2/test/v20/test_datastore_filters.py

@@ -196,8 +196,8 @@ def test_apply_common_filters3():
     assert len(resp) == 3
 
     resp = list(apply_common_filters(real_stix_objs, [filters[3]]))
-    assert resp[0].id == real_stix_objs[0].id
     assert len(resp) == 3
+    assert resp[0].id == real_stix_objs[0].id
 
 
 def test_apply_common_filters4():
@@ -338,14 +338,6 @@ def test_datetime_filter_behavior():
     filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
     filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
 
-    # check that filter value is converted from datetime to str
-    assert isinstance(filter_with_dt_obj.value, str)
-
-    # compare datetime string to filter w/ datetime obj
-    resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj]))
-    assert len(resp) == 1
-    assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
-
     # compare datetime obj to filter w/ datetime obj
     resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
     assert len(resp) == 1

stix2/utils.py

@@ -8,6 +8,8 @@ import json
 from dateutil import parser
 import pytz
 
+import stix2.base
+
 from .exceptions import (
     InvalidValueError, RevokeError, UnmodifiablePropertyError,
 )
@@ -385,3 +387,20 @@ def remove_custom_stix(stix_obj):
 
 def get_type_from_id(stix_id):
     return stix_id.split('--', 1)[0]
+
+
+def is_marking(obj_or_id):
+    """Determines whether the given object or object ID is/is for a marking
+    definition.
+
+    :param obj_or_id: A STIX object or object ID as a string.
+    :return: True if a marking definition, False otherwise.
+    """
+
+    if isinstance(obj_or_id, (stix2.base._STIXBase, dict)):
+        result = obj_or_id["type"] == "marking-definition"
+    else:
+        # it's a string ID
+        result = obj_or_id.startswith("marking-definition--")
+
+    return result