diff --git a/CHANGELOG b/CHANGELOG index b764735..dc0d91e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,19 @@ CHANGELOG ========= +1.3.1 - 2020-03-06 + +* #322 Adds encoding option FileSystemSource and MemorySource +* #354 Adds ability to specify id-contributing properties on custom SCOs +* #346 Certain SCO properties are no longer deprecated +* #327 Fixes missing 'name' property on Marking Definitions +* #303 Fixes bug with escaping quotes in patterns +* #331 Fixes crashing bug of property names that conflict with Mapping methods +* #337 Fixes bug with detecting STIX version of content when parsing +* #342, #343 Fixes bug when adding SCOs to Memory or FileSystem Stores +* #348 Fixes bug with generating deterministic IDs for SCOs +* #344 Fixes bug with propagating errors from the pattern validator + 1.3.0 - 2020-01-04 * #305 Updates support of STIX 2.1 to WD06 diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 7ceb33b..8185269 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -175,9 +175,9 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--e7fd0fe0-25ff-4fcb-abe5-b6254a9d1a22",\n",
-       "    "created": "2019-07-25T18:18:18.241Z",\n",
-       "    "modified": "2019-07-25T18:18:18.241Z",\n",
+       "    "id": "identity--d6996982-5fb7-4364-b716-b618516989b6",\n",
+       "    "created": "2020-03-05T05:06:27.349Z",\n",
+       "    "modified": "2020-03-05T05:06:27.349Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -287,9 +287,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--033b5f42-c94f-488f-8efa-2b6a167f0d6f",\n",
-       "    "created": "2019-07-25T18:18:21.352Z",\n",
-       "    "modified": "2019-07-25T18:18:21.352Z",\n",
+       "    "id": "identity--a167d2de-9fc4-4734-a1ae-57a548aad22a",\n",
+       "    "created": "2020-03-05T05:06:29.180Z",\n",
+       "    "modified": "2020-03-05T05:06:29.180Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -511,7 +511,7 @@
        "    "type": "identity",\n",
        "    "id": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
        "    "created": "2015-12-21T19:59:11.000Z",\n",
-       "    "modified": "2019-07-25T18:18:25.927Z",\n",
+       "    "modified": "2020-03-05T05:06:32.934Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual"\n",
        "}\n",
@@ -544,7 +544,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -569,7 +569,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -645,9 +645,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-animal",\n",
-       "    "id": "x-animal--b1e4fe7f-7985-451d-855c-6ba5c265b22a",\n",
-       "    "created": "2018-04-05T18:38:19.790Z",\n",
-       "    "modified": "2018-04-05T18:38:19.790Z",\n",
+       "    "id": "x-animal--1f7ce0ad-fd3a-4cf0-9cd7-13f7bef9ecd4",\n",
+       "    "created": "2020-03-05T05:06:38.010Z",\n",
+       "    "modified": "2020-03-05T05:06:38.010Z",\n",
        "    "species": "lion",\n",
        "    "animal_class": "mammal"\n",
        "}\n",
@@ -657,7 +657,7 @@
        ""
       ]
      },
-     "execution_count": 8,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -677,7 +677,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -703,7 +703,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -784,7 +784,7 @@
        ""
       ]
      },
-     "execution_count": 10,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -811,7 +811,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -846,7 +846,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -931,7 +931,7 @@
        ""
       ]
      },
-     "execution_count": 12,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -962,7 +962,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -1043,7 +1043,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1125,7 +1125,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1155,6 +1155,316 @@
     "print(obs_data.objects[\"0\"].property_2)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### ID-Contributing Properties for Custom Cyber Observables\n",
+    "STIX 2.1 Cyber Observables (SCOs) have deterministic IDs, meaning that the ID of a SCO is based on the values of some of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID-contributing properties, then these SCOs will have the same ID. UUIDv5 is used for the deterministic IDs, using the namespace `\"00abedb4-aa42-466c-9c01-fed23315a9b7\"`. A SCO's ID-contributing properties may consist of a combination of required properties and optional properties.\n",
+    "\n",
+    "If a SCO type does not have any ID contributing properties defined, or all of the ID-contributing properties are not present on the object, then the SCO uses a randomly-generated UUIDv4. Thus, you can optionally define which of your custom SCO's properties should be ID-contributing properties. Similar to standard SCOs, your custom SCO's ID-contributing properties can be any combination of the SCO's required and optional properties.\n",
+    "\n",
+    "You define the ID-contributing properties when defining your custom SCO with the `CustomObservable` decorator. After the list of properties, you can optionally define the list of id-contributing properties. If you do not want to specify any id-contributing properties for your custom SCO, then you do not need to do anything additional.\n",
+    "\n",
+    "See the example below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 2000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--1e56f9c3-a73b-5fbd-b348-83c76523c4df",\n",
+       "    "a_property": "A different property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from stix2.v21 import CustomObservable # IDs and Deterministic IDs are NOT part of STIX 2.0 Custom Observables\n",
+    "\n",
+    "@CustomObservable('x-new-observable-2', [\n",
+    "    ('a_property', properties.StringProperty(required=True)),\n",
+    "    ('property_2', properties.IntegerProperty()),\n",
+    "], [\n",
+    "    'a_property'\n",
+    "])\n",
+    "class NewObservable2():\n",
+    "    pass\n",
+    "\n",
+    "new_observable_a = NewObservable2(a_property=\"A property\", property_2=2000)\n",
+    "print(new_observable_a)\n",
+    "\n",
+    "new_observable_b = NewObservable2(a_property=\"A property\", property_2=3000)\n",
+    "print(new_observable_b)\n",
+    "\n",
+    "new_observable_c = NewObservable2(a_property=\"A different property\", property_2=3000)\n",
+    "print(new_observable_c)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this example, `a_property` is the only id-contributing property. Notice that the ID for `new_observable_a` and `new_observable_b` is the same since they have the same value for the id-contributing `a_property` property."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1483,21 +1793,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15+"
   }
  },
  "nbformat": 4,
diff --git a/setup.cfg b/setup.cfg
index 659a1cd..7e89c66 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.3.1
 commit = True
 tag = True
 
diff --git a/stix2/custom.py b/stix2/custom.py
index a00498b..802fd07 100644
--- a/stix2/custom.py
+++ b/stix2/custom.py
@@ -53,7 +53,10 @@ def _custom_marking_builder(cls, type, properties, version):
     return _CustomMarking
 
 
-def _custom_observable_builder(cls, type, properties, version):
+def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None):
+    if id_contrib_props is None:
+        id_contrib_props = []
+
     class _CustomObservable(cls, _Observable):
 
         if not re.match(TYPE_REGEX, type):
@@ -98,6 +101,8 @@ def _custom_observable_builder(cls, type, properties, version):
 
         _type = type
         _properties = OrderedDict(properties)
+        if version != '2.0':
+            _id_contributing_properties = id_contrib_props
 
         def __init__(self, **kwargs):
             _Observable.__init__(self, **kwargs)
diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py
index e8442e6..d5acc24 100644
--- a/stix2/datastore/filesystem.py
+++ b/stix2/datastore/filesystem.py
@@ -15,7 +15,7 @@ from stix2.datastore import (
     DataSink, DataSource, DataSourceError, DataStoreMixin,
 )
 from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
-from stix2.utils import format_datetime, get_type_from_id, is_marking
+from stix2.utils import format_datetime, get_type_from_id
 
 
 def _timestamp2filename(timestamp):
@@ -329,11 +329,50 @@ def _check_object_from_file(query, filepath, allow_custom, version, encoding):
     return result
 
 
+def _is_versioned_type_dir(type_path, type_name):
+    """
+    Try to detect whether the given directory is for a versioned type of STIX
+    object.  This is done by looking for a directory whose name is a STIX ID
+    of the appropriate type.  If found, treat this type as versioned.  This
+    doesn't work when a versioned type directory is empty (it will be
+    mis-classified as unversioned), but this detection is only necessary when
+    reading/querying data.  If a directory is empty, you'll get no results
+    either way.
+
+    Args:
+        type_path: A path to a directory containing one type of STIX object.
+        type_name: The STIX type name.
+
+    Returns:
+        True if the directory looks like it contains versioned objects; False
+        if not.
+
+    Raises:
+        OSError: If there are errors accessing directory contents or stat()'ing
+            files
+    """
+    id_regex = re.compile(
+        r"^" + re.escape(type_name) +
+        r"--[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}"
+        r"-[0-9a-f]{12}$",
+        re.I,
+    )
+
+    for entry in os.listdir(type_path):
+        s = os.stat(os.path.join(type_path, entry))
+        if stat.S_ISDIR(s.st_mode) and id_regex.match(entry):
+            is_versioned = True
+            break
+    else:
+        is_versioned = False
+
+    return is_versioned
+
+
 def _search_versioned(query, type_path, auth_ids, allow_custom, version, encoding):
     """
     Searches the given directory, which contains data for STIX objects of a
-    particular versioned type (i.e. not markings), and return any which match
-    the query.
+    particular versioned type, and return any which match the query.
 
     Args:
         query: The query to match against
@@ -390,36 +429,24 @@ def _search_versioned(query, type_path, auth_ids, allow_custom, version, encodin
 
     # For backward-compatibility, also search for plain files named after
     # object IDs, in the type directory.
-    id_files = _get_matching_dir_entries(
-        type_path, auth_ids, stat.S_ISREG,
-        ".json",
+    backcompat_results = _search_unversioned(
+        query, type_path, auth_ids, allow_custom, version, encoding,
     )
-    for id_file in id_files:
-        id_path = os.path.join(type_path, id_file)
-
-        try:
-            stix_obj = _check_object_from_file(
-                query, id_path, allow_custom,
-                version, encoding,
-            )
-            if stix_obj:
-                results.append(stix_obj)
-        except IOError as e:
-            if e.errno != errno.ENOENT:
-                raise
-            # else, file-not-found is ok, just skip
+    results.extend(backcompat_results)
 
     return results
 
 
-def _search_markings(query, markings_path, auth_ids, allow_custom, version, encoding):
+def _search_unversioned(
+    query, type_path, auth_ids, allow_custom, version, encoding,
+):
     """
-    Searches the given directory, which contains markings data, and return any
-    which match the query.
+    Searches the given directory, which contains unversioned data, and return
+    any objects which match the query.
 
     Args:
         query: The query to match against
-        markings_path: The directory with STIX markings files
+        type_path: The directory with STIX files of unversioned type
         auth_ids: Search optimization based on object ID
         allow_custom (bool): Whether to allow custom properties as well unknown
             custom objects.
@@ -441,11 +468,11 @@ def _search_markings(query, markings_path, auth_ids, allow_custom, version, enco
     """
     results = []
     id_files = _get_matching_dir_entries(
-        markings_path, auth_ids, stat.S_ISREG,
+        type_path, auth_ids, stat.S_ISREG,
         ".json",
     )
     for id_file in id_files:
-        id_path = os.path.join(markings_path, id_file)
+        id_path = os.path.join(type_path, id_file)
 
         try:
             stix_obj = _check_object_from_file(
@@ -530,12 +557,14 @@ class FileSystemSink(DataSink):
         """Write the given STIX object to a file in the STIX file directory.
         """
         type_dir = os.path.join(self._stix_dir, stix_obj["type"])
-        if is_marking(stix_obj):
-            filename = stix_obj["id"]
-            obj_dir = type_dir
-        else:
+
+        # All versioned objects should have a "modified" property.
+        if "modified" in stix_obj:
             filename = _timestamp2filename(stix_obj["modified"])
             obj_dir = os.path.join(type_dir, stix_obj["id"])
+        else:
+            filename = stix_obj["id"]
+            obj_dir = type_dir
 
         file_path = os.path.join(obj_dir, filename + ".json")
 
@@ -649,12 +678,14 @@ class FileSystemSource(DataSource):
         all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters)
 
         if all_data:
-            if is_marking(stix_id):
-                # Markings are unversioned; there shouldn't be more than one
-                # result.
-                stix_obj = all_data[0]
-            else:
+            # Simple check for a versioned STIX type: see if the objects have a
+            # "modified" property.  (Need only check one, since they are all of
+            # the same type.)
+            is_versioned = "modified" in all_data[0]
+            if is_versioned:
                 stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1]
+            else:
+                stix_obj = all_data[0]
         else:
             stix_obj = None
 
@@ -720,14 +751,15 @@ class FileSystemSource(DataSource):
         )
         for type_dir in type_dirs:
             type_path = os.path.join(self._stix_dir, type_dir)
-            if type_dir == "marking-definition":
-                type_results = _search_markings(
+            type_is_versioned = _is_versioned_type_dir(type_path, type_dir)
+            if type_is_versioned:
+                type_results = _search_versioned(
                     query, type_path, auth_ids,
                     self.allow_custom, version,
                     self.encoding,
                 )
             else:
-                type_results = _search_versioned(
+                type_results = _search_unversioned(
                     query, type_path, auth_ids,
                     self.allow_custom, version,
                     self.encoding,
diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py
index b2d7a53..6ac3e98 100644
--- a/stix2/pattern_visitor.py
+++ b/stix2/pattern_visitor.py
@@ -1,16 +1,12 @@
 import importlib
 import inspect
 
-from antlr4 import CommonTokenStream, InputStream
-from antlr4.tree.Trees import Trees
-import six
 from stix2patterns.exceptions import ParseException
-from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer
 from stix2patterns.grammars.STIXPatternParser import (
     STIXPatternParser, TerminalNode,
 )
 from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor
-from stix2patterns.validator import STIXPatternErrorListener
+from stix2patterns.v20.pattern import Pattern
 
 from .patterns import *
 from .patterns import _BooleanExpression
@@ -328,41 +324,9 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor):
 
 def create_pattern_object(pattern, module_suffix="", module_name=""):
     """
-    Validates a pattern against the STIX Pattern grammar.  Error messages are
-    returned in a list.  The test passed if the returned list is empty.
+    Create a STIX pattern AST from a pattern string.
     """
 
-    start = ''
-    if isinstance(pattern, six.string_types):
-        start = pattern[:2]
-        pattern = InputStream(pattern)
-
-    if not start:
-        start = pattern.readline()[:2]
-        pattern.seek(0)
-
-    parseErrListener = STIXPatternErrorListener()
-
-    lexer = STIXPatternLexer(pattern)
-    # it always adds a console listener by default... remove it.
-    lexer.removeErrorListeners()
-
-    stream = CommonTokenStream(lexer)
-
-    parser = STIXPatternParser(stream)
-
-    parser.buildParseTrees = True
-    # it always adds a console listener by default... remove it.
-    parser.removeErrorListeners()
-    parser.addErrorListener(parseErrListener)
-
-    # To improve error messages, replace "" in the literal
-    # names with symbolic names.  This is a hack, but seemed like
-    # the simplest workaround.
-    for i, lit_name in enumerate(parser.literalNames):
-        if lit_name == u"":
-            parser.literalNames[i] = parser.symbolicNames[i]
-
-    tree = parser.pattern()
+    pattern_obj = Pattern(pattern)
     builder = STIXPatternVisitorForSTIX2(module_suffix, module_name)
-    return builder.visit(tree)
+    return pattern_obj.visit(builder)
diff --git a/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json b/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json
new file mode 100644
index 0000000..3812ed4
--- /dev/null
+++ b/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json
@@ -0,0 +1,11 @@
+{
+    "ctime": "2020-10-06T01:54:32.000Z",
+    "contains_refs": [
+        "directory--80539e31-85f3-4304-bd14-e2e8c10859a5",
+        "file--e9e03175-0357-41b5-a2aa-eb99b455cd0c",
+        "directory--f6c54233-027b-4464-8126-da1324d8f66c"
+    ],
+    "path": "/performance/Democrat.gif",
+    "type": "directory",
+    "id": "directory--572827aa-e0cd-44fd-afd5-a717a7585f39"
+}
diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py
index 9c650eb..b46288d 100644
--- a/stix2/test/v21/test_custom.py
+++ b/stix2/test/v21/test_custom.py
@@ -1,3 +1,5 @@
+import uuid
+
 import pytest
 
 import stix2
@@ -665,6 +667,76 @@ def test_observed_data_with_custom_observable_object():
     assert ob_data.objects['0'].property1 == 'something'
 
 
+def test_custom_observable_object_det_id_1():
+    @stix2.v21.CustomObservable(
+        'x-det-id-observable-1', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+            ('property2', stix2.properties.IntegerProperty()),
+        ], [
+            'property1',
+        ],
+    )
+    class DetIdObs1():
+        pass
+
+    dio_1 = DetIdObs1(property1='I am property1!', property2=42)
+    dio_2 = DetIdObs1(property1='I am property1!', property2=24)
+    assert dio_1.property1 == dio_2.property1 == 'I am property1!'
+    assert dio_1.id == dio_2.id
+
+    uuid_obj = uuid.UUID(dio_1.id[-36:])
+    assert uuid_obj.variant == uuid.RFC_4122
+    assert uuid_obj.version == 5
+
+    dio_3 = DetIdObs1(property1='I am property1!', property2=42)
+    dio_4 = DetIdObs1(property1='I am also property1!', property2=24)
+    assert dio_3.property1 == 'I am property1!'
+    assert dio_4.property1 == 'I am also property1!'
+    assert dio_3.id != dio_4.id
+
+
+def test_custom_observable_object_det_id_2():
+    @stix2.v21.CustomObservable(
+        'x-det-id-observable-2', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+            ('property2', stix2.properties.IntegerProperty()),
+        ], [
+            'property1', 'property2',
+        ],
+    )
+    class DetIdObs2():
+        pass
+
+    dio_1 = DetIdObs2(property1='I am property1!', property2=42)
+    dio_2 = DetIdObs2(property1='I am property1!', property2=42)
+    assert dio_1.property1 == dio_2.property1 == 'I am property1!'
+    assert dio_1.property2 == dio_2.property2 == 42
+    assert dio_1.id == dio_2.id
+
+    dio_3 = DetIdObs2(property1='I am property1!', property2=42)
+    dio_4 = DetIdObs2(property1='I am also property1!', property2=42)
+    assert dio_3.property1 == 'I am property1!'
+    assert dio_4.property1 == 'I am also property1!'
+    assert dio_3.property2 == dio_4.property2 == 42
+    assert dio_3.id != dio_4.id
+
+
+def test_custom_observable_object_no_id_contrib_props():
+    @stix2.v21.CustomObservable(
+        'x-det-id-observable-3', [
+            ('property1', stix2.properties.StringProperty(required=True)),
+        ],
+    )
+    class DetIdObs3():
+        pass
+
+    dio = DetIdObs3(property1="I am property1!")
+
+    uuid_obj = uuid.UUID(dio.id[-36:])
+    assert uuid_obj.variant == uuid.RFC_4122
+    assert uuid_obj.version == 4
+
+
 @stix2.v21.CustomExtension(
     stix2.v21.DomainName, 'x-new-ext', [
         ('property1', stix2.properties.StringProperty(required=True)),
diff --git a/stix2/test/v21/test_datastore_filesystem.py b/stix2/test/v21/test_datastore_filesystem.py
index 9917ccd..3eb8aaa 100644
--- a/stix2/test/v21/test_datastore_filesystem.py
+++ b/stix2/test/v21/test_datastore_filesystem.py
@@ -221,6 +221,16 @@ def test_filesystem_source_backward_compatible(fs_source):
     assert result.malware_types == ["version four"]
 
 
+def test_filesystem_source_sco(fs_source):
+    results = fs_source.query([stix2.Filter("type", "=", "directory")])
+
+    assert len(results) == 1
+    result = results[0]
+    assert result["type"] == "directory"
+    assert result["id"] == "directory--572827aa-e0cd-44fd-afd5-a717a7585f39"
+    assert result["path"] == "/performance/Democrat.gif"
+
+
 def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source):
     # add python stix object
     camp1 = stix2.v21.Campaign(
@@ -435,6 +445,24 @@ def test_filesystem_sink_marking(fs_sink):
     os.remove(marking_filepath)
 
 
+def test_filesystem_sink_sco(fs_sink):
+    file_sco = {
+        "type": "file",
+        "id": "file--decfcc48-31b3-45f5-87c8-1b3a5d71a307",
+        "name": "cats.png",
+    }
+
+    fs_sink.add(file_sco)
+    sco_filepath = os.path.join(
+        FS_PATH, "file", file_sco["id"] + ".json",
+    )
+
+    assert os.path.exists(sco_filepath)
+
+    os.remove(sco_filepath)
+    os.rmdir(os.path.dirname(sco_filepath))
+
+
 def test_filesystem_store_get_stored_as_bundle(fs_store):
     coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f")
     assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f"
@@ -473,9 +501,10 @@ def test_filesystem_store_query_single_filter(fs_store):
 
 def test_filesystem_store_empty_query(fs_store):
     results = fs_store.query()  # returns all
-    assert len(results) == 30
+    assert len(results) == 31
     assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results]
     assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results]
+    assert "directory--572827aa-e0cd-44fd-afd5-a717a7585f39" in [obj.id for obj in results]
 
 
 def test_filesystem_store_query_multiple_filters(fs_store):
@@ -487,7 +516,7 @@ def test_filesystem_store_query_multiple_filters(fs_store):
 
 def test_filesystem_store_query_dont_include_type_folder(fs_store):
     results = fs_store.query(stix2.Filter("type", "!=", "tool"))
-    assert len(results) == 28
+    assert len(results) == 29
 
 
 def test_filesystem_store_add(fs_store):
@@ -574,6 +603,26 @@ def test_filesystem_store_add_marking(fs_store):
     os.remove(marking_filepath)
 
 
+def test_filesystem_store_add_sco(fs_store):
+    sco = stix2.v21.EmailAddress(
+        value="jdoe@example.com",
+    )
+
+    fs_store.add(sco)
+    sco_filepath = os.path.join(
+        FS_PATH, "email-addr", sco["id"] + ".json",
+    )
+
+    assert os.path.exists(sco_filepath)
+
+    sco_r = fs_store.get(sco["id"])
+    assert sco_r["id"] == sco["id"]
+    assert sco_r["value"] == sco["value"]
+
+    os.remove(sco_filepath)
+    os.rmdir(os.path.dirname(sco_filepath))
+
+
 def test_filesystem_object_with_custom_property(fs_store):
     camp = stix2.v21.Campaign(
         name="Scipio Africanus",
@@ -1024,6 +1073,7 @@ def test_search_auth_set_black_empty(rel_fs_store):
         "attack-pattern",
         "campaign",
         "course-of-action",
+        "directory",
         "identity",
         "indicator",
         "intrusion-set",
diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py
index 152f253..d7d7e47 100644
--- a/stix2/test/v21/test_indicator.py
+++ b/stix2/test/v21/test_indicator.py
@@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern():
         )
 
     assert excinfo.value.cls == stix2.v21.Indicator
-    assert "FAIL: The same qualifier is used more than once" in str(excinfo.value)
+    assert "FAIL: Duplicate qualifier type encountered: WITHIN" in str(excinfo.value)
 
     ind = stix2.v21.Indicator(
         type="indicator",
diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py
index 76880be..0c298f8 100644
--- a/stix2/test/v21/test_pattern_expressions.py
+++ b/stix2/test/v21/test_pattern_expressions.py
@@ -1,6 +1,7 @@
 import datetime
 
 import pytest
+from stix2patterns.exceptions import ParseException
 
 import stix2
 from stix2.pattern_visitor import create_pattern_object
@@ -515,3 +516,8 @@ def test_list_constant():
 def test_parsing_multiple_slashes_quotes():
     patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]")
     assert str(patt_obj) == "[file:name = 'weird_name\\'']"
+
+
+def test_parse_error():
+    with pytest.raises(ParseException):
+        create_pattern_object("[ file: name = 'weirdname]")
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index ed560a6..e8c1925 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -966,7 +966,7 @@ class X509Certificate(_Observable):
         self._check_at_least_one_property(att_list)
 
 
-def CustomObservable(type='x-custom-observable', properties=None):
+def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=None):
     """Custom STIX Cyber Observable Object type decorator.
 
     Example:
@@ -987,7 +987,7 @@ def CustomObservable(type='x-custom-observable', properties=None):
             properties,
             [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))],
         ]))
-        return _custom_observable_builder(cls, type, _properties, '2.1')
+        return _custom_observable_builder(cls, type, _properties, '2.1', id_contrib_props)
     return wrapper
 
 
diff --git a/stix2/version.py b/stix2/version.py
index 67bc602..9c73af2 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "1.3.0"
+__version__ = "1.3.1"