From 4c67142b9274049cfc9d983ca6153585ca97696c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Sat, 15 Feb 2020 19:02:53 -0500 Subject: [PATCH 01/17] Fix the filesystem store to support the new top-level 2.1 SCOs. --- stix2/datastore/filesystem.py | 108 ++++++++++++------ ...-572827aa-e0cd-44fd-afd5-a717a7585f39.json | 11 ++ stix2/test/v21/test_datastore_filesystem.py | 54 ++++++++- 3 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index e8442e6..d5acc24 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -15,7 +15,7 @@ from stix2.datastore import ( DataSink, DataSource, DataSourceError, DataStoreMixin, ) from stix2.datastore.filters import Filter, FilterSet, apply_common_filters -from stix2.utils import format_datetime, get_type_from_id, is_marking +from stix2.utils import format_datetime, get_type_from_id def _timestamp2filename(timestamp): @@ -329,11 +329,50 @@ def _check_object_from_file(query, filepath, allow_custom, version, encoding): return result +def _is_versioned_type_dir(type_path, type_name): + """ + Try to detect whether the given directory is for a versioned type of STIX + object. This is done by looking for a directory whose name is a STIX ID + of the appropriate type. If found, treat this type as versioned. This + doesn't work when a versioned type directory is empty (it will be + mis-classified as unversioned), but this detection is only necessary when + reading/querying data. If a directory is empty, you'll get no results + either way. + + Args: + type_path: A path to a directory containing one type of STIX object. + type_name: The STIX type name. + + Returns: + True if the directory looks like it contains versioned objects; False + if not. + + Raises: + OSError: If there are errors accessing directory contents or stat()'ing + files + """ + id_regex = re.compile( + r"^" + re.escape(type_name) + + r"--[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}" + r"-[0-9a-f]{12}$", + re.I, + ) + + for entry in os.listdir(type_path): + s = os.stat(os.path.join(type_path, entry)) + if stat.S_ISDIR(s.st_mode) and id_regex.match(entry): + is_versioned = True + break + else: + is_versioned = False + + return is_versioned + + def _search_versioned(query, type_path, auth_ids, allow_custom, version, encoding): """ Searches the given directory, which contains data for STIX objects of a - particular versioned type (i.e. not markings), and return any which match - the query. + particular versioned type, and return any which match the query. Args: query: The query to match against @@ -390,36 +429,24 @@ def _search_versioned(query, type_path, auth_ids, allow_custom, version, encodin # For backward-compatibility, also search for plain files named after # object IDs, in the type directory. - id_files = _get_matching_dir_entries( - type_path, auth_ids, stat.S_ISREG, - ".json", + backcompat_results = _search_unversioned( + query, type_path, auth_ids, allow_custom, version, encoding, ) - for id_file in id_files: - id_path = os.path.join(type_path, id_file) - - try: - stix_obj = _check_object_from_file( - query, id_path, allow_custom, - version, encoding, - ) - if stix_obj: - results.append(stix_obj) - except IOError as e: - if e.errno != errno.ENOENT: - raise - # else, file-not-found is ok, just skip + results.extend(backcompat_results) return results -def _search_markings(query, markings_path, auth_ids, allow_custom, version, encoding): +def _search_unversioned( + query, type_path, auth_ids, allow_custom, version, encoding, +): """ - Searches the given directory, which contains markings data, and return any - which match the query. + Searches the given directory, which contains unversioned data, and return + any objects which match the query. Args: query: The query to match against - markings_path: The directory with STIX markings files + type_path: The directory with STIX files of unversioned type auth_ids: Search optimization based on object ID allow_custom (bool): Whether to allow custom properties as well unknown custom objects. @@ -441,11 +468,11 @@ def _search_markings(query, markings_path, auth_ids, allow_custom, version, enco """ results = [] id_files = _get_matching_dir_entries( - markings_path, auth_ids, stat.S_ISREG, + type_path, auth_ids, stat.S_ISREG, ".json", ) for id_file in id_files: - id_path = os.path.join(markings_path, id_file) + id_path = os.path.join(type_path, id_file) try: stix_obj = _check_object_from_file( @@ -530,12 +557,14 @@ class FileSystemSink(DataSink): """Write the given STIX object to a file in the STIX file directory. """ type_dir = os.path.join(self._stix_dir, stix_obj["type"]) - if is_marking(stix_obj): - filename = stix_obj["id"] - obj_dir = type_dir - else: + + # All versioned objects should have a "modified" property. + if "modified" in stix_obj: filename = _timestamp2filename(stix_obj["modified"]) obj_dir = os.path.join(type_dir, stix_obj["id"]) + else: + filename = stix_obj["id"] + obj_dir = type_dir file_path = os.path.join(obj_dir, filename + ".json") @@ -649,12 +678,14 @@ class FileSystemSource(DataSource): all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters) if all_data: - if is_marking(stix_id): - # Markings are unversioned; there shouldn't be more than one - # result. - stix_obj = all_data[0] - else: + # Simple check for a versioned STIX type: see if the objects have a + # "modified" property. (Need only check one, since they are all of + # the same type.) + is_versioned = "modified" in all_data[0] + if is_versioned: stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1] + else: + stix_obj = all_data[0] else: stix_obj = None @@ -720,14 +751,15 @@ class FileSystemSource(DataSource): ) for type_dir in type_dirs: type_path = os.path.join(self._stix_dir, type_dir) - if type_dir == "marking-definition": - type_results = _search_markings( + type_is_versioned = _is_versioned_type_dir(type_path, type_dir) + if type_is_versioned: + type_results = _search_versioned( query, type_path, auth_ids, self.allow_custom, version, self.encoding, ) else: - type_results = _search_versioned( + type_results = _search_unversioned( query, type_path, auth_ids, self.allow_custom, version, self.encoding, diff --git a/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json b/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json new file mode 100644 index 0000000..3812ed4 --- /dev/null +++ b/stix2/test/v21/stix2_data/directory/directory--572827aa-e0cd-44fd-afd5-a717a7585f39.json @@ -0,0 +1,11 @@ +{ + "ctime": "2020-10-06T01:54:32.000Z", + "contains_refs": [ + "directory--80539e31-85f3-4304-bd14-e2e8c10859a5", + "file--e9e03175-0357-41b5-a2aa-eb99b455cd0c", + "directory--f6c54233-027b-4464-8126-da1324d8f66c" + ], + "path": "/performance/Democrat.gif", + "type": "directory", + "id": "directory--572827aa-e0cd-44fd-afd5-a717a7585f39" +} diff --git a/stix2/test/v21/test_datastore_filesystem.py b/stix2/test/v21/test_datastore_filesystem.py index 9917ccd..3eb8aaa 100644 --- a/stix2/test/v21/test_datastore_filesystem.py +++ b/stix2/test/v21/test_datastore_filesystem.py @@ -221,6 +221,16 @@ def test_filesystem_source_backward_compatible(fs_source): assert result.malware_types == ["version four"] +def test_filesystem_source_sco(fs_source): + results = fs_source.query([stix2.Filter("type", "=", "directory")]) + + assert len(results) == 1 + result = results[0] + assert result["type"] == "directory" + assert result["id"] == "directory--572827aa-e0cd-44fd-afd5-a717a7585f39" + assert result["path"] == "/performance/Democrat.gif" + + def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): # add python stix object camp1 = stix2.v21.Campaign( @@ -435,6 +445,24 @@ def test_filesystem_sink_marking(fs_sink): os.remove(marking_filepath) +def test_filesystem_sink_sco(fs_sink): + file_sco = { + "type": "file", + "id": "file--decfcc48-31b3-45f5-87c8-1b3a5d71a307", + "name": "cats.png", + } + + fs_sink.add(file_sco) + sco_filepath = os.path.join( + FS_PATH, "file", file_sco["id"] + ".json", + ) + + assert os.path.exists(sco_filepath) + + os.remove(sco_filepath) + os.rmdir(os.path.dirname(sco_filepath)) + + def test_filesystem_store_get_stored_as_bundle(fs_store): coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f") assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f" @@ -473,9 +501,10 @@ def test_filesystem_store_query_single_filter(fs_store): def test_filesystem_store_empty_query(fs_store): results = fs_store.query() # returns all - assert len(results) == 30 + assert len(results) == 31 assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] + assert "directory--572827aa-e0cd-44fd-afd5-a717a7585f39" in [obj.id for obj in results] def test_filesystem_store_query_multiple_filters(fs_store): @@ -487,7 +516,7 @@ def test_filesystem_store_query_multiple_filters(fs_store): def test_filesystem_store_query_dont_include_type_folder(fs_store): results = fs_store.query(stix2.Filter("type", "!=", "tool")) - assert len(results) == 28 + assert len(results) == 29 def test_filesystem_store_add(fs_store): @@ -574,6 +603,26 @@ def test_filesystem_store_add_marking(fs_store): os.remove(marking_filepath) +def test_filesystem_store_add_sco(fs_store): + sco = stix2.v21.EmailAddress( + value="jdoe@example.com", + ) + + fs_store.add(sco) + sco_filepath = os.path.join( + FS_PATH, "email-addr", sco["id"] + ".json", + ) + + assert os.path.exists(sco_filepath) + + sco_r = fs_store.get(sco["id"]) + assert sco_r["id"] == sco["id"] + assert sco_r["value"] == sco["value"] + + os.remove(sco_filepath) + os.rmdir(os.path.dirname(sco_filepath)) + + def test_filesystem_object_with_custom_property(fs_store): camp = stix2.v21.Campaign( name="Scipio Africanus", @@ -1024,6 +1073,7 @@ def test_search_auth_set_black_empty(rel_fs_store): "attack-pattern", "campaign", "course-of-action", + "directory", "identity", "indicator", "intrusion-set", From cfb7c4c73bdad7597ae9f815bb9b8f5a33e919aa Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 17 Feb 2020 19:26:21 -0500 Subject: [PATCH 02/17] Fix stix2.pattern_visitor.create_pattern_object() so its documentation at least isn't wrong, and it behaves better. --- stix2/pattern_visitor.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index b2d7a53..e4601ba 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,16 +1,15 @@ import importlib import inspect -from antlr4 import CommonTokenStream, InputStream -from antlr4.tree.Trees import Trees +from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream +import antlr4.error.Errors import six -from stix2patterns.exceptions import ParseException +from stix2patterns.exceptions import ParseException, ParserErrorListener from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor -from stix2patterns.validator import STIXPatternErrorListener from .patterns import * from .patterns import _BooleanExpression @@ -328,20 +327,12 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def create_pattern_object(pattern, module_suffix="", module_name=""): """ - Validates a pattern against the STIX Pattern grammar. Error messages are - returned in a list. The test passed if the returned list is empty. + Create a STIX pattern AST from a pattern string. """ - start = '' - if isinstance(pattern, six.string_types): - start = pattern[:2] - pattern = InputStream(pattern) + pattern = InputStream(pattern) - if not start: - start = pattern.readline()[:2] - pattern.seek(0) - - parseErrListener = STIXPatternErrorListener() + parseErrListener = ParserErrorListener() lexer = STIXPatternLexer(pattern) # it always adds a console listener by default... remove it. @@ -350,6 +341,7 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): stream = CommonTokenStream(lexer) parser = STIXPatternParser(stream) + parser._errHandler = BailErrorStrategy() parser.buildParseTrees = True # it always adds a console listener by default... remove it. @@ -363,6 +355,15 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): if lit_name == u"": parser.literalNames[i] = parser.symbolicNames[i] - tree = parser.pattern() + try: + tree = parser.pattern() + except antlr4.error.Errors.ParseCancellationException as e: + real_exc = e.args[0] + parser._errHandler.reportError(parser, real_exc) + six.raise_from( + ParseException(parseErrListener.error_message), + real_exc, + ) + builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) return builder.visit(tree) From 14daa1edae88cb6c08474143b6b7b26d775a4e3c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 15:39:23 -0500 Subject: [PATCH 03/17] Add a test case to test parse exceptions from create_pattern_object(). --- stix2/test/v21/test_pattern_expressions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 76880be..0c298f8 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -1,6 +1,7 @@ import datetime import pytest +from stix2patterns.exceptions import ParseException import stix2 from stix2.pattern_visitor import create_pattern_object @@ -515,3 +516,8 @@ def test_list_constant(): def test_parsing_multiple_slashes_quotes(): patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]") assert str(patt_obj) == "[file:name = 'weird_name\\'']" + + +def test_parse_error(): + with pytest.raises(ParseException): + create_pattern_object("[ file: name = 'weirdname]") From 76a6eb58736e3ebd566914ca8e1b53a430a9e35d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:39:15 -0500 Subject: [PATCH 04/17] Greatly simplify the create_pattern_object() function to take advantage of the pattern validator library's Pattern.visit() method. --- stix2/pattern_visitor.py | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index e4601ba..571c66f 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -10,6 +10,7 @@ from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor +from stix2patterns.v20.pattern import Pattern from .patterns import * from .patterns import _BooleanExpression @@ -330,40 +331,6 @@ def create_pattern_object(pattern, module_suffix="", module_name=""): Create a STIX pattern AST from a pattern string. """ - pattern = InputStream(pattern) - - parseErrListener = ParserErrorListener() - - lexer = STIXPatternLexer(pattern) - # it always adds a console listener by default... remove it. - lexer.removeErrorListeners() - - stream = CommonTokenStream(lexer) - - parser = STIXPatternParser(stream) - parser._errHandler = BailErrorStrategy() - - parser.buildParseTrees = True - # it always adds a console listener by default... remove it. - parser.removeErrorListeners() - parser.addErrorListener(parseErrListener) - - # To improve error messages, replace "" in the literal - # names with symbolic names. This is a hack, but seemed like - # the simplest workaround. - for i, lit_name in enumerate(parser.literalNames): - if lit_name == u"": - parser.literalNames[i] = parser.symbolicNames[i] - - try: - tree = parser.pattern() - except antlr4.error.Errors.ParseCancellationException as e: - real_exc = e.args[0] - parser._errHandler.reportError(parser, real_exc) - six.raise_from( - ParseException(parseErrListener.error_message), - real_exc, - ) - + pattern_obj = Pattern(pattern) builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) - return builder.visit(tree) + return pattern_obj.visit(builder) From 1959cc609737ebf734768be8ec1b53104b0f7859 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 19 Feb 2020 16:45:15 -0500 Subject: [PATCH 05/17] Removed a bunch of no-longer-used imports from pattern_visitor.py --- stix2/pattern_visitor.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 571c66f..6ac3e98 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -1,11 +1,7 @@ import importlib import inspect -from antlr4 import BailErrorStrategy, CommonTokenStream, InputStream -import antlr4.error.Errors -import six -from stix2patterns.exceptions import ParseException, ParserErrorListener -from stix2patterns.grammars.STIXPatternLexer import STIXPatternLexer +from stix2patterns.exceptions import ParseException from stix2patterns.grammars.STIXPatternParser import ( STIXPatternParser, TerminalNode, ) From 274abc52e91ab2d0e229ccfb67b8cc5c833b8316 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 24 Feb 2020 20:02:26 -0500 Subject: [PATCH 06/17] An exception message changed as a result of a pattern-validator update. This broke a unit test which was testing the message. I updated the test. --- stix2/test/v21/test_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 152f253..d7d7e47 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern(): ) assert excinfo.value.cls == stix2.v21.Indicator - assert "FAIL: The same qualifier is used more than once" in str(excinfo.value) + assert "FAIL: Duplicate qualifier type encountered: WITHIN" in str(excinfo.value) ind = stix2.v21.Indicator( type="indicator", From 41e541959d3c174917030fcee99c07150f8a05f4 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 24 Feb 2020 21:11:42 -0500 Subject: [PATCH 07/17] Add _id_contributing_properties functionality to custom SCOs. Tests coming soon. Fixes #351 --- stix2/custom.py | 4 +++- stix2/v21/observables.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index a00498b..81991e9 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -53,7 +53,7 @@ def _custom_marking_builder(cls, type, properties, version): return _CustomMarking -def _custom_observable_builder(cls, type, properties, version): +def _custom_observable_builder(cls, type, properties, version, id_contrib_props=[]): class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): @@ -98,6 +98,8 @@ def _custom_observable_builder(cls, type, properties, version): _type = type _properties = OrderedDict(properties) + if version != '2.0': + _id_contributing_properties = id_contrib_props def __init__(self, **kwargs): _Observable.__init__(self, **kwargs) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index ed560a6..5c23362 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -966,7 +966,7 @@ class X509Certificate(_Observable): self._check_at_least_one_property(att_list) -def CustomObservable(type='x-custom-observable', properties=None): +def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=[]): """Custom STIX Cyber Observable Object type decorator. Example: @@ -987,7 +987,7 @@ def CustomObservable(type='x-custom-observable', properties=None): properties, [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))], ])) - return _custom_observable_builder(cls, type, _properties, '2.1') + return _custom_observable_builder(cls, type, _properties, '2.1', id_contrib_props) return wrapper From 055ad97a7aee4e14d1acd8258e9eeccf38254578 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 27 Feb 2020 15:15:37 -0500 Subject: [PATCH 08/17] Add tests for _id_contributing_properties for custom observables --- stix2/test/v21/test_custom.py | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 9c650eb..b46288d 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1,3 +1,5 @@ +import uuid + import pytest import stix2 @@ -665,6 +667,76 @@ def test_observed_data_with_custom_observable_object(): assert ob_data.objects['0'].property1 == 'something' +def test_custom_observable_object_det_id_1(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-1', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], [ + 'property1', + ], + ) + class DetIdObs1(): + pass + + dio_1 = DetIdObs1(property1='I am property1!', property2=42) + dio_2 = DetIdObs1(property1='I am property1!', property2=24) + assert dio_1.property1 == dio_2.property1 == 'I am property1!' + assert dio_1.id == dio_2.id + + uuid_obj = uuid.UUID(dio_1.id[-36:]) + assert uuid_obj.variant == uuid.RFC_4122 + assert uuid_obj.version == 5 + + dio_3 = DetIdObs1(property1='I am property1!', property2=42) + dio_4 = DetIdObs1(property1='I am also property1!', property2=24) + assert dio_3.property1 == 'I am property1!' + assert dio_4.property1 == 'I am also property1!' + assert dio_3.id != dio_4.id + + +def test_custom_observable_object_det_id_2(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], [ + 'property1', 'property2', + ], + ) + class DetIdObs2(): + pass + + dio_1 = DetIdObs2(property1='I am property1!', property2=42) + dio_2 = DetIdObs2(property1='I am property1!', property2=42) + assert dio_1.property1 == dio_2.property1 == 'I am property1!' + assert dio_1.property2 == dio_2.property2 == 42 + assert dio_1.id == dio_2.id + + dio_3 = DetIdObs2(property1='I am property1!', property2=42) + dio_4 = DetIdObs2(property1='I am also property1!', property2=42) + assert dio_3.property1 == 'I am property1!' + assert dio_4.property1 == 'I am also property1!' + assert dio_3.property2 == dio_4.property2 == 42 + assert dio_3.id != dio_4.id + + +def test_custom_observable_object_no_id_contrib_props(): + @stix2.v21.CustomObservable( + 'x-det-id-observable-3', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class DetIdObs3(): + pass + + dio = DetIdObs3(property1="I am property1!") + + uuid_obj = uuid.UUID(dio.id[-36:]) + assert uuid_obj.variant == uuid.RFC_4122 + assert uuid_obj.version == 4 + + @stix2.v21.CustomExtension( stix2.v21.DomainName, 'x-new-ext', [ ('property1', stix2.properties.StringProperty(required=True)), From 4f00c7ca4fe3088adb664b368d8c1c65ee323362 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 4 Mar 2020 13:33:54 -0500 Subject: [PATCH 09/17] Fix patterning test --- stix2/test/v21/test_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 152f253..23bad29 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -271,7 +271,7 @@ def test_indicator_stix20_invalid_pattern(): ) assert excinfo.value.cls == stix2.v21.Indicator - assert "FAIL: The same qualifier is used more than once" in str(excinfo.value) + assert "FAIL: Duplicate qualifier type encountered" in str(excinfo.value) ind = stix2.v21.Indicator( type="indicator", From fc95b400ff4794b0418fab427fd6324163aee352 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Mar 2020 14:29:35 -0500 Subject: [PATCH 10/17] Change default parameters from empty lists to None. Fixes #351 --- stix2/custom.py | 2 +- stix2/v21/observables.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 81991e9..0db9d25 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -53,7 +53,7 @@ def _custom_marking_builder(cls, type, properties, version): return _CustomMarking -def _custom_observable_builder(cls, type, properties, version, id_contrib_props=[]): +def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None): class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 5c23362..88d2f6c 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -966,7 +966,7 @@ class X509Certificate(_Observable): self._check_at_least_one_property(att_list) -def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=[]): +def CustomObservable(type='x-custom-observable', properties=None, id_contrib_props=None): """Custom STIX Cyber Observable Object type decorator. Example: @@ -980,6 +980,9 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro ... pass """ + if id_contrib_props is None: + id_contrib_props = [] + def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type))], From a5cd0fdc505e5cc6376331232ea4f71e0f4c131d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Mar 2020 14:46:55 -0500 Subject: [PATCH 11/17] Change location of None-check for id_contrib_props. Fixes #351 --- stix2/custom.py | 3 +++ stix2/v21/observables.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 0db9d25..802fd07 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -54,6 +54,9 @@ def _custom_marking_builder(cls, type, properties, version): def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None): + if id_contrib_props is None: + id_contrib_props = [] + class _CustomObservable(cls, _Observable): if not re.match(TYPE_REGEX, type): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 88d2f6c..e8c1925 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -980,9 +980,6 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro ... pass """ - if id_contrib_props is None: - id_contrib_props = [] - def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type))], From 47b28d11940a44ee7bf2dcff9977114a5963c965 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Tue, 18 Feb 2020 16:05:00 -0500 Subject: [PATCH 12/17] Fixes #323 --- docs/guide/custom.ipynb | 390 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 389 insertions(+), 1 deletion(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 7ceb33b..bdcc80e 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -1479,6 +1479,394 @@ "print(obs_data2.objects[\"0\"].extensions[\"x-new-ext\"].property1)\n", "print(obs_data2.objects[\"0\"].extensions[\"x-new-ext\"].property2)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deterministic IDs for Cyber Observables\n", + "### Deterministic IDs\n", + "STIX 2.1 Cyber Observables (SCOs) have an ID property since SCOs are now top-level objects. However, SCOs have deterministic IDs, meaning that the ID of a SCO is based on the values of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID contributing properties (explained below), then these SCOs will have the same ID; the SCOs' ID is deterministic because the ID will not change if the values of the ID contributing properties do not change. \n", + "\n", + "A UUIDv5 is generated for the deterministic IDs, using the namespace `\"00abedb4-aa42-466c-9c01-fed23315a9b7\"`.\n", + "\n", + "In the case where a SCO does not have any defined ID contributing properties, or in the case where all of the values for the ID contributing properties are not specified, then the SCO will be assigned a randomly-generated UUIDv4.\n", + "\n", + "### ID Contributing Properties\n", + "So, what are ID contributing properties? \n", + "Every SCO has multiple defined properties, but the values of only some of those properties will contribute to the determination of the SCO's ID. \n", + "\n", + "A SCO's ID contributing properties may contain a combination of required properties and optional properties. And it is possible for all of the ID contributing properties to be optional, which means the corresponding SCO could be created without values for any of those properties, which would lead the SCO to have a randomly-generated UUIDv4 ID.\n", + "\n", + "\n", + "We will demonstrate deterministic IDs by creating four `EmailAddress` SCOs. The `EmailAddress` SCO has one ID contributing property: `value`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
email-addr--d3742ef4-9452-5935-bc42-e8c35a119757\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2.v21 import EmailAddress\n", + "\n", + "email_addr_1 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnny Doe\")\n", + "print (email_addr_1.id)\n", + "\n", + "email_addr_2 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnny Doe\")\n", + "print (email_addr_2.id)\n", + "\n", + "email_addr_3 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnathon Doe\")\n", + "print (email_addr_3.id)\n", + "\n", + "email_addr_4 = EmailAddress(value=\"johnnydoe@notmatching.com\", display_name=\"Johnny Doe\")\n", + "print (email_addr_4.id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that the ID for the first three `EmailAddress` objects is the same while the ID for the fourth is different. This is because the first three objects all have the same value for the ID contributing property. And this is despite having a different value for the `display_name` property, since it is not an ID contributing property for the `EmailAddress` SCO.\n", + "\n", + "Also note that the fourth object has a different ID despite having the same `display_name` as the first two objects; the value for the fourth object's ID contributing property is different." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1497,7 +1885,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.7" } }, "nbformat": 4, From 02171996fac8d9bceeba5891291ab03103466611 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 27 Feb 2020 16:00:59 -0500 Subject: [PATCH 13/17] Add documentation about _id_contributing_properties for custom SCOs. Will add in code example once PR 354 is merged in --- docs/guide/custom.ipynb | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index bdcc80e..0609139 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -1486,17 +1486,17 @@ "source": [ "## Deterministic IDs for Cyber Observables\n", "### Deterministic IDs\n", - "STIX 2.1 Cyber Observables (SCOs) have an ID property since SCOs are now top-level objects. However, SCOs have deterministic IDs, meaning that the ID of a SCO is based on the values of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID contributing properties (explained below), then these SCOs will have the same ID; the SCOs' ID is deterministic because the ID will not change if the values of the ID contributing properties do not change. \n", + "STIX 2.1 Cyber Observables (SCOs) have an ID property since SCOs are now top-level objects. However, SCOs have deterministic IDs, meaning that the ID of a SCO is based on the values of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID-contributing properties (explained below), then these SCOs will have the same ID; the SCOs' ID is deterministic because the ID will not change if the values of the ID contributing properties do not change. \n", "\n", "A UUIDv5 is generated for the deterministic IDs, using the namespace `\"00abedb4-aa42-466c-9c01-fed23315a9b7\"`.\n", "\n", - "In the case where a SCO does not have any defined ID contributing properties, or in the case where all of the values for the ID contributing properties are not specified, then the SCO will be assigned a randomly-generated UUIDv4.\n", + "In the case where a SCO does not have any defined ID contributing properties, or in the case where all of the values for the ID-contributing properties are not specified, then the SCO will be assigned a randomly-generated UUIDv4.\n", "\n", - "### ID Contributing Properties\n", - "So, what are ID contributing properties? \n", + "### ID-Contributing Properties\n", + "So, what are ID-contributing properties? \n", "Every SCO has multiple defined properties, but the values of only some of those properties will contribute to the determination of the SCO's ID. \n", "\n", - "A SCO's ID contributing properties may contain a combination of required properties and optional properties. And it is possible for all of the ID contributing properties to be optional, which means the corresponding SCO could be created without values for any of those properties, which would lead the SCO to have a randomly-generated UUIDv4 ID.\n", + "A SCO's ID-contributing properties may contain a combination of required properties and optional properties. And it is possible for all of the ID-contributing properties to be optional, which means the corresponding SCO could be created without values for any of those properties, which would lead the SCO to have a randomly-generated UUIDv4 ID.\n", "\n", "\n", "We will demonstrate deterministic IDs by creating four `EmailAddress` SCOs. The `EmailAddress` SCO has one ID contributing property: `value`." @@ -1856,9 +1856,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Notice that the ID for the first three `EmailAddress` objects is the same while the ID for the fourth is different. This is because the first three objects all have the same value for the ID contributing property. And this is despite having a different value for the `display_name` property, since it is not an ID contributing property for the `EmailAddress` SCO.\n", + "Notice that the ID for the first three `EmailAddress` objects is the same while the ID for the fourth is different. This is because the first three objects all have the same value for the ID-contributing property. And this is despite having a different value for the `display_name` property, since it is not an ID-contributing property for the `EmailAddress` SCO.\n", "\n", - "Also note that the fourth object has a different ID despite having the same `display_name` as the first two objects; the value for the fourth object's ID contributing property is different." + "Also note that the fourth object has a different ID despite having the same `display_name` as the first two objects; the value for the fourth object's ID-contributing property is different." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ID-Contributing Properties for Custom Cyber Observables\n", + "While all custom STIX 2.1 Cyber Observables (SCOs) have randomly-generated UUIDv4 IDs by default, you can optionally define which, if any, of your custom SCO's properties should be ID-contributing properties. Similar to standard SCOs, your custom SCO's ID-contributing properties can be any combination of the SCO's required and optional properties.\n", + "\n", + "You define the ID-contributing properties right when creating your custom SCO. See the example below: (taken from `stix2/test/v21/test_custom.py` `test_custom_observable_object_det_id_1()` )" ] }, { From 5df319bcbb2279dbda518401eac9a3840efb0ecd Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 5 Mar 2020 13:33:45 -0500 Subject: [PATCH 14/17] Add documentation about id-contributing properties for 2.1 custom SCOs --- docs/guide/custom.ipynb | 320 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 309 insertions(+), 11 deletions(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 0609139..d758550 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -322,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -403,7 +403,7 @@ "" ] }, - "execution_count": 6, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -544,7 +544,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -846,7 +846,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -931,7 +931,7 @@ "" ] }, - "execution_count": 12, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -1504,7 +1504,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -1585,7 +1585,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, @@ -1667,7 +1667,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, @@ -1749,7 +1749,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, @@ -1831,7 +1831,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -1868,7 +1868,305 @@ "### ID-Contributing Properties for Custom Cyber Observables\n", "While all custom STIX 2.1 Cyber Observables (SCOs) have randomly-generated UUIDv4 IDs by default, you can optionally define which, if any, of your custom SCO's properties should be ID-contributing properties. Similar to standard SCOs, your custom SCO's ID-contributing properties can be any combination of the SCO's required and optional properties.\n", "\n", - "You define the ID-contributing properties right when creating your custom SCO. See the example below: (taken from `stix2/test/v21/test_custom.py` `test_custom_observable_object_det_id_1()` )" + "You define the ID-contributing properties right when creating the custom SCO; after the list of properties, you can optionally define the list of id-contributing properties. If you do not want to specify any id-contributing properties for your custom SCO, then you do not need to do anything additional. \n", + "\n", + "See the example below:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 2000\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--1e56f9c3-a73b-5fbd-b348-83c76523c4df",\n",
+       "    "a_property": "A different property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from stix2.v21 import CustomObservable # IDs and Deterministic IDs are NOT part of STIX 2.0 Custom Observables\n", + "\n", + "@CustomObservable('x-new-observable-2', [\n", + " ('a_property', properties.StringProperty(required=True)),\n", + " ('property_2', properties.IntegerProperty()),\n", + "], [\n", + " 'a_property'\n", + "])\n", + "class NewObservable2():\n", + " pass\n", + "\n", + "new_observable_a = NewObservable2(a_property=\"A property\", property_2=2000)\n", + "print(new_observable_a)\n", + "\n", + "new_observable_b = NewObservable2(a_property=\"A property\", property_2=3000)\n", + "print(new_observable_b)\n", + "\n", + "new_observable_c = NewObservable2(a_property=\"A different property\", property_2=3000)\n", + "print(new_observable_c)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice how the ID for `new_observable_a` and `new_observable_b` is the same since they have the same value for the id-contributing `a_property` property." ] }, { From d260aa8ebdfc22e579b13810c28b67eb40f53426 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 5 Mar 2020 00:08:20 -0500 Subject: [PATCH 15/17] Tweak custom SCO ID-generating properties docs --- docs/guide/custom.ipynb | 1064 +++++++++++++-------------------------- 1 file changed, 339 insertions(+), 725 deletions(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index d758550..8185269 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -175,9 +175,9 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--e7fd0fe0-25ff-4fcb-abe5-b6254a9d1a22",\n",
-       "    "created": "2019-07-25T18:18:18.241Z",\n",
-       "    "modified": "2019-07-25T18:18:18.241Z",\n",
+       "    "id": "identity--d6996982-5fb7-4364-b716-b618516989b6",\n",
+       "    "created": "2020-03-05T05:06:27.349Z",\n",
+       "    "modified": "2020-03-05T05:06:27.349Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -287,9 +287,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "identity",\n",
-       "    "id": "identity--033b5f42-c94f-488f-8efa-2b6a167f0d6f",\n",
-       "    "created": "2019-07-25T18:18:21.352Z",\n",
-       "    "modified": "2019-07-25T18:18:21.352Z",\n",
+       "    "id": "identity--a167d2de-9fc4-4734-a1ae-57a548aad22a",\n",
+       "    "created": "2020-03-05T05:06:29.180Z",\n",
+       "    "modified": "2020-03-05T05:06:29.180Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual",\n",
        "    "x_foo": "bar"\n",
@@ -322,7 +322,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -403,7 +403,7 @@
        ""
       ]
      },
-     "execution_count": 3,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -511,7 +511,7 @@
        "    "type": "identity",\n",
        "    "id": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c",\n",
        "    "created": "2015-12-21T19:59:11.000Z",\n",
-       "    "modified": "2019-07-25T18:18:25.927Z",\n",
+       "    "modified": "2020-03-05T05:06:32.934Z",\n",
        "    "name": "John Smith",\n",
        "    "identity_class": "individual"\n",
        "}\n",
@@ -544,7 +544,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -569,7 +569,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -645,9 +645,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "x-animal",\n",
-       "    "id": "x-animal--b1e4fe7f-7985-451d-855c-6ba5c265b22a",\n",
-       "    "created": "2018-04-05T18:38:19.790Z",\n",
-       "    "modified": "2018-04-05T18:38:19.790Z",\n",
+       "    "id": "x-animal--1f7ce0ad-fd3a-4cf0-9cd7-13f7bef9ecd4",\n",
+       "    "created": "2020-03-05T05:06:38.010Z",\n",
+       "    "modified": "2020-03-05T05:06:38.010Z",\n",
        "    "species": "lion",\n",
        "    "animal_class": "mammal"\n",
        "}\n",
@@ -657,7 +657,7 @@
        ""
       ]
      },
-     "execution_count": 8,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -677,7 +677,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -703,7 +703,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -784,7 +784,7 @@
        ""
       ]
      },
-     "execution_count": 10,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -811,7 +811,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -846,7 +846,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -931,7 +931,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -962,7 +962,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -1043,7 +1043,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1125,7 +1125,7 @@
        ""
       ]
      },
-     "execution_count": 13,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1155,6 +1155,316 @@
     "print(obs_data.objects[\"0\"].property_2)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### ID-Contributing Properties for Custom Cyber Observables\n",
+    "STIX 2.1 Cyber Observables (SCOs) have deterministic IDs, meaning that the ID of a SCO is based on the values of some of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID-contributing properties, then these SCOs will have the same ID. UUIDv5 is used for the deterministic IDs, using the namespace `\"00abedb4-aa42-466c-9c01-fed23315a9b7\"`. A SCO's ID-contributing properties may consist of a combination of required properties and optional properties.\n",
+    "\n",
+    "If a SCO type does not have any ID contributing properties defined, or all of the ID-contributing properties are not present on the object, then the SCO uses a randomly-generated UUIDv4. Thus, you can optionally define which of your custom SCO's properties should be ID-contributing properties. Similar to standard SCOs, your custom SCO's ID-contributing properties can be any combination of the SCO's required and optional properties.\n",
+    "\n",
+    "You define the ID-contributing properties when defining your custom SCO with the `CustomObservable` decorator. After the list of properties, you can optionally define the list of id-contributing properties. If you do not want to specify any id-contributing properties for your custom SCO, then you do not need to do anything additional.\n",
+    "\n",
+    "See the example below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 2000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
+       "    "a_property": "A property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "x-new-observable-2",\n",
+       "    "id": "x-new-observable-2--1e56f9c3-a73b-5fbd-b348-83c76523c4df",\n",
+       "    "a_property": "A different property",\n",
+       "    "property_2": 3000\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from stix2.v21 import CustomObservable # IDs and Deterministic IDs are NOT part of STIX 2.0 Custom Observables\n",
+    "\n",
+    "@CustomObservable('x-new-observable-2', [\n",
+    "    ('a_property', properties.StringProperty(required=True)),\n",
+    "    ('property_2', properties.IntegerProperty()),\n",
+    "], [\n",
+    "    'a_property'\n",
+    "])\n",
+    "class NewObservable2():\n",
+    "    pass\n",
+    "\n",
+    "new_observable_a = NewObservable2(a_property=\"A property\", property_2=2000)\n",
+    "print(new_observable_a)\n",
+    "\n",
+    "new_observable_b = NewObservable2(a_property=\"A property\", property_2=3000)\n",
+    "print(new_observable_b)\n",
+    "\n",
+    "new_observable_c = NewObservable2(a_property=\"A different property\", property_2=3000)\n",
+    "print(new_observable_c)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this example, `a_property` is the only id-contributing property. Notice that the ID for `new_observable_a` and `new_observable_b` is the same since they have the same value for the id-contributing `a_property` property."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1479,721 +1789,25 @@
     "print(obs_data2.objects[\"0\"].extensions[\"x-new-ext\"].property1)\n",
     "print(obs_data2.objects[\"0\"].extensions[\"x-new-ext\"].property2)"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Deterministic IDs for Cyber Observables\n",
-    "### Deterministic IDs\n",
-    "STIX 2.1 Cyber Observables (SCOs) have an ID property since SCOs are now top-level objects. However, SCOs have deterministic IDs, meaning that the ID of a SCO is based on the values of its properties. Thus, if multiple cyber observables of the same type have the same values for their ID-contributing properties (explained below), then these SCOs will have the same ID; the SCOs' ID is deterministic because the ID will not change if the values of the ID contributing properties do not change. \n",
-    "\n",
-    "A UUIDv5 is generated for the deterministic IDs, using the namespace `\"00abedb4-aa42-466c-9c01-fed23315a9b7\"`.\n",
-    "\n",
-    "In the case where a SCO does not have any defined ID contributing properties, or in the case where all of the values for the ID-contributing properties are not specified, then the SCO will be assigned a randomly-generated UUIDv4.\n",
-    "\n",
-    "### ID-Contributing Properties\n",
-    "So, what are ID-contributing properties? \n",
-    "Every SCO has multiple defined properties, but the values of only some of those properties will contribute to the determination of the SCO's ID. \n",
-    "\n",
-    "A SCO's ID-contributing properties may contain a combination of required properties and optional properties. And it is possible for all of the ID-contributing properties to be optional, which means the corresponding SCO could be created without values for any of those properties, which would lead the SCO to have a randomly-generated UUIDv4 ID.\n",
-    "\n",
-    "\n",
-    "We will demonstrate deterministic IDs by creating four `EmailAddress` SCOs. The `EmailAddress` SCO has one ID contributing property: `value`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "text/html": [
-       "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "text/html": [
-       "
email-addr--8fab0ad0-03c0-5cba-bf7b-c2bd41fb73a7\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "text/html": [
-       "
email-addr--d3742ef4-9452-5935-bc42-e8c35a119757\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from stix2.v21 import EmailAddress\n",
-    "\n",
-    "email_addr_1 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnny Doe\")\n",
-    "print (email_addr_1.id)\n",
-    "\n",
-    "email_addr_2 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnny Doe\")\n",
-    "print (email_addr_2.id)\n",
-    "\n",
-    "email_addr_3 = EmailAddress(value=\"johnnydoe@matching.com\", display_name=\"Johnathon Doe\")\n",
-    "print (email_addr_3.id)\n",
-    "\n",
-    "email_addr_4 = EmailAddress(value=\"johnnydoe@notmatching.com\", display_name=\"Johnny Doe\")\n",
-    "print (email_addr_4.id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Notice that the ID for the first three `EmailAddress` objects is the same while the ID for the fourth is different. This is because the first three objects all have the same value for the ID-contributing property. And this is despite having a different value for the `display_name` property, since it is not an ID-contributing property for the `EmailAddress` SCO.\n",
-    "\n",
-    "Also note that the fourth object has a different ID despite having the same `display_name` as the first two objects; the value for the fourth object's ID-contributing property is different."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### ID-Contributing Properties for Custom Cyber Observables\n",
-    "While all custom STIX 2.1 Cyber Observables (SCOs) have randomly-generated UUIDv4 IDs by default, you can optionally define which, if any, of your custom SCO's properties should be ID-contributing properties. Similar to standard SCOs, your custom SCO's ID-contributing properties can be any combination of the SCO's required and optional properties.\n",
-    "\n",
-    "You define the ID-contributing properties right when creating the custom SCO; after the list of properties, you can optionally define the list of id-contributing properties. If you do not want to specify any id-contributing properties for your custom SCO, then you do not need to do anything additional. \n",
-    "\n",
-    "See the example below:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
-       "    "a_property": "A property",\n",
-       "    "property_2": 2000\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--6bc655d6-dcb8-52a3-a862-46848c17e599",\n",
-       "    "a_property": "A property",\n",
-       "    "property_2": 3000\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "text/html": [
-       "
{\n",
-       "    "type": "x-new-observable-2",\n",
-       "    "id": "x-new-observable-2--1e56f9c3-a73b-5fbd-b348-83c76523c4df",\n",
-       "    "a_property": "A different property",\n",
-       "    "property_2": 3000\n",
-       "}\n",
-       "
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from stix2.v21 import CustomObservable # IDs and Deterministic IDs are NOT part of STIX 2.0 Custom Observables\n",
-    "\n",
-    "@CustomObservable('x-new-observable-2', [\n",
-    "    ('a_property', properties.StringProperty(required=True)),\n",
-    "    ('property_2', properties.IntegerProperty()),\n",
-    "], [\n",
-    "    'a_property'\n",
-    "])\n",
-    "class NewObservable2():\n",
-    "    pass\n",
-    "\n",
-    "new_observable_a = NewObservable2(a_property=\"A property\", property_2=2000)\n",
-    "print(new_observable_a)\n",
-    "\n",
-    "new_observable_b = NewObservable2(a_property=\"A property\", property_2=3000)\n",
-    "print(new_observable_b)\n",
-    "\n",
-    "new_observable_c = NewObservable2(a_property=\"A different property\", property_2=3000)\n",
-    "print(new_observable_c)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Notice how the ID for `new_observable_a` and `new_observable_b` is the same since they have the same value for the id-contributing `a_property` property."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15+"
   }
  },
  "nbformat": 4,

From 013e8ece4c3631f5559bac7692e8f5b336f11ee1 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 6 Mar 2020 09:48:03 -0500
Subject: [PATCH 16/17] Update CHANGELOG for v1.3.1

---
 CHANGELOG | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index b764735..dc0d91e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,19 @@
 CHANGELOG
 =========
 
+1.3.1 - 2020-03-06
+
+* #322 Adds encoding option FileSystemSource and MemorySource
+* #354 Adds ability to specify id-contributing properties on custom SCOs
+* #346 Certain SCO properties are no longer deprecated
+* #327 Fixes missing 'name' property on Marking Definitions
+* #303 Fixes bug with escaping quotes in patterns
+* #331 Fixes crashing bug of property names that conflict with Mapping methods
+* #337 Fixes bug with detecting STIX version of content when parsing
+* #342, #343 Fixes bug when adding SCOs to Memory or FileSystem Stores
+* #348 Fixes bug with generating deterministic IDs for SCOs
+* #344 Fixes bug with propagating errors from the pattern validator
+
 1.3.0 - 2020-01-04
 
 * #305 Updates support of STIX 2.1 to WD06

From 380926cff59f9e0d70914c2de933e4651f13259d Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 6 Mar 2020 09:50:09 -0500
Subject: [PATCH 17/17] =?UTF-8?q?Bump=20version:=201.3.0=20=E2=86=92=201.3?=
 =?UTF-8?q?.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 659a1cd..7e89c66 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.3.1
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index 67bc602..9c73af2 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "1.3.0"
+__version__ = "1.3.1"