From 19707677c9a99db3e4cf9b88679311917988a77f Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 7 Feb 2020 18:17:12 -0500 Subject: [PATCH 1/9] Fix STIX version detection from dicts. In particular, 2.1 SCOs without the spec_version property ought to be correctly detected as 2.1 now. --- stix2/core.py | 65 +++++--- stix2/test/test_spec_version_detect.py | 210 +++++++++++++++++++++++++ 2 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 stix2/test/test_spec_version_detect.py diff --git a/stix2/core.py b/stix2/core.py index aa9044d..ae9e1e4 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -58,6 +58,47 @@ def parse(data, allow_custom=False, version=None): return obj +def _detect_spec_version(stix_dict): + """ + Given a dict representing a STIX object, try to detect what spec version + it is likely to comply with. + + :param stix_dict: A dict with some STIX content. Must at least have a + "type" property. + :return: A string in "vXX" format, where "XX" indicates the spec version, + e.g. "v20", "v21", etc. + """ + + obj_type = stix_dict["type"] + + if 'spec_version' in stix_dict: + # For STIX 2.0, applies to bundles only. + # For STIX 2.1+, applies to SCOs, SDOs, SROs, and markings only. + v = 'v' + stix_dict['spec_version'].replace('.', '') + elif "id" not in stix_dict: + # Only 2.0 SCOs don't have ID properties + v = "v20" + elif obj_type == 'bundle': + # Bundle without a spec_version property: must be 2.1. But to + # future-proof, use max version over all contained SCOs, with 2.1 + # minimum. + v = max( + "v21", + max( + _detect_spec_version(obj) for obj in stix_dict["objects"] + ) + ) + elif obj_type in STIX2_OBJ_MAPS["v21"]["observables"]: + # Non-bundle object with an ID and without spec_version. Could be a + # 2.1 SCO or 2.0 SDO/SRO/marking. Check for 2.1 SCO... + v = "v21" + else: + # Not a 2.1 SCO; must be a 2.0 object. + v = "v20" + + return v + + def dict_to_stix2(stix_dict, allow_custom=False, version=None): """convert dictionary to full python-stix2 object @@ -93,21 +134,8 @@ def dict_to_stix2(stix_dict, allow_custom=False, version=None): if version: # If the version argument was passed, override other approaches. v = 'v' + version.replace('.', '') - elif 'spec_version' in stix_dict: - # For STIX 2.0, applies to bundles only. - # For STIX 2.1+, applies to SDOs, SROs, and markings only. - v = 'v' + stix_dict['spec_version'].replace('.', '') - elif stix_dict['type'] == 'bundle': - # bundles without spec_version are ambiguous. - if any('spec_version' in x for x in stix_dict['objects']): - # Only on 2.1 we are allowed to have 'spec_version' in SDOs/SROs. - v = 'v21' - else: - v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') else: - # The spec says that SDO/SROs without spec_version will default to a - # '2.0' representation. - v = 'v20' + v = _detect_spec_version(stix_dict) OBJ_MAP = dict(STIX2_OBJ_MAPS[v]['objects'], **STIX2_OBJ_MAPS[v]['observables']) @@ -142,6 +170,10 @@ def parse_observable(data, _valid_refs=None, allow_custom=False, version=None): """ obj = _get_dict(data) + + if 'type' not in obj: + raise ParseError("Can't parse observable with no 'type' property: %s" % str(obj)) + # get deep copy since we are going modify the dict and might # modify the original dict as _get_dict() does not return new # dict when passed a dict @@ -153,11 +185,8 @@ def parse_observable(data, _valid_refs=None, allow_custom=False, version=None): # If the version argument was passed, override other approaches. v = 'v' + version.replace('.', '') else: - # Use default version (latest) if no version was provided. - v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') + v = _detect_spec_version(obj) - if 'type' not in obj: - raise ParseError("Can't parse observable with no 'type' property: %s" % str(obj)) try: OBJ_MAP_OBSERVABLE = STIX2_OBJ_MAPS[v]['observables'] obj_class = OBJ_MAP_OBSERVABLE[obj['type']] diff --git a/stix2/test/test_spec_version_detect.py b/stix2/test/test_spec_version_detect.py new file mode 100644 index 0000000..a5d3ee5 --- /dev/null +++ b/stix2/test/test_spec_version_detect.py @@ -0,0 +1,210 @@ +from __future__ import unicode_literals + +import pytest + +from stix2.core import _detect_spec_version + + +@pytest.mark.parametrize("obj_dict, expected_ver", [ + # STIX 2.0 examples + ( + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual" + }, + "v20" + ), + ( + { + "type": "relationship", + "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", + "created": "1981-08-11T13:48:19.000Z", + "modified": "2000-02-16T15:33:15.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets" + }, + "v20" + ), + ( + { + "type": "file", + "name": "notes.txt" + }, + "v20" + ), + ( + { + "type": "marking-definition", + "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", + "created": "1998-03-27T19:44:53.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (c) ACME Corp." + } + }, + "v20" + ), + ( + { + "type": "bundle", + "id": "bundle--8379cb02-8131-47c8-8a7c-9a1f0e0986b1", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual" + }, + { + "type": "marking-definition", + "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", + "created": "1998-03-27T19:44:53.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (c) ACME Corp." + } + } + ] + }, + "v20" + ), + # STIX 2.1 examples + ( + { + "type": "identity", + "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", + "spec_version": "2.1", + "created": "1995-07-24T04:07:48.000Z", + "modified": "2001-07-01T09:33:17.000Z", + "name": "alice" + }, + "v21" + ), + ( + { + "type": "relationship", + "id": "relationship--0eec232d-e1ea-4f85-8e78-0de6ae9d09f0", + "spec_version": "2.1", + "created": "1975-04-05T10:47:22.000Z", + "modified": "1983-04-25T20:56:00.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets" + }, + "v21" + ), + ( + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "spec_version": "2.1", + "name": "notes.txt" + }, + "v21" + ), + ( + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt" + }, + "v21" + ), + ( + { + "type": "marking-definition", + "spec_version": "2.1", + "id": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + "created": "2017-01-20T00:00:00.000Z", + "definition_type": "tlp", + "name": "TLP:GREEN", + "definition": { + "tlp": "green" + } + }, + "v21" + ), + ( + { + "type": "bundle", + "id": "bundle--d5787acd-1ffd-4630-ada3-6857698f6287", + "objects": [ + { + "type": "identity", + "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", + "spec_version": "2.1", + "created": "1995-07-24T04:07:48.000Z", + "modified": "2001-07-01T09:33:17.000Z", + "name": "alice" + }, + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt" + }, + ] + }, + "v21" + ), + # Mixed spec examples + ( + { + "type": "bundle", + "id": "bundle--e1a01e29-3432-401a-ab9f-c1082b056605", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual" + }, + { + "type": "relationship", + "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", + "created": "1981-08-11T13:48:19.000Z", + "modified": "2000-02-16T15:33:15.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets" + }, + ] + }, + "v21" + ), + ( + { + "type": "bundle", + "id": "bundle--eecad3d9-bb9a-4263-93f6-1c0ccc984574", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual" + }, + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt" + }, + ] + }, + "v21" + ) +]) +def test_spec_version_detect(obj_dict, expected_ver): + detected_ver = _detect_spec_version(obj_dict) + + assert detected_ver == expected_ver From bf83ca62b39e379d05806b4d83dcd4647fc5b0a7 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 7 Feb 2020 18:58:45 -0500 Subject: [PATCH 2/9] Add trailing commas for the pre-commit hook...... --- stix2/core.py | 2 +- stix2/test/test_spec_version_detect.py | 396 +++++++++++++------------ 2 files changed, 200 insertions(+), 198 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index ae9e1e4..0d1fee5 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -86,7 +86,7 @@ def _detect_spec_version(stix_dict): "v21", max( _detect_spec_version(obj) for obj in stix_dict["objects"] - ) + ), ) elif obj_type in STIX2_OBJ_MAPS["v21"]["observables"]: # Non-bundle object with an ID and without spec_version. Could be a diff --git a/stix2/test/test_spec_version_detect.py b/stix2/test/test_spec_version_detect.py index a5d3ee5..bd0e80c 100644 --- a/stix2/test/test_spec_version_detect.py +++ b/stix2/test/test_spec_version_detect.py @@ -5,205 +5,207 @@ import pytest from stix2.core import _detect_spec_version -@pytest.mark.parametrize("obj_dict, expected_ver", [ - # STIX 2.0 examples - ( - { - "type": "identity", - "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", - "created": "1972-05-21T05:33:09.000Z", - "modified": "1973-05-28T02:10:54.000Z", - "name": "alice", - "identity_class": "individual" - }, - "v20" - ), - ( - { - "type": "relationship", - "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", - "created": "1981-08-11T13:48:19.000Z", - "modified": "2000-02-16T15:33:15.000Z", - "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", - "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", - "relationship_type": "targets" - }, - "v20" - ), - ( - { - "type": "file", - "name": "notes.txt" - }, - "v20" - ), - ( - { - "type": "marking-definition", - "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", - "created": "1998-03-27T19:44:53.000Z", - "definition_type": "statement", - "definition": { - "statement": "Copyright (c) ACME Corp." - } - }, - "v20" - ), - ( - { - "type": "bundle", - "id": "bundle--8379cb02-8131-47c8-8a7c-9a1f0e0986b1", - "spec_version": "2.0", - "objects": [ - { - "type": "identity", - "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", - "created": "1972-05-21T05:33:09.000Z", - "modified": "1973-05-28T02:10:54.000Z", - "name": "alice", - "identity_class": "individual" +@pytest.mark.parametrize( + "obj_dict, expected_ver", [ + # STIX 2.0 examples + ( + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual", + }, + "v20" + ), + ( + { + "type": "relationship", + "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", + "created": "1981-08-11T13:48:19.000Z", + "modified": "2000-02-16T15:33:15.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets", + }, + "v20" + ), + ( + { + "type": "file", + "name": "notes.txt", + }, + "v20" + ), + ( + { + "type": "marking-definition", + "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", + "created": "1998-03-27T19:44:53.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (c) ACME Corp.", }, - { - "type": "marking-definition", - "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", - "created": "1998-03-27T19:44:53.000Z", - "definition_type": "statement", - "definition": { - "statement": "Copyright (c) ACME Corp." - } - } - ] - }, - "v20" - ), - # STIX 2.1 examples - ( - { - "type": "identity", - "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", - "spec_version": "2.1", - "created": "1995-07-24T04:07:48.000Z", - "modified": "2001-07-01T09:33:17.000Z", - "name": "alice" - }, - "v21" - ), - ( - { - "type": "relationship", - "id": "relationship--0eec232d-e1ea-4f85-8e78-0de6ae9d09f0", - "spec_version": "2.1", - "created": "1975-04-05T10:47:22.000Z", - "modified": "1983-04-25T20:56:00.000Z", - "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", - "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", - "relationship_type": "targets" - }, - "v21" - ), - ( - { - "type": "file", - "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", - "spec_version": "2.1", - "name": "notes.txt" - }, - "v21" - ), - ( - { - "type": "file", - "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", - "name": "notes.txt" - }, - "v21" - ), - ( - { - "type": "marking-definition", - "spec_version": "2.1", - "id": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", - "created": "2017-01-20T00:00:00.000Z", - "definition_type": "tlp", - "name": "TLP:GREEN", - "definition": { - "tlp": "green" - } - }, - "v21" - ), - ( - { - "type": "bundle", - "id": "bundle--d5787acd-1ffd-4630-ada3-6857698f6287", - "objects": [ - { - "type": "identity", - "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", - "spec_version": "2.1", - "created": "1995-07-24T04:07:48.000Z", - "modified": "2001-07-01T09:33:17.000Z", - "name": "alice" + }, + "v20" + ), + ( + { + "type": "bundle", + "id": "bundle--8379cb02-8131-47c8-8a7c-9a1f0e0986b1", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual", + }, + { + "type": "marking-definition", + "id": "marking-definition--2a13090f-a493-4b70-85fe-fa021d91dcd2", + "created": "1998-03-27T19:44:53.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (c) ACME Corp.", + }, + }, + ], + }, + "v20" + ), + # STIX 2.1 examples + ( + { + "type": "identity", + "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", + "spec_version": "2.1", + "created": "1995-07-24T04:07:48.000Z", + "modified": "2001-07-01T09:33:17.000Z", + "name": "alice", + }, + "v21" + ), + ( + { + "type": "relationship", + "id": "relationship--0eec232d-e1ea-4f85-8e78-0de6ae9d09f0", + "spec_version": "2.1", + "created": "1975-04-05T10:47:22.000Z", + "modified": "1983-04-25T20:56:00.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets", + }, + "v21" + ), + ( + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "spec_version": "2.1", + "name": "notes.txt", + }, + "v21" + ), + ( + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt", + }, + "v21" + ), + ( + { + "type": "marking-definition", + "spec_version": "2.1", + "id": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + "created": "2017-01-20T00:00:00.000Z", + "definition_type": "tlp", + "name": "TLP:GREEN", + "definition": { + "tlp": "green", }, - { - "type": "file", - "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", - "name": "notes.txt" - }, - ] - }, - "v21" - ), - # Mixed spec examples - ( - { - "type": "bundle", - "id": "bundle--e1a01e29-3432-401a-ab9f-c1082b056605", - "objects": [ - { - "type": "identity", - "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", - "created": "1972-05-21T05:33:09.000Z", - "modified": "1973-05-28T02:10:54.000Z", - "name": "alice", - "identity_class": "individual" - }, - { - "type": "relationship", - "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", - "created": "1981-08-11T13:48:19.000Z", - "modified": "2000-02-16T15:33:15.000Z", - "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", - "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", - "relationship_type": "targets" - }, - ] - }, - "v21" - ), - ( - { - "type": "bundle", - "id": "bundle--eecad3d9-bb9a-4263-93f6-1c0ccc984574", - "objects": [ - { - "type": "identity", - "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", - "created": "1972-05-21T05:33:09.000Z", - "modified": "1973-05-28T02:10:54.000Z", - "name": "alice", - "identity_class": "individual" - }, - { - "type": "file", - "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", - "name": "notes.txt" - }, - ] - }, - "v21" - ) -]) + }, + "v21" + ), + ( + { + "type": "bundle", + "id": "bundle--d5787acd-1ffd-4630-ada3-6857698f6287", + "objects": [ + { + "type": "identity", + "id": "identity--22299b4c-bc38-4485-ad7d-8222f01c58c7", + "spec_version": "2.1", + "created": "1995-07-24T04:07:48.000Z", + "modified": "2001-07-01T09:33:17.000Z", + "name": "alice", + }, + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt", + }, + ], + }, + "v21" + ), + # Mixed spec examples + ( + { + "type": "bundle", + "id": "bundle--e1a01e29-3432-401a-ab9f-c1082b056605", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual", + }, + { + "type": "relationship", + "id": "relationship--63b0f1b7-925e-4795-ac9b-61fb9f235f1a", + "created": "1981-08-11T13:48:19.000Z", + "modified": "2000-02-16T15:33:15.000Z", + "source_ref": "attack-pattern--9391504a-ef29-4a41-a257-5634d9edc391", + "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", + "relationship_type": "targets", + }, + ], + }, + "v21" + ), + ( + { + "type": "bundle", + "id": "bundle--eecad3d9-bb9a-4263-93f6-1c0ccc984574", + "objects": [ + { + "type": "identity", + "id": "identity--d7f72e8d-657a-43ec-9324-b3ec67a97486", + "created": "1972-05-21T05:33:09.000Z", + "modified": "1973-05-28T02:10:54.000Z", + "name": "alice", + "identity_class": "individual", + }, + { + "type": "file", + "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", + "name": "notes.txt", + }, + ], + }, + "v21" + ), + ], +) def test_spec_version_detect(obj_dict, expected_ver): detected_ver = _detect_spec_version(obj_dict) From f86b6e8a662903b087596fc21344f77b71f477bc Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 7 Feb 2020 19:15:59 -0500 Subject: [PATCH 3/9] More add-trailing-comma junk, which is not done by the python 3.8 pre-commit add-trailing-comma library. Hopefully this satisfies the travis tests for other versions of python! --- stix2/test/test_spec_version_detect.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/stix2/test/test_spec_version_detect.py b/stix2/test/test_spec_version_detect.py index bd0e80c..d2a9da8 100644 --- a/stix2/test/test_spec_version_detect.py +++ b/stix2/test/test_spec_version_detect.py @@ -17,7 +17,7 @@ from stix2.core import _detect_spec_version "name": "alice", "identity_class": "individual", }, - "v20" + "v20", ), ( { @@ -29,14 +29,14 @@ from stix2.core import _detect_spec_version "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", "relationship_type": "targets", }, - "v20" + "v20", ), ( { "type": "file", "name": "notes.txt", }, - "v20" + "v20", ), ( { @@ -48,7 +48,7 @@ from stix2.core import _detect_spec_version "statement": "Copyright (c) ACME Corp.", }, }, - "v20" + "v20", ), ( { @@ -75,7 +75,7 @@ from stix2.core import _detect_spec_version }, ], }, - "v20" + "v20", ), # STIX 2.1 examples ( @@ -87,7 +87,7 @@ from stix2.core import _detect_spec_version "modified": "2001-07-01T09:33:17.000Z", "name": "alice", }, - "v21" + "v21", ), ( { @@ -100,7 +100,7 @@ from stix2.core import _detect_spec_version "target_ref": "identity--ba18dde2-56d3-4a34-aa0b-fc56f5be568f", "relationship_type": "targets", }, - "v21" + "v21", ), ( { @@ -109,7 +109,7 @@ from stix2.core import _detect_spec_version "spec_version": "2.1", "name": "notes.txt", }, - "v21" + "v21", ), ( { @@ -117,7 +117,7 @@ from stix2.core import _detect_spec_version "id": "file--5eef3404-6a94-4db3-9a1a-5684cbea0dfe", "name": "notes.txt", }, - "v21" + "v21", ), ( { @@ -131,7 +131,7 @@ from stix2.core import _detect_spec_version "tlp": "green", }, }, - "v21" + "v21", ), ( { @@ -153,7 +153,7 @@ from stix2.core import _detect_spec_version }, ], }, - "v21" + "v21", ), # Mixed spec examples ( @@ -180,7 +180,7 @@ from stix2.core import _detect_spec_version }, ], }, - "v21" + "v21", ), ( { @@ -202,7 +202,7 @@ from stix2.core import _detect_spec_version }, ], }, - "v21" + "v21", ), ], ) From 98a654884dc61024180fd238ef558285a40b0cf5 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 13 Feb 2020 17:11:58 -0500 Subject: [PATCH 4/9] Fix the memory store to support the new top-level 2.1 SCOs. --- stix2/datastore/memory.py | 42 +++++++++++-------------- stix2/test/v20/test_datastore_memory.py | 21 +++++++++++++ stix2/test/v21/test_datastore_memory.py | 35 +++++++++++++++++++++ 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index 52bf4c8..52da168 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -10,7 +10,6 @@ from stix2.base import _STIXBase from stix2.core import parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import FilterSet, apply_common_filters -from stix2.utils import is_marking def _add(store, stix_data, allow_custom=True, version=None): @@ -47,12 +46,10 @@ def _add(store, stix_data, allow_custom=True, version=None): else: stix_obj = parse(stix_data, allow_custom, version) - # Map ID directly to the object, if it is a marking. Otherwise, - # map to a family, so we can track multiple versions. - if is_marking(stix_obj): - store._data[stix_obj["id"]] = stix_obj - - else: + # Map ID to a _ObjectFamily if the object is versioned, so we can track + # multiple versions. Otherwise, map directly to the object. All + # versioned objects should have a "modified" property. + if "modified" in stix_obj: if stix_obj["id"] in store._data: obj_family = store._data[stix_obj["id"]] else: @@ -61,6 +58,9 @@ def _add(store, stix_data, allow_custom=True, version=None): obj_family.add(stix_obj) + else: + store._data[stix_obj["id"]] = stix_obj + class _ObjectFamily(object): """ @@ -267,12 +267,12 @@ class MemorySource(DataSource): """ stix_obj = None - if is_marking(stix_id): - stix_obj = self._data.get(stix_id) - else: - object_family = self._data.get(stix_id) - if object_family: - stix_obj = object_family.latest_version + mapped_value = self._data.get(stix_id) + if mapped_value: + if isinstance(mapped_value, _ObjectFamily): + stix_obj = mapped_value.latest_version + else: + stix_obj = mapped_value if stix_obj: all_filters = list( @@ -300,17 +300,13 @@ class MemorySource(DataSource): """ results = [] - stix_objs_to_filter = None - if is_marking(stix_id): - stix_obj = self._data.get(stix_id) - if stix_obj: - stix_objs_to_filter = [stix_obj] - else: - object_family = self._data.get(stix_id) - if object_family: - stix_objs_to_filter = object_family.all_versions.values() + mapped_value = self._data.get(stix_id) + if mapped_value: + if isinstance(mapped_value, _ObjectFamily): + stix_objs_to_filter = mapped_value.all_versions.values() + else: + stix_objs_to_filter = [mapped_value] - if stix_objs_to_filter: all_filters = list( itertools.chain( _composite_filters or [], diff --git a/stix2/test/v20/test_datastore_memory.py b/stix2/test/v20/test_datastore_memory.py index fba96dd..c86f94d 100644 --- a/stix2/test/v20/test_datastore_memory.py +++ b/stix2/test/v20/test_datastore_memory.py @@ -423,3 +423,24 @@ def test_object_family_internal_components(mem_source): assert "latest=2017-01-27 13:49:53.936000+00:00>>" in str_representation assert "latest=2017-01-27 13:49:53.936000+00:00>>" in repr_representation + + +def test_unversioned_objects(mem_store): + marking = { + "type": "marking-definition", + "id": "marking-definition--48e83cde-e902-4404-85b3-6e81f75ccb62", + "created": "1988-01-02T16:44:04.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (C) ACME Corp." + } + } + + mem_store.add(marking) + + obj = mem_store.get(marking["id"]) + assert obj["id"] == marking["id"] + + objs = mem_store.all_versions(marking["id"]) + assert len(objs) == 1 + assert objs[0]["id"] == marking["id"] diff --git a/stix2/test/v21/test_datastore_memory.py b/stix2/test/v21/test_datastore_memory.py index 4f63a06..ad61431 100644 --- a/stix2/test/v21/test_datastore_memory.py +++ b/stix2/test/v21/test_datastore_memory.py @@ -438,3 +438,38 @@ def test_object_family_internal_components(mem_source): assert "latest=2017-01-27 13:49:53.936000+00:00>>" in str_representation assert "latest=2017-01-27 13:49:53.936000+00:00>>" in repr_representation + + +def test_unversioned_objects(mem_store): + marking = { + "type": "marking-definition", + "spec_version": "2.1", + "id": "marking-definition--48e83cde-e902-4404-85b3-6e81f75ccb62", + "created": "1988-01-02T16:44:04.000Z", + "definition_type": "statement", + "definition": { + "statement": "Copyright (C) ACME Corp." + } + } + + file_sco = { + "type": "file", + "id": "file--bbd59c0c-1aa4-44f1-96de-80b8325372c7", + "name": "cats.png" + } + + mem_store.add([marking, file_sco]) + + obj = mem_store.get(marking["id"]) + assert obj["id"] == marking["id"] + + obj = mem_store.get(file_sco["id"]) + assert obj["id"] == file_sco["id"] + + objs = mem_store.all_versions(marking["id"]) + assert len(objs) == 1 + assert objs[0]["id"] == marking["id"] + + objs = mem_store.all_versions(file_sco["id"]) + assert len(objs) == 1 + assert objs[0]["id"] == file_sco["id"] From be5274878d9217c9d94269a72dde206bbde93fb1 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 13 Feb 2020 17:37:59 -0500 Subject: [PATCH 5/9] Add trailing commas for pre-commit hook... --- stix2/test/v20/test_datastore_memory.py | 4 ++-- stix2/test/v21/test_datastore_memory.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stix2/test/v20/test_datastore_memory.py b/stix2/test/v20/test_datastore_memory.py index c86f94d..7852746 100644 --- a/stix2/test/v20/test_datastore_memory.py +++ b/stix2/test/v20/test_datastore_memory.py @@ -432,8 +432,8 @@ def test_unversioned_objects(mem_store): "created": "1988-01-02T16:44:04.000Z", "definition_type": "statement", "definition": { - "statement": "Copyright (C) ACME Corp." - } + "statement": "Copyright (C) ACME Corp.", + }, } mem_store.add(marking) diff --git a/stix2/test/v21/test_datastore_memory.py b/stix2/test/v21/test_datastore_memory.py index ad61431..e07943c 100644 --- a/stix2/test/v21/test_datastore_memory.py +++ b/stix2/test/v21/test_datastore_memory.py @@ -448,14 +448,14 @@ def test_unversioned_objects(mem_store): "created": "1988-01-02T16:44:04.000Z", "definition_type": "statement", "definition": { - "statement": "Copyright (C) ACME Corp." - } + "statement": "Copyright (C) ACME Corp.", + }, } file_sco = { "type": "file", "id": "file--bbd59c0c-1aa4-44f1-96de-80b8325372c7", - "name": "cats.png" + "name": "cats.png", } mem_store.add([marking, file_sco]) From 86f9e51a42673674cc9826767e44b7ecbbf4d64e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 19 Feb 2020 09:11:30 -0500 Subject: [PATCH 6/9] Fixes #336 --- stix2/test/v21/test_observed_data.py | 36 -------------------------- stix2/v21/observables.py | 38 ---------------------------- 2 files changed, 74 deletions(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 371018c..6f36d88 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1538,39 +1538,3 @@ def test_deterministic_id_no_contributing_props(): uuid_obj_2 = uuid.UUID(email_msg_2.id[-36:]) assert uuid_obj_2.variant == uuid.RFC_4122 assert uuid_obj_2.version == 4 - - -def test_ipv4_resolves_to_refs_deprecation(): - with pytest.warns(stix2.exceptions.STIXDeprecationWarning): - - stix2.v21.IPv4Address( - value="26.09.19.70", - resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], - ) - - -def test_ipv4_belongs_to_refs_deprecation(): - with pytest.warns(stix2.exceptions.STIXDeprecationWarning): - - stix2.v21.IPv4Address( - value="21.12.19.64", - belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], - ) - - -def test_ipv6_resolves_to_refs_deprecation(): - with pytest.warns(stix2.exceptions.STIXDeprecationWarning): - - stix2.v21.IPv6Address( - value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", - resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], - ) - - -def test_ipv6_belongs_to_refs_deprecation(): - with pytest.warns(stix2.exceptions.STIXDeprecationWarning): - - stix2.v21.IPv6Address( - value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", - belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], - ) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 1703263..35c1e05 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -122,14 +122,6 @@ class DomainName(_Observable): ]) _id_contributing_properties = ["value"] - def _check_object_constraints(self): - if self.get('resolves_to_refs'): - warnings.warn( - "The 'resolves_to_refs' property of domain-name is deprecated in " - "STIX 2.1. Use the 'resolves-to' relationship type instead", - STIXDeprecationWarning, - ) - class EmailAddress(_Observable): # TODO: Add link @@ -421,21 +413,6 @@ class IPv4Address(_Observable): ]) _id_contributing_properties = ["value"] - def _check_object_constraints(self): - if self.get('resolves_to_refs'): - warnings.warn( - "The 'resolves_to_refs' property of ipv4-addr is deprecated in " - "STIX 2.1. Use the 'resolves-to' relationship type instead", - STIXDeprecationWarning, - ) - - if self.get('belongs_to_refs'): - warnings.warn( - "The 'belongs_to_refs' property of ipv4-addr is deprecated in " - "STIX 2.1. Use the 'belongs-to' relationship type instead", - STIXDeprecationWarning, - ) - class IPv6Address(_Observable): # TODO: Add link @@ -458,21 +435,6 @@ class IPv6Address(_Observable): ]) _id_contributing_properties = ["value"] - def _check_object_constraints(self): - if self.get('resolves_to_refs'): - warnings.warn( - "The 'resolves_to_refs' property of ipv6-addr is deprecated in " - "STIX 2.1. Use the 'resolves-to' relationship type instead", - STIXDeprecationWarning, - ) - - if self.get('belongs_to_refs'): - warnings.warn( - "The 'belongs_to_refs' property of ipv6-addr is deprecated in " - "STIX 2.1. Use the 'belongs-to' relationship type instead", - STIXDeprecationWarning, - ) - class MACAddress(_Observable): # TODO: Add link From 8219b34ea4a4cef8cefb3d44af28375f28b6f11e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 19 Feb 2020 09:24:27 -0500 Subject: [PATCH 7/9] Fix formatting issues. Fixes #336 --- stix2/v21/observables.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 35c1e05..ed560a6 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -7,13 +7,10 @@ Observable and do not have a ``_type`` attribute. from collections import OrderedDict import itertools -import warnings from ..base import _Extension, _Observable, _STIXBase from ..custom import _custom_extension_builder, _custom_observable_builder -from ..exceptions import ( - AtLeastOnePropertyError, DependentPropertiesError, STIXDeprecationWarning, -) +from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError from ..properties import ( BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, From 1084c75d333ad817836b3f2e9bd6d3840c6865d5 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 19 Feb 2020 16:29:13 -0500 Subject: [PATCH 8/9] Fixes #334 --- stix2/base.py | 24 +++++++++++---- stix2/test/v21/test_observed_data.py | 44 ++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index a283902..22b1128 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -388,11 +388,10 @@ class _Observable(_STIXBase): temp_deep_copy = copy.deepcopy(dict(kwargs[key])) _recursive_stix_to_dict(temp_deep_copy) streamlined_obj_vals.append(temp_deep_copy) - elif isinstance(kwargs[key], list) and isinstance(kwargs[key][0], _STIXBase): - for obj in kwargs[key]: - temp_deep_copy = copy.deepcopy(dict(obj)) - _recursive_stix_to_dict(temp_deep_copy) - streamlined_obj_vals.append(temp_deep_copy) + elif isinstance(kwargs[key], list): + temp_deep_copy = copy.deepcopy(kwargs[key]) + _recursive_stix_list_to_dict(temp_deep_copy) + streamlined_obj_vals.append(temp_deep_copy) else: streamlined_obj_vals.append(kwargs[key]) @@ -448,5 +447,20 @@ def _recursive_stix_to_dict(input_dict): # There may stil be nested _STIXBase objects _recursive_stix_to_dict(input_dict[key]) + elif isinstance(input_dict[key], list): + _recursive_stix_list_to_dict(input_dict[key]) else: return + + +def _recursive_stix_list_to_dict(input_list): + for i in range(len(input_list)): + if isinstance(input_list[i], _STIXBase): + input_list[i] = dict(input_list[i]) + elif isinstance(input_list[i], dict): + pass + elif isinstance(input_list[i], list): + _recursive_stix_list_to_dict(input_list[i]) + else: + continue + _recursive_stix_to_dict(input_list[i]) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 371018c..8254594 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1574,3 +1574,47 @@ def test_ipv6_belongs_to_refs_deprecation(): value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], ) + + +def test_id_gen_recursive_dict_conversion_1(): + file_observable = stix2.v21.File( + name="example.exe", + size=68 * 1000, + magic_number_hex="50000000", + hashes={ + "SHA-256": "841a8921140aba50671ebb0770fecc4ee308c4952cfeff8de154ab14eeef4649", + }, + extensions={ + "windows-pebinary-ext": stix2.v21.WindowsPEBinaryExt( + pe_type="exe", + machine_hex="014c", + sections=[ + stix2.v21.WindowsPESection( + name=".data", + size=4096, + entropy=7.980693, + hashes={"SHA-256": "6e3b6f3978e5cd96ba7abee35c24e867b7e64072e2ecb22d0ee7a6e6af6894d0"}, + ), + ], + ), + }, + ) + + assert file_observable.id == "file--5219d93d-13c1-5f1f-896b-039f10ec67ea" + + +def test_id_gen_recursive_dict_conversion_2(): + wrko = stix2.v21.WindowsRegistryKey( + values=[ + stix2.v21.WindowsRegistryValueType( + name="Foo", + data="qwerty", + ), + stix2.v21.WindowsRegistryValueType( + name="Bar", + data="42", + ), + ], + ) + + assert wrko.id == "windows-registry-key--c087d9fe-a03e-5922-a1cd-da116e5b8a7b" From 796a4e20fa108f5cd90f5c8ac08f73246e9e4a18 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 21 Feb 2020 15:26:19 -0500 Subject: [PATCH 9/9] Correct bug in recursive dict loop. Fixes #334 --- stix2/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 22b1128..4248075 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -394,7 +394,6 @@ class _Observable(_STIXBase): streamlined_obj_vals.append(temp_deep_copy) else: streamlined_obj_vals.append(kwargs[key]) - if streamlined_obj_vals: data = canonicalize(streamlined_obj_vals, utf8=False) @@ -450,7 +449,7 @@ def _recursive_stix_to_dict(input_dict): elif isinstance(input_dict[key], list): _recursive_stix_list_to_dict(input_dict[key]) else: - return + pass def _recursive_stix_list_to_dict(input_list):