From caa1d45ae29bae63299e80cac754cc492d42909c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 11 Jun 2019 18:10:02 -0400 Subject: [PATCH 01/62] Update stix2.1 course-of-action support to the latest spec. --- stix2/test/v21/test_course_of_action.py | 31 ++++++++++++++++++++++++- stix2/v21/sdo.py | 12 ++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/stix2/test/v21/test_course_of_action.py b/stix2/test/v21/test_course_of_action.py index 73e8eca..ce7f96f 100644 --- a/stix2/test/v21/test_course_of_action.py +++ b/stix2/test/v21/test_course_of_action.py @@ -4,6 +4,7 @@ import pytest import pytz import stix2 +import stix2.exceptions from .constants import COURSE_OF_ACTION_ID @@ -15,7 +16,14 @@ EXPECTED = """{ "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", - "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ..." + "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", + "action_type": "textual:text/plain", + "os_execution_envs": [ + "a", + "b", + "c" + ], + "action_bin": "aGVsbG8gd29ybGQ=" }""" @@ -27,6 +35,9 @@ def test_course_of_action_example(): modified="2016-04-06T20:03:48.000Z", name="Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", description="This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", + action_type="textual:text/plain", + os_execution_envs=["a", "b", "c"], + action_bin="aGVsbG8gd29ybGQ=" ) assert str(coa) == EXPECTED @@ -44,6 +55,9 @@ def test_course_of_action_example(): "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", "spec_version": "2.1", "type": "course-of-action", + "action_type": "textual:text/plain", + "os_execution_envs": ["a", "b", "c"], + "action_bin": "aGVsbG8gd29ybGQ=" }, ], ) @@ -58,5 +72,20 @@ def test_parse_course_of_action(data): assert coa.created_by_ref == "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff" assert coa.description == "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ..." assert coa.name == "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter" + assert coa.action_type == "textual:text/plain" + assert coa.os_execution_envs == ["a", "b", "c"] + assert coa.action_bin == "aGVsbG8gd29ybGQ=" + + +def test_course_of_action_constraint(): + with pytest.raises(stix2.exceptions.MutuallyExclusivePropertiesError): + stix2.v21.CourseOfAction( + name="Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", + action_bin="aGVsbG8gd29ybGQ=", + action_reference=stix2.v21.ExternalReference( + source_name="a source", + description="description of a source" + ) + ) # TODO: Add other examples diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 37699a6..2b1b49c 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -97,6 +97,10 @@ class CourseOfAction(STIXDomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), + ('action_type', StringProperty()), + ('os_execution_envs', ListProperty(StringProperty)), + ('action_bin', StringProperty()), + ('action_reference', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -106,6 +110,14 @@ class CourseOfAction(STIXDomainObject): ('granular_markings', ListProperty(GranularMarking)), ]) + def _check_object_constraints(self): + super(CourseOfAction, self)._check_object_constraints() + + self._check_mutually_exclusive_properties( + ["action_bin", "action_reference"], + at_least_one=False + ) + class Identity(STIXDomainObject): # TODO: Add link From 4f593e6d165dde7a19b081c56c0358b2f8533903 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 12 Jun 2019 14:49:34 -0400 Subject: [PATCH 02/62] Changes from the add-trailing-comma pre-commit hook --- stix2/test/v21/test_course_of_action.py | 8 ++++---- stix2/v21/sdo.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stix2/test/v21/test_course_of_action.py b/stix2/test/v21/test_course_of_action.py index ce7f96f..5354d59 100644 --- a/stix2/test/v21/test_course_of_action.py +++ b/stix2/test/v21/test_course_of_action.py @@ -37,7 +37,7 @@ def test_course_of_action_example(): description="This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", action_type="textual:text/plain", os_execution_envs=["a", "b", "c"], - action_bin="aGVsbG8gd29ybGQ=" + action_bin="aGVsbG8gd29ybGQ=", ) assert str(coa) == EXPECTED @@ -57,7 +57,7 @@ def test_course_of_action_example(): "type": "course-of-action", "action_type": "textual:text/plain", "os_execution_envs": ["a", "b", "c"], - "action_bin": "aGVsbG8gd29ybGQ=" + "action_bin": "aGVsbG8gd29ybGQ=", }, ], ) @@ -84,8 +84,8 @@ def test_course_of_action_constraint(): action_bin="aGVsbG8gd29ybGQ=", action_reference=stix2.v21.ExternalReference( source_name="a source", - description="description of a source" - ) + description="description of a source", + ), ) # TODO: Add other examples diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 2b1b49c..8e5cdc2 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -115,7 +115,7 @@ class CourseOfAction(STIXDomainObject): self._check_mutually_exclusive_properties( ["action_bin", "action_reference"], - at_least_one=False + at_least_one=False, ) From ed106f23ff736277e0faf00fa2efc2c961ab1712 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 12 Jun 2019 20:19:47 -0400 Subject: [PATCH 03/62] Update IDProperty and ReferenceProperty to support both stix 2.0 and 2.1 rules regarding identifiers. Change relevant property tests to specify which spec version to use, and modify tests according to the specs. --- stix2/properties.py | 83 ++++++++++++++++++++++--------- stix2/test/v20/test_properties.py | 21 +++++--- stix2/test/v21/test_properties.py | 30 ++++++----- 3 files changed, 91 insertions(+), 43 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 24549aa..d0349c3 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -16,22 +16,29 @@ from .core import STIX2_OBJ_MAPS, parse, parse_observable from .exceptions import CustomContentError, DictionaryKeyError from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime -# This uses the regular expression for a RFC 4122, Version 4 UUID. In the -# 8-4-4-4-12 hexadecimal representation, the first hex digit of the third -# component must be a 4, and the first hex digit of the fourth component -# must be 8, 9, a, or b (10xx bit pattern). -ID_REGEX = re.compile( - r"^[a-z0-9][a-z0-9-]+[a-z0-9]--" # object type - "[0-9a-fA-F]{8}-" - "[0-9a-fA-F]{4}-" - "4[0-9a-fA-F]{3}-" - "[89abAB][0-9a-fA-F]{3}-" - "[0-9a-fA-F]{12}$", +ERROR_INVALID_ID = ( + "not a valid STIX identifier, must match --: {}" ) -ERROR_INVALID_ID = ( - "not a valid STIX identifier, must match --" -) + +def _check_uuid(uuid_str, spec_version): + """ + Check whether the given UUID string is valid with respect to the given STIX + spec version. STIX 2.0 requires UUIDv4; 2.1 only requires the RFC 4122 + variant. + + :param uuid_str: A UUID as a string + :param spec_version: The STIX spec version + :return: True if the UUID is valid, False if not + :raises ValueError: If uuid_str is malformed + """ + uuid_obj = uuid.UUID(uuid_str) + + ok = uuid_obj.variant == uuid.RFC_4122 + if ok and spec_version == "2.0": + ok = uuid_obj.version == 4 + + return ok class Property(object): @@ -185,15 +192,25 @@ class TypeProperty(Property): class IDProperty(Property): - def __init__(self, type): + def __init__(self, type, spec_version="2.1"): self.required_prefix = type + "--" + self.spec_version = spec_version super(IDProperty, self).__init__() def clean(self, value): if not value.startswith(self.required_prefix): raise ValueError("must start with '{}'.".format(self.required_prefix)) - if not ID_REGEX.match(value): - raise ValueError(ERROR_INVALID_ID) + + uuid_part = value[len(self.required_prefix):] + try: + result = _check_uuid(uuid_part, self.spec_version) + except ValueError: + # replace their ValueError with ours + raise ValueError(ERROR_INVALID_ID.format(value)) + + if not result: + raise ValueError(ERROR_INVALID_ID.format(value)) + return value def default(self): @@ -366,22 +383,40 @@ class HexProperty(Property): class ReferenceProperty(Property): - def __init__(self, type=None, **kwargs): + def __init__(self, type=None, spec_version="2.1", **kwargs): """ references sometimes must be to a specific object type """ - self.type = type + self.required_prefix = type + "--" if type else None + self.spec_version = spec_version super(ReferenceProperty, self).__init__(**kwargs) def clean(self, value): if isinstance(value, _STIXBase): value = value.id value = str(value) - if self.type: - if not value.startswith(self.type): - raise ValueError("must start with '{}'.".format(self.type)) - if not ID_REGEX.match(value): - raise ValueError(ERROR_INVALID_ID) + + if self.required_prefix: + if not value.startswith(self.required_prefix): + raise ValueError( + "must start with '{}'.".format(self.required_prefix), + ) + + try: + if self.required_prefix: + uuid_part = value[len(self.required_prefix):] + else: + idx = value.index("--") + uuid_part = value[idx+2:] + + result = _check_uuid(uuid_part, self.spec_version) + except ValueError: + # replace their ValueError with ours + raise ValueError(ERROR_INVALID_ID.format(value)) + + if not result: + raise ValueError(ERROR_INVALID_ID.format(value)) + return value diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 24c1c99..cbd9e98 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -5,7 +5,7 @@ import pytest import stix2 from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError from stix2.properties import ( - ERROR_INVALID_ID, BinaryProperty, BooleanProperty, DictionaryProperty, + BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, Property, ReferenceProperty, STIXObjectProperty, StringProperty, @@ -89,7 +89,7 @@ def test_type_property(): assert prop.clean(prop.default()) -ID_PROP = IDProperty('my-type') +ID_PROP = IDProperty('my-type', spec_version="2.0") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' @@ -127,7 +127,7 @@ CONSTANT_IDS.extend(constants.RELATIONSHIP_IDS) @pytest.mark.parametrize("value", CONSTANT_IDS) def test_id_property_valid_for_type(value): type = value.split('--', 1)[0] - assert IDProperty(type=type).clean(value) == value + assert IDProperty(type=type, spec_version="2.0").clean(value) == value def test_id_property_wrong_type(): @@ -147,9 +147,8 @@ def test_id_property_wrong_type(): ], ) def test_id_property_not_a_valid_hex_uuid(value): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): ID_PROP.clean(value) - assert str(excinfo.value) == ERROR_INVALID_ID def test_id_property_default(): @@ -275,7 +274,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty() + ref_prop = ReferenceProperty(spec_version="2.0") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): @@ -286,6 +285,16 @@ def test_reference_property(): ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000") +def test_reference_property_specific_type(): + ref_prop = ReferenceProperty("my-type", spec_version="2.0") + + with pytest.raises(ValueError): + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") + + assert ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") == \ + "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" + + @pytest.mark.parametrize( "value", [ '2017-01-01T12:34:56Z', diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 611ec5e..3e4ee92 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -1,11 +1,9 @@ -import uuid - import pytest import stix2 from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError from stix2.properties import ( - ERROR_INVALID_ID, BinaryProperty, BooleanProperty, DictionaryProperty, + BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, Property, ReferenceProperty, StringProperty, TimestampProperty, @@ -89,7 +87,7 @@ def test_type_property(): assert prop.clean(prop.default()) -ID_PROP = IDProperty('my-type') +ID_PROP = IDProperty('my-type', spec_version="2.1") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' @@ -127,7 +125,7 @@ CONSTANT_IDS.extend(constants.RELATIONSHIP_IDS) @pytest.mark.parametrize("value", CONSTANT_IDS) def test_id_property_valid_for_type(value): type = value.split('--', 1)[0] - assert IDProperty(type=type).clean(value) == value + assert IDProperty(type=type, spec_version="2.1").clean(value) == value def test_id_property_wrong_type(): @@ -139,17 +137,13 @@ def test_id_property_wrong_type(): @pytest.mark.parametrize( "value", [ 'my-type--foo', - # Not a v4 UUID + # Not a RFC 4122 UUID 'my-type--00000000-0000-0000-0000-000000000000', - 'my-type--' + str(uuid.uuid1()), - 'my-type--' + str(uuid.uuid3(uuid.NAMESPACE_DNS, "example.org")), - 'my-type--' + str(uuid.uuid5(uuid.NAMESPACE_DNS, "example.org")), ], ) def test_id_property_not_a_valid_hex_uuid(value): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): ID_PROP.clean(value) - assert str(excinfo.value) == ERROR_INVALID_ID def test_id_property_default(): @@ -275,17 +269,27 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty() + ref_prop = ReferenceProperty(spec_version="2.1") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): ref_prop.clean("foo") - # This is not a valid V4 UUID + # This is not a valid RFC 4122 UUID with pytest.raises(ValueError): ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000") +def test_reference_property_specific_type(): + ref_prop = ReferenceProperty("my-type", spec_version="2.1") + + with pytest.raises(ValueError): + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") + + assert ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") == \ + "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" + + @pytest.mark.parametrize( "value", [ '2017-01-01T12:34:56Z', From da5978d31735124fd05373840ecba3318f1d0969 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 13 Jun 2019 18:37:21 -0400 Subject: [PATCH 04/62] Factored out more of the STIX identifier validity checking, partly inspired by PR #263. This resulted in some error message format changes (an improvement, I think), which caused some unit test breakage. Removed those asserts from the unit tests, since tests shouldn't be testing human-targeted error messages. --- stix2/properties.py | 67 ++++++++++++++++---------------- stix2/test/v20/test_indicator.py | 2 - stix2/test/v20/test_report.py | 2 - stix2/test/v20/test_sighting.py | 4 -- stix2/test/v21/test_indicator.py | 2 - stix2/test/v21/test_report.py | 2 - stix2/test/v21/test_sighting.py | 2 - 7 files changed, 34 insertions(+), 47 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index d0349c3..71138e7 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -41,6 +41,38 @@ def _check_uuid(uuid_str, spec_version): return ok +def _validate_id(id_, spec_version, required_prefix): + """ + Check the STIX identifier for correctness, raise an exception if there are + errors. + + :param id_: The STIX identifier + :param spec_version: The STIX specification version to use + :param required_prefix: The required prefix on the identifier, if any. + This function doesn't add a "--" suffix to the prefix, so callers must + add it if it is important. Pass None to skip the prefix check. + :raises ValueError: If there are any errors with the identifier + """ + if required_prefix: + if not id_.startswith(required_prefix): + raise ValueError("must start with '{}'.".format(required_prefix)) + + try: + if required_prefix: + uuid_part = id_[len(required_prefix):] + else: + idx = id_.index("--") + uuid_part = id_[idx+2:] + + result = _check_uuid(uuid_part, spec_version) + except ValueError: + # replace their ValueError with ours + raise ValueError(ERROR_INVALID_ID.format(id_)) + + if not result: + raise ValueError(ERROR_INVALID_ID.format(id_)) + + class Property(object): """Represent a property of STIX data type. @@ -198,19 +230,7 @@ class IDProperty(Property): super(IDProperty, self).__init__() def clean(self, value): - if not value.startswith(self.required_prefix): - raise ValueError("must start with '{}'.".format(self.required_prefix)) - - uuid_part = value[len(self.required_prefix):] - try: - result = _check_uuid(uuid_part, self.spec_version) - except ValueError: - # replace their ValueError with ours - raise ValueError(ERROR_INVALID_ID.format(value)) - - if not result: - raise ValueError(ERROR_INVALID_ID.format(value)) - + _validate_id(value, self.spec_version, self.required_prefix) return value def default(self): @@ -396,26 +416,7 @@ class ReferenceProperty(Property): value = value.id value = str(value) - if self.required_prefix: - if not value.startswith(self.required_prefix): - raise ValueError( - "must start with '{}'.".format(self.required_prefix), - ) - - try: - if self.required_prefix: - uuid_part = value[len(self.required_prefix):] - else: - idx = value.index("--") - uuid_part = value[idx+2:] - - result = _check_uuid(uuid_part, self.spec_version) - except ValueError: - # replace their ValueError with ours - raise ValueError(ERROR_INVALID_ID.format(value)) - - if not result: - raise ValueError(ERROR_INVALID_ID.format(value)) + _validate_id(value, self.spec_version, self.required_prefix) return value diff --git a/stix2/test/v20/test_indicator.py b/stix2/test/v20/test_indicator.py index f8c3a91..5194142 100644 --- a/stix2/test/v20/test_indicator.py +++ b/stix2/test/v20/test_indicator.py @@ -112,8 +112,6 @@ def test_indicator_created_ref_invalid_format(): assert excinfo.value.cls == stix2.v20.Indicator assert excinfo.value.prop_name == "created_by_ref" - assert excinfo.value.reason == "must start with 'identity'." - assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must start with 'identity'." def test_indicator_revoked_invalid(): diff --git a/stix2/test/v20/test_report.py b/stix2/test/v20/test_report.py index 072fc95..fc4f288 100644 --- a/stix2/test/v20/test_report.py +++ b/stix2/test/v20/test_report.py @@ -87,8 +87,6 @@ def test_report_example_objects_in_object_refs_with_bad_id(): assert excinfo.value.cls == stix2.v20.Report assert excinfo.value.prop_name == "object_refs" - assert excinfo.value.reason == stix2.properties.ERROR_INVALID_ID - assert str(excinfo.value) == "Invalid value for Report 'object_refs': " + stix2.properties.ERROR_INVALID_ID @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_sighting.py b/stix2/test/v20/test_sighting.py index e93ca7e..f1d40c5 100644 --- a/stix2/test/v20/test_sighting.py +++ b/stix2/test/v20/test_sighting.py @@ -59,8 +59,6 @@ def test_sighting_bad_where_sighted_refs(): assert excinfo.value.cls == stix2.v20.Sighting assert excinfo.value.prop_name == "where_sighted_refs" - assert excinfo.value.reason == "must start with 'identity'." - assert str(excinfo.value) == "Invalid value for Sighting 'where_sighted_refs': must start with 'identity'." def test_sighting_type_must_be_sightings(): @@ -69,8 +67,6 @@ def test_sighting_type_must_be_sightings(): assert excinfo.value.cls == stix2.v20.Sighting assert excinfo.value.prop_name == "type" - assert excinfo.value.reason == "must equal 'sighting'." - assert str(excinfo.value) == "Invalid value for Sighting 'type': must equal 'sighting'." def test_invalid_kwarg_to_sighting(): diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 628bdff..a54ea62 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -116,8 +116,6 @@ def test_indicator_created_ref_invalid_format(): assert excinfo.value.cls == stix2.v21.Indicator assert excinfo.value.prop_name == "created_by_ref" - assert excinfo.value.reason == "must start with 'identity'." - assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must start with 'identity'." def test_indicator_revoked_invalid(): diff --git a/stix2/test/v21/test_report.py b/stix2/test/v21/test_report.py index c9d790e..8b7ee78 100644 --- a/stix2/test/v21/test_report.py +++ b/stix2/test/v21/test_report.py @@ -88,8 +88,6 @@ def test_report_example_objects_in_object_refs_with_bad_id(): assert excinfo.value.cls == stix2.v21.Report assert excinfo.value.prop_name == "object_refs" - assert excinfo.value.reason == stix2.properties.ERROR_INVALID_ID - assert str(excinfo.value) == "Invalid value for Report 'object_refs': " + stix2.properties.ERROR_INVALID_ID @pytest.mark.parametrize( diff --git a/stix2/test/v21/test_sighting.py b/stix2/test/v21/test_sighting.py index 8fcbb6d..cfdc7ea 100644 --- a/stix2/test/v21/test_sighting.py +++ b/stix2/test/v21/test_sighting.py @@ -61,8 +61,6 @@ def test_sighting_bad_where_sighted_refs(): assert excinfo.value.cls == stix2.v21.Sighting assert excinfo.value.prop_name == "where_sighted_refs" - assert excinfo.value.reason == "must start with 'identity'." - assert str(excinfo.value) == "Invalid value for Sighting 'where_sighted_refs': must start with 'identity'." def test_sighting_type_must_be_sightings(): From a150b0f4aa7e6c7273f900b8af6f90c0d7aa723e Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 14 Jun 2019 17:58:51 -0400 Subject: [PATCH 05/62] Change all uses of IDProperty and ReferenceProperty to specify a particular spec_version. --- stix2/v20/bundle.py | 2 +- stix2/v20/common.py | 8 ++-- stix2/v20/sdo.py | 80 ++++++++++++++++++------------------- stix2/v20/sro.py | 22 +++++------ stix2/v21/bundle.py | 2 +- stix2/v21/common.py | 14 +++---- stix2/v21/sdo.py | 96 ++++++++++++++++++++++----------------------- stix2/v21/sro.py | 22 +++++------ 8 files changed, 123 insertions(+), 123 deletions(-) diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index 76386ef..9dd2df4 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -16,7 +16,7 @@ class Bundle(_STIXBase): _type = 'bundle' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.0')), # Not technically correct: STIX 2.0 spec doesn't say spec_version must # have this value, but it's all we support for now. ('spec_version', StringProperty(fixed='2.0')), diff --git a/stix2/v20/common.py b/stix2/v20/common.py index afd7812..b8149a2 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -48,7 +48,7 @@ class GranularMarking(_STIXBase): """ _properties = OrderedDict([ - ('marking_ref', ReferenceProperty(required=True, type='marking-definition')), + ('marking_ref', ReferenceProperty(required=True, spec_version='2.0', type='marking-definition')), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -103,11 +103,11 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): _type = 'marking-definition' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index cd99e69..0055295 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -22,8 +22,8 @@ class AttackPattern(STIXDomainObject): _type = 'attack-pattern' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -32,7 +32,7 @@ class AttackPattern(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -45,8 +45,8 @@ class Campaign(STIXDomainObject): _type = 'campaign' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -58,7 +58,7 @@ class Campaign(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -71,8 +71,8 @@ class CourseOfAction(STIXDomainObject): _type = 'course-of-action' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -80,7 +80,7 @@ class CourseOfAction(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -93,8 +93,8 @@ class Identity(STIXDomainObject): _type = 'identity' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -105,7 +105,7 @@ class Identity(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -118,8 +118,8 @@ class Indicator(STIXDomainObject): _type = 'indicator' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -131,7 +131,7 @@ class Indicator(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -144,8 +144,8 @@ class IntrusionSet(STIXDomainObject): _type = 'intrusion-set' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -160,7 +160,7 @@ class IntrusionSet(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -173,8 +173,8 @@ class Malware(STIXDomainObject): _type = 'malware' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -183,7 +183,7 @@ class Malware(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -196,8 +196,8 @@ class ObservedData(STIXDomainObject): _type = 'observed-data' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), @@ -207,7 +207,7 @@ class ObservedData(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -226,18 +226,18 @@ class Report(STIXDomainObject): _type = 'report' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -250,8 +250,8 @@ class ThreatActor(STIXDomainObject): _type = 'threat-actor' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -267,7 +267,7 @@ class ThreatActor(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -280,8 +280,8 @@ class Tool(STIXDomainObject): _type = 'tool' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -291,7 +291,7 @@ class Tool(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -304,8 +304,8 @@ class Vulnerability(STIXDomainObject): _type = 'vulnerability' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -313,7 +313,7 @@ class Vulnerability(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -351,8 +351,8 @@ def CustomObject(type='x-custom-type', properties=None): _properties = list(itertools.chain.from_iterable([ [ ('type', TypeProperty(type)), - ('id', IDProperty(type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -361,7 +361,7 @@ def CustomObject(type='x-custom-type', properties=None): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index dbf6812..f450003 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -19,18 +19,18 @@ class Relationship(STIXRelationshipObject): _type = 'relationship' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(required=True)), - ('target_ref', ReferenceProperty(required=True)), + ('source_ref', ReferenceProperty(spec_version='2.0', required=True)), + ('target_ref', ReferenceProperty(spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -58,21 +58,21 @@ class Sighting(STIXRelationshipObject): _type = 'sighting' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.0')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity'))), + ('sighting_of_ref', ReferenceProperty(spec_version='2.0', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.0'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index c9e083a..c84058e 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -17,7 +17,7 @@ class Bundle(_STIXBase): _type = 'bundle' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('objects', ListProperty(STIXObjectProperty(spec_version='2.1'))), ]) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 0aded3b..110529e 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -53,7 +53,7 @@ class GranularMarking(_STIXBase): _properties = OrderedDict([ ('lang', StringProperty()), - ('marking_ref', ReferenceProperty(type='marking-definition')), + ('marking_ref', ReferenceProperty(type='marking-definition', spec_version='2.1')), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -72,11 +72,11 @@ class LanguageContent(_STIXBase): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('object_ref', ReferenceProperty(required=True)), + ('object_ref', ReferenceProperty(spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(required=True, precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx @@ -85,7 +85,7 @@ class LanguageContent(_STIXBase): ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -144,10 +144,10 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 8e5cdc2..ffdc5e1 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -24,8 +24,8 @@ class AttackPattern(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -36,7 +36,7 @@ class AttackPattern(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -51,8 +51,8 @@ class Campaign(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -66,7 +66,7 @@ class Campaign(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -91,8 +91,8 @@ class CourseOfAction(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -106,7 +106,7 @@ class CourseOfAction(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -129,8 +129,8 @@ class Identity(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -144,7 +144,7 @@ class Identity(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -159,8 +159,8 @@ class Indicator(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -175,7 +175,7 @@ class Indicator(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -200,8 +200,8 @@ class IntrusionSet(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -218,7 +218,7 @@ class IntrusionSet(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -243,8 +243,8 @@ class Location(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('description', StringProperty()), @@ -262,7 +262,7 @@ class Location(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -286,8 +286,8 @@ class Malware(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -299,7 +299,7 @@ class Malware(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -314,8 +314,8 @@ class Note(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('abstract', StringProperty()), @@ -327,7 +327,7 @@ class Note(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -342,8 +342,8 @@ class ObservedData(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), @@ -355,7 +355,7 @@ class ObservedData(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -390,8 +390,8 @@ class Opinion(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('explanation', StringProperty()), @@ -413,7 +413,7 @@ class Opinion(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -428,8 +428,8 @@ class Report(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -442,7 +442,7 @@ class Report(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -457,8 +457,8 @@ class ThreatActor(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -477,7 +477,7 @@ class ThreatActor(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -492,8 +492,8 @@ class Tool(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -506,7 +506,7 @@ class Tool(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -521,8 +521,8 @@ class Vulnerability(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -532,7 +532,7 @@ class Vulnerability(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -571,8 +571,8 @@ def CustomObject(type='x-custom-type', properties=None): [ ('type', TypeProperty(type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -583,7 +583,7 @@ def CustomObject(type='x-custom-type', properties=None): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index f947b2e..144df59 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -21,14 +21,14 @@ class Relationship(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(required=True)), - ('target_ref', ReferenceProperty(required=True)), + ('source_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -36,7 +36,7 @@ class Relationship(STIXRelationshipObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -76,23 +76,23 @@ class Sighting(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), - ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity'))), + ('sighting_of_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.1'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.1'))), ('summary', BooleanProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition'))), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) From d61b54326642f64bf16581492d6950491277eebd Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 14 Jun 2019 18:10:38 -0400 Subject: [PATCH 06/62] Style changes to satisfy the 'style' tox check --- stix2/markings/utils.py | 2 +- stix2/properties.py | 2 +- stix2/test/v20/test_observed_data.py | 26 +++++++++++++------------- stix2/test/v21/test_observed_data.py | 20 ++++++++++---------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 4b4841c..d8bbf1d 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -39,7 +39,7 @@ def _validate_selector(obj, selector): def _get_marking_id(marking): - if type(marking).__name__ is 'MarkingDefinition': # avoid circular import + if type(marking).__name__ == 'MarkingDefinition': # avoid circular import return marking.id return marking diff --git a/stix2/properties.py b/stix2/properties.py index 71138e7..f904e2a 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -182,7 +182,7 @@ class ListProperty(Property): if type(self.contained) is EmbeddedObjectProperty: obj_type = self.contained.type - elif type(self.contained).__name__ is "STIXObjectProperty": + elif type(self.contained).__name__ == "STIXObjectProperty": # ^ this way of checking doesn't require a circular import # valid is already an instance of a python-stix2 class; no need # to turn it into a dictionary and then pass it to the class diff --git a/stix2/test/v20/test_observed_data.py b/stix2/test/v20/test_observed_data.py index 41a80d6..cbdd394 100644 --- a/stix2/test/v20/test_observed_data.py +++ b/stix2/test/v20/test_observed_data.py @@ -1095,11 +1095,11 @@ def test_process_example_empty_error(): def test_process_example_empty_with_extensions(): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.v20.Process( - extensions={ - "windows-process-ext": {}, - }, - ) + stix2.v20.Process( + extensions={ + "windows-process-ext": {}, + }, + ) assert excinfo.value.cls == stix2.v20.WindowsProcessExt properties_of_extension = list(stix2.v20.WindowsProcessExt._properties.keys()) @@ -1127,13 +1127,13 @@ def test_process_example_windows_process_ext(): def test_process_example_windows_process_ext_empty(): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.v20.Process( - pid=1221, - name="gedit-bin", - extensions={ - "windows-process-ext": {}, - }, - ) + stix2.v20.Process( + pid=1221, + name="gedit-bin", + extensions={ + "windows-process-ext": {}, + }, + ) assert excinfo.value.cls == stix2.v20.WindowsProcessExt properties_of_extension = list(stix2.v20.WindowsProcessExt._properties.keys()) @@ -1142,7 +1142,7 @@ def test_process_example_windows_process_ext_empty(): def test_process_example_extensions_empty(): with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: - stix2.v20.Process(extensions={}) + stix2.v20.Process(extensions={}) assert excinfo.value.cls == stix2.v20.Process assert excinfo.value.prop_name == 'extensions' diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 5a5881a..e729f60 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1073,9 +1073,9 @@ def test_process_example_empty_error(): def test_process_example_empty_with_extensions(): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.v21.Process(extensions={ - "windows-process-ext": {}, - }) + stix2.v21.Process(extensions={ + "windows-process-ext": {}, + }) assert excinfo.value.cls == stix2.v21.WindowsProcessExt properties_of_extension = list(stix2.v21.WindowsProcessExt._properties.keys()) @@ -1102,12 +1102,12 @@ def test_process_example_windows_process_ext(): def test_process_example_windows_process_ext_empty(): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.v21.Process( - pid=1221, - extensions={ - "windows-process-ext": {}, - }, - ) + stix2.v21.Process( + pid=1221, + extensions={ + "windows-process-ext": {}, + }, + ) assert excinfo.value.cls == stix2.v21.WindowsProcessExt properties_of_extension = list(stix2.v21.WindowsProcessExt._properties.keys()) @@ -1116,7 +1116,7 @@ def test_process_example_windows_process_ext_empty(): def test_process_example_extensions_empty(): with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: - stix2.v21.Process(extensions={}) + stix2.v21.Process(extensions={}) assert excinfo.value.cls == stix2.v21.Process assert excinfo.value.prop_name == 'extensions' From ea98a53fae5588bdd9b0c0871d32c6b82c8d46b5 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 21 Jun 2019 13:18:51 -0400 Subject: [PATCH 07/62] Change all hard-coded spec_version defaults in property classes to stix2.DEFAULT_VERSION. --- stix2/properties.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index f904e2a..85bf5f8 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -15,6 +15,7 @@ from .base import _STIXBase from .core import STIX2_OBJ_MAPS, parse, parse_observable from .exceptions import CustomContentError, DictionaryKeyError from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime +import stix2 ERROR_INVALID_ID = ( "not a valid STIX identifier, must match --: {}" @@ -224,7 +225,7 @@ class TypeProperty(Property): class IDProperty(Property): - def __init__(self, type, spec_version="2.1"): + def __init__(self, type, spec_version=stix2.DEFAULT_VERSION): self.required_prefix = type + "--" self.spec_version = spec_version super(IDProperty, self).__init__() @@ -319,7 +320,7 @@ class TimestampProperty(Property): class DictionaryProperty(Property): - def __init__(self, spec_version='2.0', **kwargs): + def __init__(self, spec_version=stix2.DEFAULT_VERSION, **kwargs): self.spec_version = spec_version super(DictionaryProperty, self).__init__(**kwargs) @@ -403,7 +404,7 @@ class HexProperty(Property): class ReferenceProperty(Property): - def __init__(self, type=None, spec_version="2.1", **kwargs): + def __init__(self, type=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ @@ -485,7 +486,7 @@ class ObservableProperty(Property): """Property for holding Cyber Observable Objects. """ - def __init__(self, spec_version='2.0', allow_custom=False, *args, **kwargs): + def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, *args, **kwargs): self.allow_custom = allow_custom self.spec_version = spec_version super(ObservableProperty, self).__init__(*args, **kwargs) @@ -520,7 +521,7 @@ class ExtensionsProperty(DictionaryProperty): """Property for representing extensions on Observable objects. """ - def __init__(self, spec_version='2.0', allow_custom=False, enclosing_type=None, required=False): + def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, enclosing_type=None, required=False): self.allow_custom = allow_custom self.enclosing_type = enclosing_type super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required) @@ -561,7 +562,7 @@ class ExtensionsProperty(DictionaryProperty): class STIXObjectProperty(Property): - def __init__(self, spec_version='2.0', allow_custom=False, *args, **kwargs): + def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, *args, **kwargs): self.allow_custom = allow_custom self.spec_version = spec_version super(STIXObjectProperty, self).__init__(*args, **kwargs) From f9578313a011b14b046933e550a9d9273925c869 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 21 Jun 2019 13:20:37 -0400 Subject: [PATCH 08/62] Change stix2.DEFAULT_VERSION to "2.1" on the stix2.1 branch. --- stix2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index 3bedec8..cc7d780 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -65,4 +65,4 @@ from .version import __version__ _collect_stix2_mappings() -DEFAULT_VERSION = '2.0' # Default version will always be the latest STIX 2.X version +DEFAULT_VERSION = '2.1' # Default version will always be the latest STIX 2.X version From 8bb6c79f1db22202b0ef6b27c3d3047de72608ac Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 21 Jun 2019 14:25:36 -0400 Subject: [PATCH 09/62] Change import order to satisfy style checkers... --- stix2/properties.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stix2/properties.py b/stix2/properties.py index 85bf5f8..8e0066c 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -11,11 +11,12 @@ import uuid from six import string_types, text_type from stix2patterns.validator import run_validator +import stix2 + from .base import _STIXBase from .core import STIX2_OBJ_MAPS, parse, parse_observable from .exceptions import CustomContentError, DictionaryKeyError from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime -import stix2 ERROR_INVALID_ID = ( "not a valid STIX identifier, must match --: {}" From 9cc1e6e8c1638584a9fafcb35ab55b40793eedcf Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 21 Jun 2019 14:26:48 -0400 Subject: [PATCH 10/62] Change location of DEFAULT_VERSION definition, to be before the imports. This ensures the attribute will be defined even if there are import loops. --- stix2/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index cc7d780..4498c8b 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -28,6 +28,8 @@ # flake8: noqa +DEFAULT_VERSION = '2.1' # Default version will always be the latest STIX 2.X version + from .confidence import scales from .core import _collect_stix2_mappings, parse, parse_observable from .datastore import CompositeDataSource @@ -64,5 +66,3 @@ from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ _collect_stix2_mappings() - -DEFAULT_VERSION = '2.1' # Default version will always be the latest STIX 2.X version From 23d5bef2ecbbba0966a871b74d4a0887e63c22a2 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 21 Jun 2019 14:29:08 -0400 Subject: [PATCH 11/62] Change all uses of multi-STIX-version properties (i.e. those with a spec_version constructor argument) in STIX-version-specific contexts, to explicitly specify the STIX version. --- stix2/test/v20/test_properties.py | 14 ++++---- stix2/v20/bundle.py | 2 +- stix2/v20/observables.py | 54 +++++++++++++++---------------- stix2/v20/sdo.py | 2 +- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index cbd9e98..4305e57 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -338,7 +338,7 @@ def test_hex_property(): ], ) def test_dictionary_property_valid(d): - dict_prop = DictionaryProperty() + dict_prop = DictionaryProperty(spec_version="2.0") assert dict_prop.clean(d) @@ -359,7 +359,7 @@ def test_dictionary_property_valid(d): ], ) def test_dictionary_property_invalid_key(d): - dict_prop = DictionaryProperty() + dict_prop = DictionaryProperty(spec_version="2.0") with pytest.raises(DictionaryKeyError) as excinfo: dict_prop.clean(d[0]) @@ -382,7 +382,7 @@ def test_dictionary_property_invalid_key(d): ], ) def test_dictionary_property_invalid(d): - dict_prop = DictionaryProperty() + dict_prop = DictionaryProperty(spec_version="2.0") with pytest.raises(ValueError) as excinfo: dict_prop.clean(d[0]) @@ -392,7 +392,7 @@ def test_dictionary_property_invalid(d): def test_property_list_of_dictionary(): @stix2.v20.CustomObject( 'x-new-obj', [ - ('property1', ListProperty(DictionaryProperty(), required=True)), + ('property1', ListProperty(DictionaryProperty(spec_version="2.0"), required=True)), ], ) class NewObj(): @@ -458,7 +458,7 @@ def test_enum_property_invalid(): def test_extension_property_valid(): - ext_prop = ExtensionsProperty(enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') assert ext_prop({ 'windows-pebinary-ext': { 'pe_type': 'exe', @@ -475,13 +475,13 @@ def test_extension_property_valid(): ], ) def test_extension_property_invalid(data): - ext_prop = ExtensionsProperty(enclosing_type='file') + ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') with pytest.raises(ValueError): ext_prop.clean(data) def test_extension_property_invalid_type(): - ext_prop = ExtensionsProperty(enclosing_type='indicator') + ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='indicator') with pytest.raises(ValueError) as excinfo: ext_prop.clean( { diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index 9dd2df4..fe6b62e 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -20,7 +20,7 @@ class Bundle(_STIXBase): # Not technically correct: STIX 2.0 spec doesn't say spec_version must # have this value, but it's all we support for now. ('spec_version', StringProperty(fixed='2.0')), - ('objects', ListProperty(STIXObjectProperty)), + ('objects', ListProperty(STIXObjectProperty(spec_version="2.0"))), ]) def __init__(self, *args, **kwargs): diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 0e7c4a0..ba28214 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -31,7 +31,7 @@ class Artifact(_Observable): ('payload_bin', BinaryProperty()), ('url', StringProperty()), ('hashes', HashesProperty()), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -51,7 +51,7 @@ class AutonomousSystem(_Observable): ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -70,7 +70,7 @@ class Directory(_Observable): ('modified', TimestampProperty()), ('accessed', TimestampProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -84,7 +84,7 @@ class DomainName(_Observable): ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -99,7 +99,7 @@ class EmailAddress(_Observable): ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -138,11 +138,11 @@ class EmailMessage(_Observable): ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), - ('additional_header_fields', DictionaryProperty()), + ('additional_header_fields', DictionaryProperty(spec_version="2.0")), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -199,7 +199,7 @@ class PDFExt(_Extension): _properties = OrderedDict([ ('version', StringProperty()), ('is_optimized', BooleanProperty()), - ('document_info_dict', DictionaryProperty()), + ('document_info_dict', DictionaryProperty(spec_version="2.0")), ('pdfid0', StringProperty()), ('pdfid1', StringProperty()), ]) @@ -216,7 +216,7 @@ class RasterImageExt(_Extension): ('image_width', IntegerProperty()), ('bits_per_pixel', IntegerProperty()), ('image_compression_algorithm', StringProperty()), - ('exif_tags', DictionaryProperty()), + ('exif_tags', DictionaryProperty(spec_version="2.0")), ]) @@ -323,7 +323,7 @@ class File(_Observable): ('decryption_key', StringProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty)), ('content_ref', ObjectReferenceProperty(valid_types='artifact')), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -343,7 +343,7 @@ class IPv4Address(_Observable): ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -358,7 +358,7 @@ class IPv6Address(_Observable): ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -371,7 +371,7 @@ class MACAddress(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -384,7 +384,7 @@ class Mutex(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('name', StringProperty(required=True)), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -398,7 +398,7 @@ class HTTPRequestExt(_Extension): ('request_method', StringProperty(required=True)), ('request_value', StringProperty(required=True)), ('request_version', StringProperty()), - ('request_header', DictionaryProperty()), + ('request_header', DictionaryProperty(spec_version="2.0")), ('message_body_length', IntegerProperty()), ('message_body_data_ref', ObjectReferenceProperty(valid_types='artifact')), ]) @@ -449,7 +449,7 @@ class SocketExt(_Extension): "PF_NETROM", ]), ), - ('options', DictionaryProperty()), + ('options', DictionaryProperty(spec_version="2.0")), ( 'socket_type', EnumProperty(allowed=[ "SOCK_STREAM", @@ -496,12 +496,12 @@ class NetworkTraffic(_Observable): ('dst_byte_count', IntegerProperty()), ('src_packets', IntegerProperty()), ('dst_packets', IntegerProperty()), - ('ipfix', DictionaryProperty()), + ('ipfix', DictionaryProperty(spec_version="2.0")), ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -521,7 +521,7 @@ class WindowsProcessExt(_Extension): ('priority', StringProperty()), ('owner_sid', StringProperty()), ('window_title', StringProperty()), - ('startup_info', DictionaryProperty()), + ('startup_info', DictionaryProperty(spec_version="2.0")), ]) @@ -584,13 +584,13 @@ class Process(_Observable): ('cwd', StringProperty()), ('arguments', ListProperty(StringProperty)), ('command_line', StringProperty()), - ('environment_variables', DictionaryProperty()), + ('environment_variables', DictionaryProperty(spec_version="2.0")), ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('binary_ref', ObjectReferenceProperty(valid_types='file')), ('parent_ref', ObjectReferenceProperty(valid_types='process')), ('child_refs', ListProperty(ObjectReferenceProperty('process'))), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) def _check_object_constraints(self): @@ -621,7 +621,7 @@ class Software(_Observable): ('languages', ListProperty(StringProperty)), ('vendor', StringProperty()), ('version', StringProperty()), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -634,7 +634,7 @@ class URL(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -673,7 +673,7 @@ class UserAccount(_Observable): ('password_last_changed', TimestampProperty()), ('account_first_login', TimestampProperty()), ('account_last_login', TimestampProperty()), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -720,7 +720,7 @@ class WindowsRegistryKey(_Observable): ('modified', TimestampProperty()), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @property @@ -776,7 +776,7 @@ class X509Certificate(_Observable): ('subject_public_key_modulus', StringProperty()), ('subject_public_key_exponent', IntegerProperty()), ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), - ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -798,7 +798,7 @@ def CustomObservable(type='x-custom-observable', properties=None): _properties = list(itertools.chain.from_iterable([ [('type', TypeProperty(type))], properties, - [('extensions', ExtensionsProperty(enclosing_type=type))], + [('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=type))], ])) return _custom_observable_builder(cls, type, _properties, '2.0') return wrapper diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 0055295..3fd3a84 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -203,7 +203,7 @@ class ObservedData(STIXDomainObject): ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), - ('objects', ObservableProperty(required=True)), + ('objects', ObservableProperty(spec_version="2.0", required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), From 58ff89f11268350578099f83bd2c96b70700dc90 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 12 Jun 2019 21:19:50 -0400 Subject: [PATCH 12/62] Update observed-data SDO class, adding the new stix2.1 property "object_refs". Added a couple tests for it. --- stix2/test/v21/test_observed_data.py | 62 ++++++++++++++++++++++++++++ stix2/v21/sdo.py | 7 +++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index e729f60..8dd4487 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -5,6 +5,7 @@ import pytest import pytz import stix2 +import stix2.exceptions from .constants import OBSERVED_DATA_ID @@ -101,6 +102,67 @@ def test_observed_data_example_with_refs(): assert str(observed_data) == EXPECTED_WITH_REF +EXPECTED_OBJECT_REFS = """{ + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 50, + "object_refs": [ + "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", + "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c" + ] +}""" + + +def test_observed_data_example_with_object_refs(): + observed_data = stix2.v21.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + object_refs=[ + "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", + "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + ], + ) + + assert str(observed_data) == EXPECTED_OBJECT_REFS + + +def test_observed_data_object_constraint(): + with pytest.raises(stix2.exceptions.MutuallyExclusivePropertiesError): + stix2.v21.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects={ + "0": { + "name": "foo.exe", + "type": "file", + }, + }, + object_refs=[ + "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", + "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + ], + ) + + def test_observed_data_example_with_bad_refs(): with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.ObservedData( diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index ffdc5e1..70e81e4 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -349,7 +349,8 @@ class ObservedData(STIXDomainObject): ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), - ('objects', ObservableProperty(spec_version='2.1', required=True)), + ('objects', ObservableProperty(spec_version='2.1')), + ('object_refs', ListProperty(ReferenceProperty(spec_version="2.1"))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -379,6 +380,10 @@ class ObservedData(STIXDomainObject): msg = "{0.id} 'last_observed' must be greater than or equal to 'first_observed'" raise ValueError(msg.format(self)) + self._check_mutually_exclusive_properties( + ["objects", "object_refs"], + ) + class Opinion(STIXDomainObject): # TODO: Add link From 28ac284b849f1cfb0459c377fbfa6a934dca1164 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 26 Jun 2019 11:18:47 -0400 Subject: [PATCH 13/62] Remove unnecessary ObservedData constraint first_observed and last_observed are both required, so this co-constraint was removed from WD04. --- stix2/v21/sdo.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 70e81e4..46ce231 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -369,10 +369,6 @@ class ObservedData(STIXDomainObject): def _check_object_constraints(self): super(self.__class__, self)._check_object_constraints() - if self.get('number_observed', 1) == 1: - self._check_properties_dependency(['first_observed'], ['last_observed']) - self._check_properties_dependency(['last_observed'], ['first_observed']) - first_observed = self.get('first_observed') last_observed = self.get('last_observed') From 5c92db9861a9266a447b1a5199a7256dede22536 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 26 Jun 2019 17:06:26 -0400 Subject: [PATCH 14/62] Add stix2.1 malware-analysis SDO --- stix2/v21/__init__.py | 5 +++-- stix2/v21/sdo.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index 4a8fe29..f0131d1 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -21,8 +21,8 @@ from .observables import ( ) from .sdo import ( AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, - IntrusionSet, Location, Malware, Note, ObservedData, Opinion, Report, - ThreatActor, Tool, Vulnerability, + IntrusionSet, Location, Malware, MalwareAnalysis, Note, ObservedData, + Opinion, Report, ThreatActor, Tool, Vulnerability, ) from .sro import Relationship, Sighting @@ -37,6 +37,7 @@ OBJ_MAP = { 'language-content': LanguageContent, 'location': Location, 'malware': Malware, + 'malware-analysis': MalwareAnalysis, 'note': Note, 'marking-definition': MarkingDefinition, 'observed-data': ObservedData, diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index ffdc5e1..526c982 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -304,6 +304,49 @@ class Malware(STIXDomainObject): ]) +class MalwareAnalysis(STIXDomainObject): + # TODO: Add link + """For more detailed information on this object's properties, see + `the STIX 2.1 specification `__. + """ + + _type = 'malware-analysis' + _properties = OrderedDict([ + ('type', TypeProperty(_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('product', StringProperty(required=True)), + ('version', StringProperty()), + ('host_vm_ref', ReferenceProperty(type='software', spec_version='2.1')), + ('operating_system_ref', ReferenceProperty(type='software', spec_version='2.1')), + ('installed_software_refs', ListProperty(ReferenceProperty(type='software', spec_version='2.1'))), + ('configuration_version', StringProperty()), + ('module', StringProperty()), + ('analysis_engine_version', StringProperty()), + ('analysis_definition_version', StringProperty()), + ('submitted', TimestampProperty()), + ('analysis_started', TimestampProperty()), + ('analysis_ended', TimestampProperty()), + ('av_result', StringProperty()), + ('analysis_sco_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), + ]) + + def _check_object_constraints(self): + super(MalwareAnalysis, self)._check_object_constraints() + + self._check_at_least_one_property(["av_result", "analysis_sco_refs"]) + + class Note(STIXDomainObject): # TODO: Add link """For more detailed information on this object's properties, see From 68f93f4110f526d9426bf791d2c004373f1d6c06 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 26 Jun 2019 17:10:04 -0400 Subject: [PATCH 15/62] Oops, forgot to add the malware-analysis test suite... --- stix2/test/v21/test_malware_analysis.py | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 stix2/test/v21/test_malware_analysis.py diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py new file mode 100644 index 0000000..5192317 --- /dev/null +++ b/stix2/test/v21/test_malware_analysis.py @@ -0,0 +1,80 @@ +import json +import pytest + +import stix2.exceptions +import stix2.utils +import stix2.v21 + + +MALWARE_ANALYSIS_JSON = """{ + "type": "malware-analysis", + "spec_version": "2.1", + "id": "malware-analysis--f8afc020-f92f-4906-a971-88ee5882eb46", + "created": "2017-11-28T09:44:58.418Z", + "modified": "2017-12-31T21:27:49.754Z", + "created_by_ref": "identity--e0353ed3-991e-4f71-a332-114c2f10b84f", + "labels": [ + "label1", + "label2" + ], + "product": "Acme Malware Analyzer", + "version": "2.5", + "host_vm_ref": "software--1bda7336-fe67-469f-a8ca-ab6268b0449b", + "operating_system_ref": "software--c96bfaef-861b-408b-b0f1-b685881725ef", + "installed_software_refs": [ + "software--7325bf2d-de9e-441e-b3b3-63df43149897", + "software--46a6a91d-1160-4867-a4d1-b14e080e4e5b" + ], + "configuration_version": "1.7", + "module": "Super Analyzer", + "analysis_engine_version": "1.2", + "analysis_definition_version": "3.4", + "submitted": "2018-11-23T06:45:55.747Z", + "analysis_started": "2018-11-29T07:30:03.895Z", + "analysis_ended": "2018-11-29T08:30:03.895Z", + "av_result": "malicious", + "analysis_sco_refs": [ + "file--fc27e371-6c88-4c5c-868a-4dda0e60b167", + "url--6f7a74cd-8eb2-4b88-a4da-aa878e50ac2e" + ] +}""" + + +MALWARE_ANALYSIS_DICT = json.loads(MALWARE_ANALYSIS_JSON) + + +def test_malware_analysis_example(): + ma = stix2.v21.MalwareAnalysis(**MALWARE_ANALYSIS_DICT) + + assert str(ma) == MALWARE_ANALYSIS_JSON + + +@pytest.mark.parametrize("data", [ + MALWARE_ANALYSIS_JSON, + MALWARE_ANALYSIS_DICT +]) +def test_parse_malware_analysis(data): + ma = stix2.parse(data, version="2.1") + + # timestamp-valued attributes whose values (from JSON) can't be compared + # directly, since stix2 internally converts them to datetime objects. + ts_attrs = { + "created", + "modified", + "submitted", + "analysis_started", + "analysis_ended", + } + + for attr_name, attr_value in MALWARE_ANALYSIS_DICT.items(): + cmp_value = stix2.utils.parse_into_datetime(attr_value) \ + if attr_name in ts_attrs else attr_value + + assert getattr(ma, attr_name) == cmp_value + + +def test_malware_analysis_constraint(): + with pytest.raises(stix2.exceptions.AtLeastOnePropertyError): + stix2.v21.MalwareAnalysis( + product="Acme Malware Analyzer" + ) From c6132537b86e757e4f458de54dea71712cd2b5f4 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 26 Jun 2019 17:17:16 -0400 Subject: [PATCH 16/62] Changes from add-trailing-comma hook --- stix2/test/v21/test_malware_analysis.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index 5192317..42db919 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -1,11 +1,11 @@ import json + import pytest import stix2.exceptions import stix2.utils import stix2.v21 - MALWARE_ANALYSIS_JSON = """{ "type": "malware-analysis", "spec_version": "2.1", @@ -49,10 +49,12 @@ def test_malware_analysis_example(): assert str(ma) == MALWARE_ANALYSIS_JSON -@pytest.mark.parametrize("data", [ - MALWARE_ANALYSIS_JSON, - MALWARE_ANALYSIS_DICT -]) +@pytest.mark.parametrize( + "data", [ + MALWARE_ANALYSIS_JSON, + MALWARE_ANALYSIS_DICT, + ], +) def test_parse_malware_analysis(data): ma = stix2.parse(data, version="2.1") @@ -76,5 +78,5 @@ def test_parse_malware_analysis(data): def test_malware_analysis_constraint(): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError): stix2.v21.MalwareAnalysis( - product="Acme Malware Analyzer" + product="Acme Malware Analyzer", ) From de93a2ee321f354ed88f887ebff4c22071f66e14 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 25 Jun 2019 17:38:05 -0400 Subject: [PATCH 17/62] Fix stix2.1 course-of-action SDO class properties action_reference and action_bin to have the correct types. --- stix2/v21/sdo.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 522aeda..7f32cfa 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -8,9 +8,10 @@ from six.moves.urllib.parse import quote_plus from ..core import STIXDomainObject from ..custom import _custom_object_builder from ..properties import ( - BooleanProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, - ListProperty, ObservableProperty, PatternProperty, ReferenceProperty, - StringProperty, TimestampProperty, TypeProperty, + BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, + FloatProperty, IDProperty, IntegerProperty, ListProperty, + ObservableProperty, PatternProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty, ) from ..utils import NOW from .common import ExternalReference, GranularMarking, KillChainPhase @@ -101,8 +102,8 @@ class CourseOfAction(STIXDomainObject): ('description', StringProperty()), ('action_type', StringProperty()), ('os_execution_envs', ListProperty(StringProperty)), - ('action_bin', StringProperty()), - ('action_reference', StringProperty()), + ('action_bin', BinaryProperty()), + ('action_reference', EmbeddedObjectProperty(ExternalReference)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), From e779cacf3edc85ef34ac34524d46f214bf36cd89 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 26 Jun 2019 21:01:41 -0400 Subject: [PATCH 18/62] Update course of action tests, to include tests with the action_reference property. Also, stylistic changes to hopefully let it do more testing with less code. --- stix2/test/v21/test_course_of_action.py | 99 +++++++++++++------------ 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/stix2/test/v21/test_course_of_action.py b/stix2/test/v21/test_course_of_action.py index b86471a..44079fb 100644 --- a/stix2/test/v21/test_course_of_action.py +++ b/stix2/test/v21/test_course_of_action.py @@ -1,14 +1,12 @@ -import datetime as dt +import json import pytest -import pytz import stix2 import stix2.exceptions +import stix2.utils -from .constants import COURSE_OF_ACTION_ID, IDENTITY_ID - -EXPECTED = """{ +COA_WITH_BIN_JSON = """{ "type": "course-of-action", "spec_version": "2.1", "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", @@ -27,54 +25,63 @@ EXPECTED = """{ }""" -def test_course_of_action_example(): - coa = stix2.v21.CourseOfAction( - id=COURSE_OF_ACTION_ID, - created_by_ref=IDENTITY_ID, - created="2016-04-06T20:03:48.000Z", - modified="2016-04-06T20:03:48.000Z", - name="Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", - description="This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", - action_type="textual:text/plain", - os_execution_envs=["a", "b", "c"], - action_bin="aGVsbG8gd29ybGQ=", - ) +COA_WITH_REF_JSON = """{ + "type": "course-of-action", + "spec_version": "2.1", + "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c", + "created": "2016-04-06T20:03:48.000Z", + "modified": "2016-04-06T20:03:48.000Z", + "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", + "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", + "action_type": "textual:text/plain", + "os_execution_envs": [ + "a", + "b", + "c" + ], + "action_reference": { + "source_name": "a source", + "description": "description of a source" + } +}""" - assert str(coa) == EXPECTED + +COA_WITH_BIN_DICT = json.loads(COA_WITH_BIN_JSON) +COA_WITH_REF_DICT = json.loads(COA_WITH_REF_JSON) @pytest.mark.parametrize( - "data", [ - EXPECTED, - { - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": IDENTITY_ID, - "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", - "id": COURSE_OF_ACTION_ID, - "modified": "2016-04-06T20:03:48.000Z", - "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", - "spec_version": "2.1", - "type": "course-of-action", - "action_type": "textual:text/plain", - "os_execution_envs": ["a", "b", "c"], - "action_bin": "aGVsbG8gd29ybGQ=", - }, + "sdo_json,sdo_dict", [ + (COA_WITH_BIN_JSON, COA_WITH_BIN_DICT), + (COA_WITH_REF_JSON, COA_WITH_REF_DICT), ], ) -def test_parse_course_of_action(data): - coa = stix2.parse(data, version="2.1") +def test_course_of_action_example(sdo_json, sdo_dict): + coa = stix2.v21.CourseOfAction(**sdo_dict) + assert str(coa) == sdo_json - assert coa.type == 'course-of-action' - assert coa.spec_version == '2.1' - assert coa.id == COURSE_OF_ACTION_ID - assert coa.created == dt.datetime(2016, 4, 6, 20, 3, 48, tzinfo=pytz.utc) - assert coa.modified == dt.datetime(2016, 4, 6, 20, 3, 48, tzinfo=pytz.utc) - assert coa.created_by_ref == IDENTITY_ID - assert coa.description == "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ..." - assert coa.name == "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter" - assert coa.action_type == "textual:text/plain" - assert coa.os_execution_envs == ["a", "b", "c"] - assert coa.action_bin == "aGVsbG8gd29ybGQ=" + +@pytest.mark.parametrize( + "sdo_json,sdo_dict", [ + (COA_WITH_BIN_JSON, COA_WITH_BIN_DICT), + (COA_WITH_REF_JSON, COA_WITH_REF_DICT), + ], +) +def test_parse_course_of_action(sdo_json, sdo_dict): + + # Names of timestamp-valued attributes + ts_attrs = {"created", "modified"} + + for data in (sdo_json, sdo_dict): + coa = stix2.parse(data, version="2.1") + + # sdo_dict is handy as a source of attribute names/values to check + for attr_name, attr_value in sdo_dict.items(): + cmp_value = stix2.utils.parse_into_datetime(attr_value) \ + if attr_name in ts_attrs else attr_value + + assert getattr(coa, attr_name) == cmp_value def test_course_of_action_constraint(): From ef408e1971733f37e32af011850e1f87bcfebf71 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 1 Jul 2019 11:52:55 -0400 Subject: [PATCH 19/62] preliminary changes to make stix2 code conform to WD 04 specs --- stix2/v21/sdo.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 0717f88..0ae0c80 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -122,6 +122,34 @@ class CourseOfAction(STIXDomainObject): ) +class Grouping(STIXDomainObject): + # TODO: Add link + """For more detailed information on this object's properties, see + `the STIX 2.1 specification `__. + """ + + _type = 'grouping' + _properties = OrderedDict([ + ('type', TypeProperty(_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('name', StringProperty()), + ('description', StringProperty()), + ('context', StringProperty(required=True)), + ('object_refs', ListProperty(ReferenceProperty)), + ]) + + class Identity(STIXDomainObject): # TODO: Add link """For more detailed information on this object's properties, see @@ -193,6 +221,46 @@ class Indicator(STIXDomainObject): raise ValueError(msg.format(self)) +class Infrastructure(STIXDomainObject): + # TODO: Add link + """For more detailed information on this object's properties, see + `the STIX 2.1 specification `__. + """ + + _type = 'infrastructure' + _properties = OrderedDict([ + ('type', TypeProperty(_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('id', IDProperty(_type, spec_version='2.1')), + ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('revoked', BooleanProperty(default=lambda: False)), + ('labels', ListProperty(StringProperty)), + ('confidence', IntegerProperty()), + ('lang', StringProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('infrastructure_types', ListProperty(StringProperty, required=True)), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ]) + + def _check_object_constraints(self): + super(self.__class__, self)._check_object_constraints() + + first_seen = self.get('first_seen') + last_seen = self.get('last_seen') + + if first_seen and last_seen and last_seen < first_seen: + msg = "{0.id} 'last_seen' must be greater than or equal to 'first_seen'" + raise ValueError(msg.format(self)) + + class IntrusionSet(STIXDomainObject): # TODO: Add link """For more detailed information on this object's properties, see @@ -346,7 +414,16 @@ class Malware(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('malware_types', ListProperty(StringProperty, required=True)), + ('is_family', BooleanProperty(required=True)), + ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('os_execution_envs', ListProperty(StringProperty)), + ('architecture_execution_envs', ListProperty(StringProperty)), + ('implementation_languages', ListProperty(StringProperty)), + ('capabilities', ListProperty(StringProperty)), + ('sample_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -356,6 +433,16 @@ class Malware(STIXDomainObject): ('granular_markings', ListProperty(GranularMarking)), ]) + def _check_object_constraints(self): + super(self.__class__, self)._check_object_constraints() + + first_seen = self.get('first_seen') + last_seen = self.get('last_seen') + + if first_seen and last_seen and last_seen < first_seen: + msg = "{0.id} 'last_seen' must be greater than or equal to 'first_seen'" + raise ValueError(msg.format(self)) + class MalwareAnalysis(STIXDomainObject): # TODO: Add link From c98fcafb1ac1dc4dcd3883469c3775b14cd54845 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 1 Jul 2019 15:26:30 -0400 Subject: [PATCH 20/62] Update tests to address conformance to WD04 specs --- stix2/test/v21/constants.py | 3 +++ stix2/test/v21/test_bundle.py | 6 +++++- stix2/test/v21/test_core.py | 1 + stix2/test/v21/test_datastore_filters.py | 1 + stix2/test/v21/test_environment.py | 5 ++++- stix2/test/v21/test_indicator.py | 5 +++-- stix2/test/v21/test_malware.py | 10 +++++++--- stix2/test/v21/test_versioning.py | 1 + stix2/test/v21/test_workbench.py | 2 +- stix2/v21/sdo.py | 3 ++- 10 files changed, 28 insertions(+), 9 deletions(-) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index b0ba1ef..66cb956 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -78,6 +78,7 @@ IDENTITY_KWARGS = dict( INDICATOR_KWARGS = dict( indicator_types=['malicious-activity'], pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + valid_from="2017-01-01T12:34:56Z", ) INTRUSION_SET_KWARGS = dict( @@ -87,6 +88,7 @@ INTRUSION_SET_KWARGS = dict( MALWARE_KWARGS = dict( malware_types=['ransomware'], name="Cryptolocker", + is_family=False, ) MALWARE_MORE_KWARGS = dict( @@ -97,6 +99,7 @@ MALWARE_MORE_KWARGS = dict( malware_types=['ransomware'], name="Cryptolocker", description="A ransomware related to ...", + is_family=False, ) OBSERVED_DATA_KWARGS = dict( diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 47d0a7a..9b78215 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -31,7 +31,8 @@ EXPECTED_BUNDLE = """{ "name": "Cryptolocker", "malware_types": [ "ransomware" - ] + ], + "is_family": False }, { "type": "relationship", @@ -72,6 +73,7 @@ EXPECTED_BUNDLE_DICT = { "malware_types": [ "ransomware", ], + "is_family": False, }, { "type": "relationship", @@ -244,6 +246,7 @@ def test_bundle_obj_id_found(): "malware_types": [ "ransomware", ], + "is_family": False, }, { "type": "malware", @@ -255,6 +258,7 @@ def test_bundle_obj_id_found(): "malware_types": [ "ransomware", ], + "is_family": False, }, { "type": "relationship", diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index bf45f32..06a829c 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -31,6 +31,7 @@ BUNDLE = { "malware_types": [ "ransomware", ], + "is_family": False, }, { "type": "relationship", diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 4b9878a..cbe3fe4 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -16,6 +16,7 @@ stix_objs = [ "remote-access-trojan", ], "modified": "2017-01-27T13:49:53.997Z", + "is_family": False, "name": "Poison Ivy", "type": "malware", }, diff --git a/stix2/test/v21/test_environment.py b/stix2/test/v21/test_environment.py index e08971e..53ab744 100644 --- a/stix2/test/v21/test_environment.py +++ b/stix2/test/v21/test_environment.py @@ -219,7 +219,8 @@ def test_parse_malware(): "name": "Cryptolocker", "malware_types": [ "ransomware" - ] + ], + "is_family": False }""" mal = env.parse(data, version="2.1") @@ -230,6 +231,7 @@ def test_parse_malware(): assert mal.modified == FAKE_TIME assert mal.malware_types == ['ransomware'] assert mal.name == "Cryptolocker" + assert not mal.is_family def test_creator_of(): @@ -351,6 +353,7 @@ def test_related_to_no_id(ds): mal = { "type": "malware", "name": "some variant", + "is_family": False, } with pytest.raises(ValueError) as excinfo: env.related_to(mal) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 49bc6e0..d061104 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -98,7 +98,7 @@ def test_indicator_required_properties(): stix2.v21.Indicator() assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["indicator_types", "pattern"] + assert excinfo.value.properties == ["indicator_types", "pattern", "valid_from"] assert str(excinfo.value) == "No values for required properties for Indicator: (indicator_types, pattern)." @@ -107,7 +107,7 @@ def test_indicator_required_property_pattern(): stix2.v21.Indicator(indicator_types=['malicious-activity']) assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["pattern"] + assert excinfo.value.properties == ["pattern", "valid_from"] def test_indicator_created_ref_invalid_format(): @@ -184,6 +184,7 @@ def test_invalid_indicator_pattern(): stix2.v21.Indicator( indicator_types=['malicious-activity'], pattern="file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e'", + valid_from="2017-01-01T12:34:56Z", ) assert excinfo.value.cls == stix2.v21.Indicator assert excinfo.value.prop_name == 'pattern' diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index c55bfa9..e6dfc2e 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -17,7 +17,8 @@ EXPECTED_MALWARE = """{ "name": "Cryptolocker", "malware_types": [ "ransomware" - ] + ], + "is_family": False }""" @@ -31,6 +32,7 @@ def test_malware_with_all_required_properties(): modified=now, malware_types=["ransomware"], name="Cryptolocker", + is_family=False, ) assert str(mal) == EXPECTED_MALWARE @@ -77,7 +79,7 @@ def test_malware_required_properties(): stix2.v21.Malware() assert excinfo.value.cls == stix2.v21.Malware - assert excinfo.value.properties == ["malware_types", "name"] + assert excinfo.value.properties == ["is_family", "malware_types", "name"] def test_malware_required_property_name(): @@ -85,7 +87,7 @@ def test_malware_required_property_name(): stix2.v21.Malware(malware_types=['ransomware']) assert excinfo.value.cls == stix2.v21.Malware - assert excinfo.value.properties == ["name"] + assert excinfo.value.properties == ["is_family", "name"] def test_cannot_assign_to_malware_attributes(malware): @@ -115,6 +117,7 @@ def test_invalid_kwarg_to_malware(): "modified": "2016-05-12T08:17:27.000Z", "malware_types": ["ransomware"], "name": "Cryptolocker", + "is_family": False, }, ], ) @@ -128,6 +131,7 @@ def test_parse_malware(data): assert mal.modified == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) assert mal.malware_types == ['ransomware'] assert mal.name == 'Cryptolocker' + assert not mal.is_family def test_parse_malware_invalid_labels(): diff --git a/stix2/test/v21/test_versioning.py b/stix2/test/v21/test_versioning.py index a7f4a2f..c46183c 100644 --- a/stix2/test/v21/test_versioning.py +++ b/stix2/test/v21/test_versioning.py @@ -230,6 +230,7 @@ def test_remove_custom_stix_property(): malware_types=["rootkit"], x_custom="armada", allow_custom=True, + is_family=False, ) mal_nc = stix2.utils.remove_custom_stix(mal) diff --git a/stix2/test/v21/test_workbench.py b/stix2/test/v21/test_workbench.py index 0a976d7..0d84422 100644 --- a/stix2/test/v21/test_workbench.py +++ b/stix2/test/v21/test_workbench.py @@ -199,7 +199,7 @@ def test_workbench_related(): def test_workbench_related_with_filters(): malware = Malware( malware_types=["ransomware"], name="CryptorBit", - created_by_ref=IDENTITY_ID, + created_by_ref=IDENTITY_ID, is_family=False, ) rel = Relationship(malware.id, 'variant-of', MALWARE_ID) save([malware, rel]) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 0ae0c80..8ec4131 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -198,7 +198,7 @@ class Indicator(STIXDomainObject): ('description', StringProperty()), ('indicator_types', ListProperty(StringProperty, required=True)), ('pattern', PatternProperty(required=True)), - ('valid_from', TimestampProperty(default=lambda: NOW)), + ('valid_from', TimestampProperty(default=lambda: NOW, required=True)), ('valid_until', TimestampProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), ('revoked', BooleanProperty(default=lambda: False)), @@ -683,6 +683,7 @@ class Tool(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('tool_types', ListProperty(StringProperty, required=True)), + ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('tool_version', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), From ffbf5fa34cf26a84e03d3c704c934c22d2eb63b8 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 1 Jul 2019 15:41:44 -0400 Subject: [PATCH 21/62] Fix JSON encoding issue within tests --- stix2/test/v21/test_bundle.py | 2 +- stix2/test/v21/test_environment.py | 2 +- stix2/test/v21/test_indicator.py | 3 ++- stix2/test/v21/test_malware.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 9b78215..a884b1f 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -32,7 +32,7 @@ EXPECTED_BUNDLE = """{ "malware_types": [ "ransomware" ], - "is_family": False + "is_family": "False" }, { "type": "relationship", diff --git a/stix2/test/v21/test_environment.py b/stix2/test/v21/test_environment.py index 53ab744..6887e41 100644 --- a/stix2/test/v21/test_environment.py +++ b/stix2/test/v21/test_environment.py @@ -220,7 +220,7 @@ def test_parse_malware(): "malware_types": [ "ransomware" ], - "is_family": False + "is_family": "False" }""" mal = env.parse(data, version="2.1") diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index d061104..b68b887 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -99,7 +99,7 @@ def test_indicator_required_properties(): assert excinfo.value.cls == stix2.v21.Indicator assert excinfo.value.properties == ["indicator_types", "pattern", "valid_from"] - assert str(excinfo.value) == "No values for required properties for Indicator: (indicator_types, pattern)." + assert str(excinfo.value) == "No values for required properties for Indicator: (indicator_types, pattern, valid_from)." def test_indicator_required_property_pattern(): @@ -194,6 +194,7 @@ def test_invalid_indicator_pattern(): stix2.v21.Indicator( indicator_types=['malicious-activity'], pattern='[file:hashes.MD5 = "d41d8cd98f00b204e9800998ecf8427e"]', + valid_from="2017-01-01T12:34:56Z", ) assert excinfo.value.cls == stix2.v21.Indicator assert excinfo.value.prop_name == 'pattern' diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index e6dfc2e..bf5b538 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -18,7 +18,7 @@ EXPECTED_MALWARE = """{ "malware_types": [ "ransomware" ], - "is_family": False + "is_family": "False" }""" From ae35d2ab01146942e3b29ad9d3747830ba3ac85e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Tue, 2 Jul 2019 13:17:43 -0400 Subject: [PATCH 22/62] Add and update tests to conform code to WD04 SDO specs --- stix2/test/v21/conftest.py | 13 +- stix2/test/v21/constants.py | 12 ++ ...-6b616fc1-1505-48e3-8b2c-0d19337bff38.json | 3 +- .../20170531213258226477.json | 3 +- .../20181101232448456000.json | 3 +- .../20181101232448457000.json | 3 +- .../20170531213326565056.json | 3 +- .../20170531213248482655.json | 3 +- .../20170531213215263882.json | 3 +- stix2/test/v21/test_bundle.py | 2 +- stix2/test/v21/test_environment.py | 2 +- stix2/test/v21/test_grouping.py | 112 +++++++++++++ stix2/test/v21/test_infrastructure.py | 158 ++++++++++++++++++ stix2/test/v21/test_malware.py | 9 +- stix2/v21/__init__.py | 9 +- 15 files changed, 324 insertions(+), 14 deletions(-) create mode 100644 stix2/test/v21/test_grouping.py create mode 100644 stix2/test/v21/test_infrastructure.py diff --git a/stix2/test/v21/conftest.py b/stix2/test/v21/conftest.py index dea29ca..ea2853d 100644 --- a/stix2/test/v21/conftest.py +++ b/stix2/test/v21/conftest.py @@ -5,7 +5,8 @@ import pytest import stix2 from .constants import ( - FAKE_TIME, INDICATOR_KWARGS, MALWARE_KWARGS, RELATIONSHIP_KWARGS, + FAKE_TIME, GROUPING_KWARGS, INDICATOR_KWARGS, INFRASTRUCTURE_KWARGS, + MALWARE_KWARGS, RELATIONSHIP_KWARGS, ) @@ -39,6 +40,16 @@ def indicator(uuid4, clock): return stix2.v21.Indicator(**INDICATOR_KWARGS) +@pytest.fixture +def infrastructure(uuid4, clock): + return stix2.v21.Infrastructure(**INFRASTRUCTURE_KWARGS) + + +@pytest.fixture +def grouping(uuid4, clock): + return stix2.v21.Grouping(**GROUPING_KWARGS) + + @pytest.fixture def malware(uuid4, clock): return stix2.v21.Malware(**MALWARE_KWARGS) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index 66cb956..40a2bb5 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -7,8 +7,10 @@ FAKE_TIME = dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) ATTACK_PATTERN_ID = "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" CAMPAIGN_ID = "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" COURSE_OF_ACTION_ID = "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" +GROUPING_ID = "grouping--753abcde-3141-5926-ace5-0a810b1ff996" IDENTITY_ID = "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" INDICATOR_ID = "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7" +INFRASTRUCTURE_ID = "infrastructure--3000ae1b-784c-f03d-8abc-0a625b2ff018" INTRUSION_SET_ID = "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29" LOCATION_ID = "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64" MALWARE_ID = "malware--9c4638ec-f1de-4ddb-abf4-1b760417654e" @@ -70,6 +72,11 @@ COURSE_OF_ACTION_KWARGS = dict( name="Block", ) +GROUPING_KWARGS = dict( + name="Harry Potter and the Leet Hackers", + context="suspicious-activity", +) + IDENTITY_KWARGS = dict( name="John Smith", identity_class="individual", @@ -81,6 +88,11 @@ INDICATOR_KWARGS = dict( valid_from="2017-01-01T12:34:56Z", ) +INFRASTRUCTURE_KWARGS = dict( + name="Poison Ivy C2", + infrastructure_types=["command-and-control"], +) + INTRUSION_SET_KWARGS = dict( name="Bobcat Breakin", ) diff --git a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json index 54343ce..23e28bb 100644 --- a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json +++ b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json @@ -24,5 +24,6 @@ ], "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + ], + "is_family": false } diff --git a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json index 1bedc5b..f65449d 100644 --- a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json +++ b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json @@ -27,7 +27,8 @@ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" ], "spec_version": "2.1", - "type": "malware" + "type": "malware", + "is_family": false } ], "type": "bundle" diff --git a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json index 4236920..1b22cf2 100644 --- a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json +++ b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json @@ -24,5 +24,6 @@ ], "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + ], + "is_family": false } diff --git a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json index 37dd9c5..7802c50 100644 --- a/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json +++ b/stix2/test/v21/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json @@ -24,5 +24,6 @@ ], "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] + ], + "is_family": false } diff --git a/stix2/test/v21/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json b/stix2/test/v21/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json index 0b7c01e..24f3837 100644 --- a/stix2/test/v21/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json +++ b/stix2/test/v21/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json @@ -27,7 +27,8 @@ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" ], "spec_version": "2.1", - "type": "malware" + "type": "malware", + "is_family": false } ], "type": "bundle" diff --git a/stix2/test/v21/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json b/stix2/test/v21/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json index 195c973..8495bfe 100644 --- a/stix2/test/v21/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json +++ b/stix2/test/v21/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json @@ -27,7 +27,8 @@ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" ], "spec_version": "2.1", - "type": "malware" + "type": "malware", + "is_family": false } ], "type": "bundle" diff --git a/stix2/test/v21/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json b/stix2/test/v21/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json index 4d57db5..a509a5e 100644 --- a/stix2/test/v21/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json +++ b/stix2/test/v21/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json @@ -26,7 +26,8 @@ "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" ], - "type": "malware" + "type": "malware", + "is_family": false } ], "spec_version": "2.0", diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index a884b1f..7ba0729 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -32,7 +32,7 @@ EXPECTED_BUNDLE = """{ "malware_types": [ "ransomware" ], - "is_family": "False" + "is_family": false }, { "type": "relationship", diff --git a/stix2/test/v21/test_environment.py b/stix2/test/v21/test_environment.py index 6887e41..90f31cb 100644 --- a/stix2/test/v21/test_environment.py +++ b/stix2/test/v21/test_environment.py @@ -220,7 +220,7 @@ def test_parse_malware(): "malware_types": [ "ransomware" ], - "is_family": "False" + "is_family": false }""" mal = env.parse(data, version="2.1") diff --git a/stix2/test/v21/test_grouping.py b/stix2/test/v21/test_grouping.py new file mode 100644 index 0000000..405a80c --- /dev/null +++ b/stix2/test/v21/test_grouping.py @@ -0,0 +1,112 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +from .constants import FAKE_TIME, GROUPING_ID, GROUPING_KWARGS + +EXPECTED_GROUPING = """{ + "type": "grouping", + "spec_version": "2.1", + "id": "grouping--753abcde-3141-5926-ace5-0a810b1ff996", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Harry Potter and the Leet Hackers", + "context": "suspicious-activity" +}""" + + +def test_grouping_with_all_required_properties(): + now = dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + + grp = stix2.v21.Grouping( + type="grouping", + id=GROUPING_ID, + created=now, + modified=now, + name="Harry Potter and the Leet Hackers", + context="suspicious-activity", + ) + + assert str(grp) == EXPECTED_GROUPING + + +def test_grouping_autogenerated_properties(grouping): + assert grouping.type == 'grouping' + assert grouping.id == 'grouping--00000000-0000-4000-8000-000000000001' + assert grouping.created == FAKE_TIME + assert grouping.modified == FAKE_TIME + assert grouping.name == "Harry Potter and the Leet Hackers" + assert grouping.context == "suspicious-activity" + + assert grouping['type'] == 'grouping' + assert grouping['id'] == 'grouping--00000000-0000-4000-8000-000000000001' + assert grouping['created'] == FAKE_TIME + assert grouping['modified'] == FAKE_TIME + assert grouping['name'] == "Harry Potter and the Leet Hackers" + assert grouping['context'] == "suspicious-activity" + + +def test_grouping_type_must_be_grouping(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Grouping(type='xxx', **GROUPING_KWARGS) + + assert excinfo.value.cls == stix2.v21.Grouping + assert excinfo.value.prop_name == "type" + assert excinfo.value.reason == "must equal 'grouping'." + assert str(excinfo.value) == "Invalid value for Grouping 'type': must equal 'grouping'." + + +def test_grouping_id_must_start_with_grouping(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Grouping(id='my-prefix--', **GROUPING_KWARGS) + + assert excinfo.value.cls == stix2.v21.Grouping + assert excinfo.value.prop_name == "id" + assert excinfo.value.reason == "must start with 'grouping--'." + assert str(excinfo.value) == "Invalid value for Grouping 'id': must start with 'grouping--'." + + +def test_grouping_required_properties(): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: + stix2.v21.Grouping() + + assert excinfo.value.cls == stix2.v21.Grouping + assert excinfo.value.properties == ["context"] + + +def test_invalid_kwarg_to_grouping(): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: + stix2.v21.Grouping(my_custom_property="foo", **GROUPING_KWARGS) + + assert excinfo.value.cls == stix2.v21.Grouping + assert excinfo.value.properties == ['my_custom_property'] + assert str(excinfo.value) == "Unexpected properties for Grouping: (my_custom_property)." + + +@pytest.mark.parametrize( + "data", [ + EXPECTED_GROUPING, + { + "type": "grouping", + "spec_version": "2.1", + "id": GROUPING_ID, + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Harry Potter and the Leet Hackers", + "context": "suspicious-activity", + }, + ], +) +def test_parse_grouping(data): + grp = stix2.parse(data) + + assert grp.type == 'grouping' + assert grp.spec_version == '2.1' + assert grp.id == GROUPING_ID + assert grp.created == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + assert grp.modified == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + assert grp.name == "Harry Potter and the Leet Hackers" + assert grp.context == "suspicious-activity" diff --git a/stix2/test/v21/test_infrastructure.py b/stix2/test/v21/test_infrastructure.py new file mode 100644 index 0000000..30632bb --- /dev/null +++ b/stix2/test/v21/test_infrastructure.py @@ -0,0 +1,158 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +from .constants import FAKE_TIME, INFRASTRUCTURE_ID, INFRASTRUCTURE_KWARGS + +EXPECTED_INFRASTRUCTURE = """{ + "type": "infrastructure", + "spec_version": "2.1", + "id": "infrastructure--3000ae1b-784c-f03d-8abc-0a625b2ff018", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Poison Ivy C2", + "infrastructure_types": [ + "command-and-control" + ] +}""" + + +def test_infrastructure_with_all_required_properties(): + now = dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + + infra = stix2.v21.Infrastructure( + type="infrastructure", + id=INFRASTRUCTURE_ID, + created=now, + modified=now, + name="Poison Ivy C2", + infrastructure_types=["command-and-control"], + ) + + assert str(infra) == EXPECTED_INFRASTRUCTURE + + +def test_infrastructure_autogenerated_properties(infrastructure): + assert infrastructure.type == 'infrastructure' + assert infrastructure.id == 'infrastructure--00000000-0000-4000-8000-000000000001' + assert infrastructure.created == FAKE_TIME + assert infrastructure.modified == FAKE_TIME + assert infrastructure.infrastructure_types == ['command-and-control'] + assert infrastructure.name == "Poison Ivy C2" + + assert infrastructure['type'] == 'infrastructure' + assert infrastructure['id'] == 'infrastructure--00000000-0000-4000-8000-000000000001' + assert infrastructure['created'] == FAKE_TIME + assert infrastructure['modified'] == FAKE_TIME + assert infrastructure['infrastructure_types'] == ['command-and-control'] + assert infrastructure['name'] == "Poison Ivy C2" + + +def test_infrastructure_type_must_be_infrastructure(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Infrastructure(type='xxx', **INFRASTRUCTURE_KWARGS) + + assert excinfo.value.cls == stix2.v21.Infrastructure + assert excinfo.value.prop_name == "type" + assert excinfo.value.reason == "must equal 'infrastructure'." + assert str(excinfo.value) == "Invalid value for Infrastructure 'type': must equal 'infrastructure'." + + +def test_infrastructure_id_must_start_with_infrastructure(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Infrastructure(id='my-prefix--', **INFRASTRUCTURE_KWARGS) + + assert excinfo.value.cls == stix2.v21.Infrastructure + assert excinfo.value.prop_name == "id" + assert excinfo.value.reason == "must start with 'infrastructure--'." + assert str(excinfo.value) == "Invalid value for Infrastructure 'id': must start with 'infrastructure--'." + + +def test_infrastructure_required_properties(): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: + stix2.v21.Infrastructure() + + assert excinfo.value.cls == stix2.v21.Infrastructure + assert excinfo.value.properties == ["infrastructure_types", "name"] + + +def test_infrastructure_required_property_name(): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: + stix2.v21.Infrastructure(infrastructure_types=['command-and-control']) + + assert excinfo.value.cls == stix2.v21.Infrastructure + assert excinfo.value.properties == ["name"] + + +def test_invalid_kwarg_to_infrastructure(): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: + stix2.v21.Infrastructure(my_custom_property="foo", **INFRASTRUCTURE_KWARGS) + + assert excinfo.value.cls == stix2.v21.Infrastructure + assert excinfo.value.properties == ['my_custom_property'] + assert str(excinfo.value) == "Unexpected properties for Infrastructure: (my_custom_property)." + + +@pytest.mark.parametrize( + "data", [ + EXPECTED_INFRASTRUCTURE, + { + "type": "infrastructure", + "spec_version": "2.1", + "id": INFRASTRUCTURE_ID, + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "infrastructure_types": ["command-and-control"], + "name": "Poison Ivy C2", + }, + ], +) +def test_parse_infrastructure(data): + infra = stix2.parse(data) + + assert infra.type == 'infrastructure' + assert infra.spec_version == '2.1' + assert infra.id == INFRASTRUCTURE_ID + assert infra.created == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + assert infra.modified == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + assert infra.infrastructure_types == ['command-and-control'] + assert infra.name == 'Poison Ivy C2' + + +def test_parse_infrastructure_kill_chain_phases(): + kill_chain = """ + "kill_chain_phases": [ + { + "kill_chain_name": "lockheed-martin-cyber-kill-chain", + "phase_name": "reconnaissance" + } + ]""" + data = EXPECTED_INFRASTRUCTURE.replace('infrastructure"', 'infrastructure",%s' % kill_chain) + infra = stix2.parse(data, version="2.1") + assert infra.kill_chain_phases[0].kill_chain_name == "lockheed-martin-cyber-kill-chain" + assert infra.kill_chain_phases[0].phase_name == "reconnaissance" + assert infra['kill_chain_phases'][0]['kill_chain_name'] == "lockheed-martin-cyber-kill-chain" + assert infra['kill_chain_phases'][0]['phase_name'] == "reconnaissance" + + +def test_parse_infrastructure_clean_kill_chain_phases(): + kill_chain = """ + "kill_chain_phases": [ + { + "kill_chain_name": "lockheed-martin-cyber-kill-chain", + "phase_name": 1 + } + ]""" + data = EXPECTED_INFRASTRUCTURE.replace('2.1"', '2.1",%s' % kill_chain) + infra = stix2.parse(data, version="2.1") + assert infra['kill_chain_phases'][0]['phase_name'] == "1" + + +def test_infrastructure_invalid_last_before_first(): + with pytest.raises(ValueError) as excinfo: + stix2.v21.Infrastructure(first_seen="2017-01-01T12:34:56.000Z", last_seen="2017-01-01T12:33:56.000Z", **INFRASTRUCTURE_KWARGS) + + assert "'last_seen' must be greater than or equal to 'first_seen'" in str(excinfo.value) diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index bf5b538..1817c63 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -18,7 +18,7 @@ EXPECTED_MALWARE = """{ "malware_types": [ "ransomware" ], - "is_family": "False" + "is_family": false }""" @@ -168,3 +168,10 @@ def test_parse_malware_clean_kill_chain_phases(): data = EXPECTED_MALWARE.replace('2.1"', '2.1",%s' % kill_chain) mal = stix2.parse(data, version="2.1") assert mal['kill_chain_phases'][0]['phase_name'] == "1" + + +def test_malware_invalid_last_before_first(): + with pytest.raises(ValueError) as excinfo: + stix2.v21.Malware(first_seen="2017-01-01T12:34:56.000Z", last_seen="2017-01-01T12:33:56.000Z", **MALWARE_KWARGS) + + assert "'last_seen' must be greater than or equal to 'first_seen'" in str(excinfo.value) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index efac071..b2451d2 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -32,9 +32,10 @@ from .observables import ( X509Certificate, X509V3ExtenstionsType, ) from .sdo import ( - AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, - IntrusionSet, Location, Malware, MalwareAnalysis, Note, ObservedData, - Opinion, Report, ThreatActor, Tool, Vulnerability, + AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, + Indicator, Infrastructure, IntrusionSet, Location, Malware, + MalwareAnalysis, Note, ObservedData, Opinion, Report, ThreatActor, Tool, + Vulnerability, ) from .sro import Relationship, Sighting @@ -43,8 +44,10 @@ OBJ_MAP = { 'bundle': Bundle, 'campaign': Campaign, 'course-of-action': CourseOfAction, + 'grouping': Grouping, 'identity': Identity, 'indicator': Indicator, + 'infrastructure': Infrastructure, 'intrusion-set': IntrusionSet, 'language-content': LanguageContent, 'location': Location, From b464a9cc0ab82b9ef5c491a0c9e92e7fb005d895 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Tue, 9 Jul 2019 13:34:19 -0400 Subject: [PATCH 23/62] Remove certain human message assertions from test suites --- stix2/test/v20/test_datastore_filesystem.py | 9 ++--- stix2/test/v20/test_pattern_expressions.py | 39 +++++++-------------- stix2/test/v21/test_datastore_filesystem.py | 6 ++-- stix2/test/v21/test_pattern_expressions.py | 39 +++++++-------------- 4 files changed, 31 insertions(+), 62 deletions(-) diff --git a/stix2/test/v20/test_datastore_filesystem.py b/stix2/test/v20/test_datastore_filesystem.py index 25de37e..317f927 100644 --- a/stix2/test/v20/test_datastore_filesystem.py +++ b/stix2/test/v20/test_datastore_filesystem.py @@ -125,15 +125,13 @@ def rel_fs_store(): def test_filesystem_source_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FileSystemSource('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) def test_filesystem_sink_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FileSystemSink('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) def test_filesystem_source_bad_json_file(fs_source, bad_json_files): @@ -441,9 +439,8 @@ def test_filesystem_attempt_stix_file_overwrite(fs_store): ) # Now attempt to overwrite the existing file - with pytest.raises(DataSourceError) as excinfo: + with pytest.raises(DataSourceError): fs_store.add(camp8) - assert "Attempted to overwrite file" in str(excinfo) os.remove(filepath) diff --git a/stix2/test/v20/test_pattern_expressions.py b/stix2/test/v20/test_pattern_expressions.py index 3dc7cde..23a401b 100644 --- a/stix2/test/v20/test_pattern_expressions.py +++ b/stix2/test/v20/test_pattern_expressions.py @@ -257,7 +257,7 @@ def test_and_observable_expression(): def test_invalid_and_observable_expression(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.AndBooleanExpression([ stix2.EqualityComparisonExpression( "user-account:display_name", @@ -268,7 +268,6 @@ def test_invalid_and_observable_expression(): stix2.StringConstant("admin"), ), ]) - assert "All operands to an 'AND' expression must have the same object type" in str(excinfo) def test_hex(): @@ -352,30 +351,26 @@ def test_list2(): def test_invalid_constant_type(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.EqualityComparisonExpression( "artifact:payload_bin", {'foo': 'bar'}, ) - assert 'Unable to create a constant' in str(excinfo) def test_invalid_integer_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.IntegerConstant('foo') - assert 'must be an integer' in str(excinfo) def test_invalid_timestamp_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.TimestampConstant('foo') - assert 'Must be a datetime object or timestamp string' in str(excinfo) def test_invalid_float_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FloatConstant('foo') - assert 'must be a float' in str(excinfo) @pytest.mark.parametrize( @@ -400,9 +395,8 @@ def test_boolean_constant(data, result): def test_invalid_boolean_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.BooleanConstant('foo') - assert 'must be a boolean' in str(excinfo) @pytest.mark.parametrize( @@ -412,21 +406,18 @@ def test_invalid_boolean_constant(): ], ) def test_invalid_hash_constant(hashtype, data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.HashConstant(data, hashtype) - assert 'is not a valid {} hash'.format(hashtype) in str(excinfo) def test_invalid_hex_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.HexConstant('mm') - assert "must contain an even number of hexadecimal characters" in str(excinfo) def test_invalid_binary_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.BinaryConstant('foo') - assert 'must contain a base64' in str(excinfo) def test_escape_quotes_and_backslashes(): @@ -459,15 +450,13 @@ def test_repeat_qualifier(): def test_invalid_repeat_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.RepeatQualifier('foo') - assert 'is not a valid argument for a Repeat Qualifier' in str(excinfo) def test_invalid_within_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.WithinQualifier('foo') - assert 'is not a valid argument for a Within Qualifier' in str(excinfo) def test_startstop_qualifier(): @@ -485,19 +474,17 @@ def test_startstop_qualifier(): def test_invalid_startstop_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.StartStopQualifier( 'foo', stix2.TimestampConstant('2016-06-01T00:00:00Z'), ) - assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.StartStopQualifier( datetime.date(2016, 6, 1), 'foo', ) - assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) def test_make_constant_already_a_constant(): diff --git a/stix2/test/v21/test_datastore_filesystem.py b/stix2/test/v21/test_datastore_filesystem.py index 34b1088..9917ccd 100644 --- a/stix2/test/v21/test_datastore_filesystem.py +++ b/stix2/test/v21/test_datastore_filesystem.py @@ -124,15 +124,13 @@ def rel_fs_store(): def test_filesystem_source_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FileSystemSource('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) def test_filesystem_sink_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FileSystemSink('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) def test_filesystem_source_bad_json_file(fs_source, bad_json_files): diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 3dc7cde..23a401b 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -257,7 +257,7 @@ def test_and_observable_expression(): def test_invalid_and_observable_expression(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.AndBooleanExpression([ stix2.EqualityComparisonExpression( "user-account:display_name", @@ -268,7 +268,6 @@ def test_invalid_and_observable_expression(): stix2.StringConstant("admin"), ), ]) - assert "All operands to an 'AND' expression must have the same object type" in str(excinfo) def test_hex(): @@ -352,30 +351,26 @@ def test_list2(): def test_invalid_constant_type(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.EqualityComparisonExpression( "artifact:payload_bin", {'foo': 'bar'}, ) - assert 'Unable to create a constant' in str(excinfo) def test_invalid_integer_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.IntegerConstant('foo') - assert 'must be an integer' in str(excinfo) def test_invalid_timestamp_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.TimestampConstant('foo') - assert 'Must be a datetime object or timestamp string' in str(excinfo) def test_invalid_float_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.FloatConstant('foo') - assert 'must be a float' in str(excinfo) @pytest.mark.parametrize( @@ -400,9 +395,8 @@ def test_boolean_constant(data, result): def test_invalid_boolean_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.BooleanConstant('foo') - assert 'must be a boolean' in str(excinfo) @pytest.mark.parametrize( @@ -412,21 +406,18 @@ def test_invalid_boolean_constant(): ], ) def test_invalid_hash_constant(hashtype, data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.HashConstant(data, hashtype) - assert 'is not a valid {} hash'.format(hashtype) in str(excinfo) def test_invalid_hex_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.HexConstant('mm') - assert "must contain an even number of hexadecimal characters" in str(excinfo) def test_invalid_binary_constant(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.BinaryConstant('foo') - assert 'must contain a base64' in str(excinfo) def test_escape_quotes_and_backslashes(): @@ -459,15 +450,13 @@ def test_repeat_qualifier(): def test_invalid_repeat_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.RepeatQualifier('foo') - assert 'is not a valid argument for a Repeat Qualifier' in str(excinfo) def test_invalid_within_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.WithinQualifier('foo') - assert 'is not a valid argument for a Within Qualifier' in str(excinfo) def test_startstop_qualifier(): @@ -485,19 +474,17 @@ def test_startstop_qualifier(): def test_invalid_startstop_qualifier(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.StartStopQualifier( 'foo', stix2.TimestampConstant('2016-06-01T00:00:00Z'), ) - assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): stix2.StartStopQualifier( datetime.date(2016, 6, 1), 'foo', ) - assert 'is not a valid argument for a Start/Stop Qualifier' in str(excinfo) def test_make_constant_already_a_constant(): From 1b7abaf2285663762dfe016a8da7412f1b58042b Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Sun, 14 Jul 2019 15:34:31 -0400 Subject: [PATCH 24/62] WIP: updating objects to be compliant with stix2.1 WD05. This includes SDO/SRO class updates, but no unit test updates. The class updates broke unit tests, so that still needs to be addressed. --- stix2/exceptions.py | 130 ++++++++++++++++++++++---------------------- stix2/v21/sdo.py | 46 ++++++++++++++-- stix2/v21/sro.py | 1 + 3 files changed, 109 insertions(+), 68 deletions(-) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index f1f1c09..946300c 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -19,37 +19,87 @@ class InvalidValueError(STIXError, ValueError): return msg.format(self) -class MissingPropertiesError(STIXError, ValueError): +class InvalidPropertyConfigurationError(STIXError, ValueError): + """ + Represents an invalid combination of properties on a STIX object. This + class can be used directly when the object requirements are more + complicated and none of the more specific exception subclasses apply. + """ + def __init__(self, message, cls): + super(InvalidPropertyConfigurationError, self).__init__(message) + self.cls = cls + + +class MissingPropertiesError(InvalidPropertyConfigurationError): """Missing one or more required properties when constructing STIX object.""" def __init__(self, cls, properties): - super(MissingPropertiesError, self).__init__() - self.cls = cls - self.properties = sorted(list(properties)) + self.properties = sorted(properties) - def __str__(self): - msg = "No values for required properties for {0}: ({1})." - return msg.format( - self.cls.__name__, + msg = "No values for required properties for {0}: ({1}).".format( + cls.__name__, ", ".join(x for x in self.properties), ) + super(MissingPropertiesError, self).__init__(msg, cls) -class ExtraPropertiesError(STIXError, TypeError): + +class ExtraPropertiesError(InvalidPropertyConfigurationError): """One or more extra properties were provided when constructing STIX object.""" def __init__(self, cls, properties): - super(ExtraPropertiesError, self).__init__() - self.cls = cls - self.properties = sorted(list(properties)) + self.properties = sorted(properties) - def __str__(self): - msg = "Unexpected properties for {0}: ({1})." - return msg.format( - self.cls.__name__, + msg = "Unexpected properties for {0}: ({1}).".format( + cls.__name__, ", ".join(x for x in self.properties), ) + super(ExtraPropertiesError, self).__init__(msg, cls) + + +class MutuallyExclusivePropertiesError(InvalidPropertyConfigurationError): + """Violating interproperty mutually exclusive constraint of a STIX object type.""" + + def __init__(self, cls, properties): + self.properties = sorted(properties) + + msg = "The ({1}) properties for {0} are mutually exclusive.".format( + cls.__name__, + ", ".join(x for x in self.properties), + ) + + super(MutuallyExclusivePropertiesError, self).__init__(msg, cls) + + +class DependentPropertiesError(InvalidPropertyConfigurationError): + """Violating interproperty dependency constraint of a STIX object type.""" + + def __init__(self, cls, dependencies): + self.dependencies = dependencies + + msg = "The property dependencies for {0}: ({1}) are not met.".format( + cls.__name__, + ", ".join(name for x in self.dependencies for name in x), + ) + + super(DependentPropertiesError, self).__init__(msg, cls) + + +class AtLeastOnePropertyError(InvalidPropertyConfigurationError): + """Violating a constraint of a STIX object type that at least one of the given properties must be populated.""" + + def __init__(self, cls, properties): + self.properties = sorted(properties) + + msg = "At least one of the ({1}) properties for {0} must be " \ + "populated.".format( + cls.__name__, + ", ".join(x for x in self.properties), + ) + + super(AtLeastOnePropertyError, self).__init__(msg, cls) + class ImmutableError(STIXError, ValueError): """Attempted to modify an object after creation.""" @@ -103,54 +153,6 @@ class UnmodifiablePropertyError(STIXError, ValueError): return msg.format(", ".join(self.unchangable_properties)) -class MutuallyExclusivePropertiesError(STIXError, TypeError): - """Violating interproperty mutually exclusive constraint of a STIX object type.""" - - def __init__(self, cls, properties): - super(MutuallyExclusivePropertiesError, self).__init__() - self.cls = cls - self.properties = sorted(list(properties)) - - def __str__(self): - msg = "The ({1}) properties for {0} are mutually exclusive." - return msg.format( - self.cls.__name__, - ", ".join(x for x in self.properties), - ) - - -class DependentPropertiesError(STIXError, TypeError): - """Violating interproperty dependency constraint of a STIX object type.""" - - def __init__(self, cls, dependencies): - super(DependentPropertiesError, self).__init__() - self.cls = cls - self.dependencies = dependencies - - def __str__(self): - msg = "The property dependencies for {0}: ({1}) are not met." - return msg.format( - self.cls.__name__, - ", ".join(name for x in self.dependencies for name in x), - ) - - -class AtLeastOnePropertyError(STIXError, TypeError): - """Violating a constraint of a STIX object type that at least one of the given properties must be populated.""" - - def __init__(self, cls, properties): - super(AtLeastOnePropertyError, self).__init__() - self.cls = cls - self.properties = sorted(list(properties)) - - def __str__(self): - msg = "At least one of the ({1}) properties for {0} must be populated." - return msg.format( - self.cls.__name__, - ", ".join(x for x in self.properties), - ) - - class RevokeError(STIXError, ValueError): """Attempted to an operation on a revoked object.""" diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 8ec4131..66000c1 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -7,6 +7,7 @@ from six.moves.urllib.parse import quote_plus from ..core import STIXDomainObject from ..custom import _custom_object_builder +from ..exceptions import InvalidPropertyConfigurationError from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -33,6 +34,7 @@ class AttackPattern(STIXDomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -146,7 +148,7 @@ class Grouping(STIXDomainObject): ('name', StringProperty()), ('description', StringProperty()), ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty)), + ('object_refs', ListProperty(ReferenceProperty, required=True)), ]) @@ -198,6 +200,8 @@ class Indicator(STIXDomainObject): ('description', StringProperty()), ('indicator_types', ListProperty(StringProperty, required=True)), ('pattern', PatternProperty(required=True)), + ('pattern_type', StringProperty(required=True)), + ('pattern_version', StringProperty()), ('valid_from', TimestampProperty(default=lambda: NOW, required=True)), ('valid_until', TimestampProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), @@ -245,6 +249,7 @@ class Infrastructure(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('infrastructure_types', ListProperty(StringProperty, required=True)), + ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), @@ -318,6 +323,7 @@ class Location(STIXDomainObject): ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty()), ('description', StringProperty()), ('latitude', FloatProperty(min=-90.0, max=90.0)), ('longitude', FloatProperty(min=-180.0, max=180.0)), @@ -346,6 +352,20 @@ class Location(STIXDomainObject): self._check_properties_dependency(['latitude'], ['longitude']) self._check_properties_dependency(['longitude'], ['latitude']) + if not ( + 'region' in self + or 'country' in self + or ( + 'latitude' in self + and 'longitude' in self + ) + ): + raise InvalidPropertyConfigurationError( + "Location objects must have the properties 'region', " + "'country', or 'latitude' and 'longitude'", + Location + ) + def to_maps_url(self, map_engine="Google Maps"): """Return URL to this location in an online map engine. @@ -411,7 +431,7 @@ class Malware(STIXDomainObject): ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('name', StringProperty(required=True)), + ('name', StringProperty()), ('description', StringProperty()), ('malware_types', ListProperty(StringProperty, required=True)), ('is_family', BooleanProperty(required=True)), @@ -443,6 +463,12 @@ class Malware(STIXDomainObject): msg = "{0.id} 'last_seen' must be greater than or equal to 'first_seen'" raise ValueError(msg.format(self)) + if self.is_family and "name" not in self: + raise InvalidPropertyConfigurationError( + "'name' is a required property for malware families", + Malware + ) + class MalwareAnalysis(STIXDomainObject): # TODO: Add link @@ -471,7 +497,7 @@ class MalwareAnalysis(STIXDomainObject): ('operating_system_ref', ReferenceProperty(type='software', spec_version='2.1')), ('installed_software_refs', ListProperty(ReferenceProperty(type='software', spec_version='2.1'))), ('configuration_version', StringProperty()), - ('module', StringProperty()), + ('modules', ListProperty(StringProperty)), ('analysis_engine_version', StringProperty()), ('analysis_definition_version', StringProperty()), ('submitted', TimestampProperty()), @@ -580,7 +606,6 @@ class Opinion(STIXDomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('explanation', StringProperty()), ('authors', ListProperty(StringProperty)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), ( 'opinion', EnumProperty( allowed=[ @@ -592,6 +617,7 @@ class Opinion(STIXDomainObject): ], required=True, ), ), + ('object_refs', ListProperty(ReferenceProperty, required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -649,6 +675,8 @@ class ThreatActor(STIXDomainObject): ('description', StringProperty()), ('threat_actor_types', ListProperty(StringProperty, required=True)), ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), ('roles', ListProperty(StringProperty)), ('goals', ListProperty(StringProperty)), ('sophistication', StringProperty()), @@ -665,6 +693,16 @@ class ThreatActor(STIXDomainObject): ('granular_markings', ListProperty(GranularMarking)), ]) + def _check_object_constraints(self): + super(self.__class__, self)._check_object_constraints() + + first_observed = self.get('first_seen') + last_observed = self.get('last_seen') + + if first_observed and last_observed and last_observed < first_observed: + msg = "{0.id} 'last_seen' must be greater than or equal to 'first_seen'" + raise ValueError(msg.format(self)) + class Tool(STIXDomainObject): # TODO: Add link diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 144df59..149094e 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -80,6 +80,7 @@ class Sighting(STIXRelationshipObject): ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('description', StringProperty()), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), From cd0c4984fa8683d52f01790c328224a9a6bdecab Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 16 Jul 2019 16:10:25 -0400 Subject: [PATCH 25/62] Fix most unit tests to pass again. Awaiting feedback regarding possible library bugs, before I fix the remaining unit tests. --- stix2/test/v21/conftest.py | 8 ++++++++ stix2/test/v21/constants.py | 5 +++++ stix2/test/v21/test_bundle.py | 3 +++ stix2/test/v21/test_core.py | 1 + stix2/test/v21/test_datastore_filters.py | 1 + stix2/test/v21/test_datastore_memory.py | 8 ++++++++ stix2/test/v21/test_grouping.py | 20 ++++++++++++++++++-- stix2/test/v21/test_indicator.py | 12 +++++++++--- stix2/test/v21/test_malware.py | 4 ++-- stix2/test/v21/test_malware_analysis.py | 4 +++- stix2/test/v21/test_opinion.py | 9 +++++---- 11 files changed, 63 insertions(+), 12 deletions(-) diff --git a/stix2/test/v21/conftest.py b/stix2/test/v21/conftest.py index ea2853d..103db34 100644 --- a/stix2/test/v21/conftest.py +++ b/stix2/test/v21/conftest.py @@ -71,6 +71,7 @@ def stix_objs1(): "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -84,6 +85,7 @@ def stix_objs1(): "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -97,6 +99,7 @@ def stix_objs1(): "modified": "2017-01-27T13:49:53.936Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -110,6 +113,7 @@ def stix_objs1(): "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -123,6 +127,7 @@ def stix_objs1(): "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -140,6 +145,7 @@ def stix_objs2(): ], "modified": "2017-01-31T13:49:53.935Z", "name": "Malicious site hosting downloader", + "pattern_type": "stix", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", "spec_version": "2.1", "type": "indicator", @@ -153,6 +159,7 @@ def stix_objs2(): ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", + "pattern_type": "stix", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", "spec_version": "2.1", "type": "indicator", @@ -166,6 +173,7 @@ def stix_objs2(): ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", + "pattern_type": "stix", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", "spec_version": "2.1", "type": "indicator", diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index 40a2bb5..fd1ff38 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -75,6 +75,10 @@ COURSE_OF_ACTION_KWARGS = dict( GROUPING_KWARGS = dict( name="Harry Potter and the Leet Hackers", context="suspicious-activity", + object_refs=[ + "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", + "identity--988145ed-a3b4-4421-b7a7-273376be67ce" + ], ) IDENTITY_KWARGS = dict( @@ -84,6 +88,7 @@ IDENTITY_KWARGS = dict( INDICATOR_KWARGS = dict( indicator_types=['malicious-activity'], + pattern_type="stix", pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", valid_from="2017-01-01T12:34:56Z", ) diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 7ba0729..58d3b3f 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -20,6 +20,7 @@ EXPECTED_BUNDLE = """{ "malicious-activity" ], "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "2017-01-01T12:34:56Z" }, { @@ -58,6 +59,7 @@ EXPECTED_BUNDLE_DICT = { "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "2017-01-01T12:34:56Z", "indicator_types": [ "malicious-activity", @@ -234,6 +236,7 @@ def test_bundle_obj_id_found(): "malicious-activity", ], "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "2017-01-01T12:34:56Z", }, { diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index 06a829c..f04e600 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -16,6 +16,7 @@ BUNDLE = { "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "2017-01-01T12:34:56Z", "indicator_types": [ "malicious-activity", diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index cbe3fe4..b96aa4d 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -29,6 +29,7 @@ stix_objs = [ "modified": "2014-05-08T09:00:00.000Z", "name": "File hash for Poison Ivy variant", "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2014-05-08T09:00:00.000000Z", diff --git a/stix2/test/v21/test_datastore_memory.py b/stix2/test/v21/test_datastore_memory.py index b69d4d6..4f63a06 100644 --- a/stix2/test/v21/test_datastore_memory.py +++ b/stix2/test/v21/test_datastore_memory.py @@ -24,6 +24,7 @@ IND1 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -37,6 +38,7 @@ IND2 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -50,6 +52,7 @@ IND3 = { "modified": "2017-01-27T13:49:53.936Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -63,6 +66,7 @@ IND4 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -76,6 +80,7 @@ IND5 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -89,6 +94,7 @@ IND6 = { "modified": "2017-01-31T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -102,6 +108,7 @@ IND7 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", @@ -115,6 +122,7 @@ IND8 = { "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "pattern_type": "stix", "spec_version": "2.1", "type": "indicator", "valid_from": "2017-01-27T13:49:53.935382Z", diff --git a/stix2/test/v21/test_grouping.py b/stix2/test/v21/test_grouping.py index 405a80c..449400b 100644 --- a/stix2/test/v21/test_grouping.py +++ b/stix2/test/v21/test_grouping.py @@ -14,7 +14,11 @@ EXPECTED_GROUPING = """{ "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "name": "Harry Potter and the Leet Hackers", - "context": "suspicious-activity" + "context": "suspicious-activity", + "object_refs": [ + "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", + "identity--988145ed-a3b4-4421-b7a7-273376be67ce" + ] }""" @@ -28,6 +32,10 @@ def test_grouping_with_all_required_properties(): modified=now, name="Harry Potter and the Leet Hackers", context="suspicious-activity", + object_refs=[ + "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", + "identity--988145ed-a3b4-4421-b7a7-273376be67ce", + ], ) assert str(grp) == EXPECTED_GROUPING @@ -74,7 +82,7 @@ def test_grouping_required_properties(): stix2.v21.Grouping() assert excinfo.value.cls == stix2.v21.Grouping - assert excinfo.value.properties == ["context"] + assert excinfo.value.properties == ["context", "object_refs"] def test_invalid_kwarg_to_grouping(): @@ -97,6 +105,10 @@ def test_invalid_kwarg_to_grouping(): "modified": "2017-01-01T12:34:56.000Z", "name": "Harry Potter and the Leet Hackers", "context": "suspicious-activity", + "object_refs": [ + "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", + "identity--988145ed-a3b4-4421-b7a7-273376be67ce", + ], }, ], ) @@ -110,3 +122,7 @@ def test_parse_grouping(data): assert grp.modified == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) assert grp.name == "Harry Potter and the Leet Hackers" assert grp.context == "suspicious-activity" + assert grp.object_refs == [ + "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", + "identity--988145ed-a3b4-4421-b7a7-273376be67ce" + ] diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index b68b887..ea46d6d 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -18,6 +18,7 @@ EXPECTED_INDICATOR = """{ "malicious-activity" ], "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "1970-01-01T00:00:01Z" }""" @@ -29,6 +30,7 @@ EXPECTED_INDICATOR_REPR = "Indicator(" + " ".join(""" modified='2017-01-01T00:00:01.000Z', indicator_types=['malicious-activity'], pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + pattern_type='stix', valid_from='1970-01-01T00:00:01Z' """.split()) + ")" @@ -43,6 +45,7 @@ def test_indicator_with_all_required_properties(): created=now, modified=now, pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + pattern_type="stix", valid_from=epoch, indicator_types=['malicious-activity'], ) @@ -98,8 +101,8 @@ def test_indicator_required_properties(): stix2.v21.Indicator() assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["indicator_types", "pattern", "valid_from"] - assert str(excinfo.value) == "No values for required properties for Indicator: (indicator_types, pattern, valid_from)." + assert excinfo.value.properties == ["indicator_types", "pattern", "pattern_type", "valid_from"] + assert str(excinfo.value) == "No values for required properties for Indicator: (indicator_types, pattern, pattern_type, valid_from)." def test_indicator_required_property_pattern(): @@ -107,7 +110,7 @@ def test_indicator_required_property_pattern(): stix2.v21.Indicator(indicator_types=['malicious-activity']) assert excinfo.value.cls == stix2.v21.Indicator - assert excinfo.value.properties == ["pattern", "valid_from"] + assert excinfo.value.properties == ["pattern", "pattern_type", "valid_from"] def test_indicator_created_ref_invalid_format(): @@ -162,6 +165,7 @@ def test_created_modified_time_are_identical_by_default(): "malicious-activity", ], "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern_type": "stix", "valid_from": "1970-01-01T00:00:01Z", }, ], @@ -184,6 +188,7 @@ def test_invalid_indicator_pattern(): stix2.v21.Indicator( indicator_types=['malicious-activity'], pattern="file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e'", + pattern_type="stix", valid_from="2017-01-01T12:34:56Z", ) assert excinfo.value.cls == stix2.v21.Indicator @@ -194,6 +199,7 @@ def test_invalid_indicator_pattern(): stix2.v21.Indicator( indicator_types=['malicious-activity'], pattern='[file:hashes.MD5 = "d41d8cd98f00b204e9800998ecf8427e"]', + pattern_type="stix", valid_from="2017-01-01T12:34:56Z", ) assert excinfo.value.cls == stix2.v21.Indicator diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index 1817c63..9ae0ed2 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -79,7 +79,7 @@ def test_malware_required_properties(): stix2.v21.Malware() assert excinfo.value.cls == stix2.v21.Malware - assert excinfo.value.properties == ["is_family", "malware_types", "name"] + assert excinfo.value.properties == ["is_family", "malware_types"] def test_malware_required_property_name(): @@ -87,7 +87,7 @@ def test_malware_required_property_name(): stix2.v21.Malware(malware_types=['ransomware']) assert excinfo.value.cls == stix2.v21.Malware - assert excinfo.value.properties == ["is_family", "name"] + assert excinfo.value.properties == ["is_family"] def test_cannot_assign_to_malware_attributes(malware): diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index 42db919..bfb4ff4 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -26,7 +26,9 @@ MALWARE_ANALYSIS_JSON = """{ "software--46a6a91d-1160-4867-a4d1-b14e080e4e5b" ], "configuration_version": "1.7", - "module": "Super Analyzer", + "modules": [ + "Super Analyzer" + ], "analysis_engine_version": "1.2", "analysis_definition_version": "3.4", "submitted": "2018-11-23T06:45:55.747Z", diff --git a/stix2/test/v21/test_opinion.py b/stix2/test/v21/test_opinion.py index b2f6dc0..9634d6e 100644 --- a/stix2/test/v21/test_opinion.py +++ b/stix2/test/v21/test_opinion.py @@ -23,10 +23,10 @@ EXPECTED_OPINION = """{ "created": "2016-05-12T08:17:27.000Z", "modified": "2016-05-12T08:17:27.000Z", "explanation": "%s", + "opinion": "strongly-disagree", "object_refs": [ "relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471" - ], - "opinion": "strongly-disagree" + ] }""" % EXPLANATION EXPECTED_OPINION_REPR = "Opinion(" + " ".join(( @@ -37,8 +37,9 @@ EXPECTED_OPINION_REPR = "Opinion(" + " ".join(( created='2016-05-12T08:17:27.000Z', modified='2016-05-12T08:17:27.000Z', explanation="%s", - object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'], - opinion='strongly-disagree'""" % EXPLANATION + opinion='strongly-disagree', + object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'] + """ % EXPLANATION ).split()) + ")" From 4660d5ea280333391014a4becb1ea3bda15091c6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 17 Jul 2019 15:48:09 -0400 Subject: [PATCH 26/62] Update SCO specs per WD 05 specs --- stix2/test/v21/test_observed_data.py | 30 +++++++++++------------ stix2/v21/observables.py | 36 +++++++++++++++++++++------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 09e6a67..e82ecbd 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -693,16 +693,16 @@ def test_directory_example(): dir = stix2.v21.Directory( _valid_refs={"1": "file"}, path='/usr/lib', - created="2015-12-21T19:00:00Z", - modified="2015-12-24T19:00:00Z", - accessed="2015-12-21T20:00:00Z", + ctime="2015-12-21T19:00:00Z", + mtime="2015-12-24T19:00:00Z", + atime="2015-12-21T20:00:00Z", contains_refs=["1"], ) assert dir.path == '/usr/lib' - assert dir.created == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert dir.modified == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert dir.accessed == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert dir.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert dir.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert dir.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) assert dir.contains_refs == ["1"] @@ -711,9 +711,9 @@ def test_directory_example_ref_error(): stix2.v21.Directory( _valid_refs=[], path='/usr/lib', - created="2015-12-21T19:00:00Z", - modified="2015-12-24T19:00:00Z", - accessed="2015-12-21T20:00:00Z", + ctime="2015-12-21T19:00:00Z", + mtime="2015-12-24T19:00:00Z", + atime="2015-12-21T20:00:00Z", contains_refs=["1"], ) @@ -753,9 +753,9 @@ def test_file_example(): size=100, magic_number_hex="1C", mime_type="application/msword", - created="2016-12-21T19:00:00Z", - modified="2016-12-24T19:00:00Z", - accessed="2016-12-21T20:00:00Z", + ctime="2016-12-21T19:00:00Z", + mtime="2016-12-24T19:00:00Z", + atime="2016-12-21T20:00:00Z", ) assert f.name == "qwerty.dll" @@ -763,9 +763,9 @@ def test_file_example(): assert f.magic_number_hex == "1C" assert f.hashes["SHA-256"] == "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a" assert f.mime_type == "application/msword" - assert f.created == dt.datetime(2016, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert f.modified == dt.datetime(2016, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert f.accessed == dt.datetime(2016, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert f.ctime == dt.datetime(2016, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert f.mtime == dt.datetime(2016, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert f.atime == dt.datetime(2016, 12, 21, 20, 0, 0, tzinfo=pytz.utc) def test_file_example_with_NTFSExt(): diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index f383899..be2b127 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -14,7 +14,7 @@ from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError from ..properties import ( BinaryProperty, BooleanProperty, CallableValues, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, - HashesProperty, HexProperty, IntegerProperty, ListProperty, + HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) @@ -28,6 +28,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), @@ -52,6 +53,7 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), @@ -68,12 +70,13 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself - ('created', TimestampProperty()), - ('modified', TimestampProperty()), - ('accessed', TimestampProperty()), + ('ctime', TimestampProperty()), + ('mtime', TimestampProperty()), + ('atime', TimestampProperty()), ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -88,6 +91,7 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -103,6 +107,7 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), @@ -137,6 +142,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), @@ -170,7 +176,7 @@ class ArchiveExt(_Extension): _type = 'archive-ext' _properties = OrderedDict([ - ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']), required=True)), ('comment', StringProperty()), ]) @@ -323,6 +329,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), ('name', StringProperty()), @@ -330,9 +337,9 @@ class File(_Observable): ('magic_number_hex', HexProperty()), ('mime_type', StringProperty()), # these are not the created/modified timestamps of the object itself - ('created', TimestampProperty()), - ('modified', TimestampProperty()), - ('accessed', TimestampProperty()), + ('ctime', TimestampProperty()), + ('mtime', TimestampProperty()), + ('atime', TimestampProperty()), ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), ('contains_refs', ListProperty(ObjectReferenceProperty)), ('content_ref', ObjectReferenceProperty(valid_types='artifact')), @@ -353,6 +360,7 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -369,6 +377,7 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -385,6 +394,7 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -399,6 +409,7 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -505,6 +516,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), @@ -624,6 +636,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself @@ -663,6 +676,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('name', StringProperty(required=True)), ('cpe', StringProperty()), ('languages', ListProperty(StringProperty)), @@ -681,6 +695,7 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) @@ -710,6 +725,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('user_id', StringProperty()), ('credential', StringProperty()), ('account_login', StringProperty()), @@ -767,10 +783,11 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself - ('modified', TimestampProperty()), + ('modified_time', TimestampProperty()), ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -818,6 +835,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), ('version', StringProperty()), From 558948098088dd8305f598a6b76ced30582c7751 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 19 Jul 2019 14:50:11 -0400 Subject: [PATCH 27/62] Improved the exception class hierarchy: - Removed all plain python base classes (e.g. ValueError, TypeError) - Renamed InvalidPropertyConfigurationError -> PropertyPresenceError, since incorrect values could be considered a property config error, and I really just wanted this class to apply to presence (co-)constraint violations. - Added ObjectConfigurationError as a superclass of InvalidValueError, PropertyPresenceError, and any other exception that could be raised during _STIXBase object init, which is when the spec compliance checks happen. This class is intended to represent general spec violations. - Did some class reordering in exceptions.py, so all the ObjectConfigurationError subclasses were together. Changed how property "cleaning" errors were handled: - Previous docs said they should all be ValueErrors, but that would require extra exception check-and-replace complexity in the property implementations, so that requirement is removed. Doc is changed to just say that cleaning problems should cause exceptions to be raised. _STIXBase._check_property() now handles most exception types, not just ValueError. - Decided to try chaining the original clean error to the InvalidValueError, in case the extra diagnostics would be helpful in the future. This is done via 'six' adapter function and only works on python3. - A small amount of testing was removed, since it was looking at custom exception properties which became unavailable once the exception was replaced with InvalidValueError. Did another pass through unit tests to fix breakage caused by the changed exception class hierarchy. Removed unnecessary observable extension handling code from parse_observable(), since it was all duplicated in ExtensionsProperty. The redundant code in parse_observable() had different exception behavior than ExtensionsProperty, which makes the API inconsistent and unit tests more complicated. (Problems in ExtensionsProperty get replaced with InvalidValueError, but extensions problems handled directly in parse_observable() don't get the same replacement, and so the exception type is different.) Redid the workbench monkeypatching. The old way was impossible to make work, and had caused ugly ripple effect hackage in other parts of the codebase. Now, it replaces the global object maps with factory functions which behave the same way when called, as real classes. Had to fix up a few unit tests to get them all passing with this monkeypatching in place. Also remove all the xfail markings in the workbench test suite, since all tests now pass. Since workbench monkeypatching isn't currently affecting any unit tests, tox.ini was simplified to remove the special-casing for running the workbench tests. Removed the v20 workbench test suite, since the workbench currently only works with the latest stix object version. --- stix2/__init__.py | 2 +- stix2/base.py | 24 +- stix2/core.py | 13 - stix2/exceptions.py | 170 ++++++------ stix2/properties.py | 2 +- stix2/test/v20/test_bundle.py | 9 +- stix2/test/v20/test_custom.py | 9 +- stix2/test/v20/test_granular_markings.py | 16 +- stix2/test/v20/test_malware.py | 3 +- stix2/test/v20/test_object_markings.py | 3 +- stix2/test/v20/test_observed_data.py | 27 +- stix2/test/v20/test_properties.py | 30 ++- stix2/test/v20/test_workbench.py | 316 ----------------------- stix2/test/v21/constants.py | 2 +- stix2/test/v21/test_bundle.py | 9 +- stix2/test/v21/test_core.py | 2 +- stix2/test/v21/test_custom.py | 11 +- stix2/test/v21/test_granular_markings.py | 16 +- stix2/test/v21/test_grouping.py | 2 +- stix2/test/v21/test_location.py | 7 +- stix2/test/v21/test_malware.py | 3 +- stix2/test/v21/test_object_markings.py | 3 +- stix2/test/v21/test_observed_data.py | 26 +- stix2/test/v21/test_properties.py | 30 ++- stix2/test/v21/test_workbench.py | 12 - stix2/v21/sdo.py | 26 +- stix2/workbench.py | 60 ++--- tox.ini | 4 +- 28 files changed, 261 insertions(+), 576 deletions(-) delete mode 100644 stix2/test/v20/test_workbench.py diff --git a/stix2/__init__.py b/stix2/__init__.py index 714bf46..68e0264 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -54,7 +54,7 @@ from .patterns import ( WithinQualifier, ) from .utils import new_version, revoke -from .v20 import * # This import will always be the latest STIX 2.X version +from .v21 import * # This import will always be the latest STIX 2.X version from .version import __version__ _collect_stix2_mappings() diff --git a/stix2/base.py b/stix2/base.py index a9a801e..9fe1617 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -5,6 +5,7 @@ import copy import datetime as dt import simplejson as json +import six from .exceptions import ( AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, @@ -88,10 +89,25 @@ class _STIXBase(collections.Mapping): if prop_name in kwargs: try: kwargs[prop_name] = prop.clean(kwargs[prop_name]) - except ValueError as exc: - if self.__allow_custom and isinstance(exc, CustomContentError): - return - raise InvalidValueError(self.__class__, prop_name, reason=str(exc)) + except InvalidValueError: + # No point in wrapping InvalidValueError in another + # InvalidValueError... so let those propagate. + raise + except CustomContentError as exc: + if not self.__allow_custom: + six.raise_from( + InvalidValueError( + self.__class__, prop_name, reason=str(exc), + ), + exc, + ) + except Exception as exc: + six.raise_from( + InvalidValueError( + self.__class__, prop_name, reason=str(exc), + ), + exc, + ) # interproperty constraint methods diff --git a/stix2/core.py b/stix2/core.py index 830d98c..1031d61 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -169,19 +169,6 @@ def parse_observable(data, _valid_refs=None, allow_custom=False, version=None): raise CustomContentError("Can't parse unknown observable type '%s'! For custom observables, " "use the CustomObservable decorator." % obj['type']) - EXT_MAP = STIX2_OBJ_MAPS[v]['observable-extensions'] - - if 'extensions' in obj and obj['type'] in EXT_MAP: - for name, ext in obj['extensions'].items(): - try: - ext_class = EXT_MAP[obj['type']][name] - except KeyError: - if not allow_custom: - raise CustomContentError("Can't parse unknown extension type '%s'" - "for observable type '%s'!" % (name, obj['type'])) - else: # extension was found - obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) - return obj_class(allow_custom=allow_custom, **obj) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 946300c..c680774 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -5,7 +5,15 @@ class STIXError(Exception): """Base class for errors generated in the stix2 library.""" -class InvalidValueError(STIXError, ValueError): +class ObjectConfigurationError(STIXError): + """ + Represents specification violations regarding the composition of STIX + objects. + """ + pass + + +class InvalidValueError(ObjectConfigurationError): """An invalid value was provided to a STIX object's ``__init__``.""" def __init__(self, cls, prop_name, reason): @@ -19,18 +27,18 @@ class InvalidValueError(STIXError, ValueError): return msg.format(self) -class InvalidPropertyConfigurationError(STIXError, ValueError): +class PropertyPresenceError(ObjectConfigurationError): """ Represents an invalid combination of properties on a STIX object. This class can be used directly when the object requirements are more complicated and none of the more specific exception subclasses apply. """ def __init__(self, message, cls): - super(InvalidPropertyConfigurationError, self).__init__(message) + super(PropertyPresenceError, self).__init__(message) self.cls = cls -class MissingPropertiesError(InvalidPropertyConfigurationError): +class MissingPropertiesError(PropertyPresenceError): """Missing one or more required properties when constructing STIX object.""" def __init__(self, cls, properties): @@ -44,7 +52,7 @@ class MissingPropertiesError(InvalidPropertyConfigurationError): super(MissingPropertiesError, self).__init__(msg, cls) -class ExtraPropertiesError(InvalidPropertyConfigurationError): +class ExtraPropertiesError(PropertyPresenceError): """One or more extra properties were provided when constructing STIX object.""" def __init__(self, cls, properties): @@ -58,7 +66,7 @@ class ExtraPropertiesError(InvalidPropertyConfigurationError): super(ExtraPropertiesError, self).__init__(msg, cls) -class MutuallyExclusivePropertiesError(InvalidPropertyConfigurationError): +class MutuallyExclusivePropertiesError(PropertyPresenceError): """Violating interproperty mutually exclusive constraint of a STIX object type.""" def __init__(self, cls, properties): @@ -72,7 +80,7 @@ class MutuallyExclusivePropertiesError(InvalidPropertyConfigurationError): super(MutuallyExclusivePropertiesError, self).__init__(msg, cls) -class DependentPropertiesError(InvalidPropertyConfigurationError): +class DependentPropertiesError(PropertyPresenceError): """Violating interproperty dependency constraint of a STIX object type.""" def __init__(self, cls, dependencies): @@ -86,7 +94,7 @@ class DependentPropertiesError(InvalidPropertyConfigurationError): super(DependentPropertiesError, self).__init__(msg, cls) -class AtLeastOnePropertyError(InvalidPropertyConfigurationError): +class AtLeastOnePropertyError(PropertyPresenceError): """Violating a constraint of a STIX object type that at least one of the given properties must be populated.""" def __init__(self, cls, properties): @@ -94,27 +102,14 @@ class AtLeastOnePropertyError(InvalidPropertyConfigurationError): msg = "At least one of the ({1}) properties for {0} must be " \ "populated.".format( - cls.__name__, - ", ".join(x for x in self.properties), - ) + cls.__name__, + ", ".join(x for x in self.properties), + ) super(AtLeastOnePropertyError, self).__init__(msg, cls) -class ImmutableError(STIXError, ValueError): - """Attempted to modify an object after creation.""" - - def __init__(self, cls, key): - super(ImmutableError, self).__init__() - self.cls = cls - self.key = key - - def __str__(self): - msg = "Cannot modify '{0.key}' property in '{0.cls.__name__}' after creation." - return msg.format(self) - - -class DictionaryKeyError(STIXError, ValueError): +class DictionaryKeyError(ObjectConfigurationError): """Dictionary key does not conform to the correct format.""" def __init__(self, key, reason): @@ -127,7 +122,7 @@ class DictionaryKeyError(STIXError, ValueError): return msg.format(self) -class InvalidObjRefError(STIXError, ValueError): +class InvalidObjRefError(ObjectConfigurationError): """A STIX Cyber Observable Object contains an invalid object reference.""" def __init__(self, cls, prop_name, reason): @@ -141,47 +136,7 @@ class InvalidObjRefError(STIXError, ValueError): return msg.format(self) -class UnmodifiablePropertyError(STIXError, ValueError): - """Attempted to modify an unmodifiable property of object when creating a new version.""" - - def __init__(self, unchangable_properties): - super(UnmodifiablePropertyError, self).__init__() - self.unchangable_properties = unchangable_properties - - def __str__(self): - msg = "These properties cannot be changed when making a new version: {0}." - return msg.format(", ".join(self.unchangable_properties)) - - -class RevokeError(STIXError, ValueError): - """Attempted to an operation on a revoked object.""" - - def __init__(self, called_by): - super(RevokeError, self).__init__() - self.called_by = called_by - - def __str__(self): - if self.called_by == "revoke": - return "Cannot revoke an already revoked object." - else: - return "Cannot create a new version of a revoked object." - - -class ParseError(STIXError, ValueError): - """Could not parse object.""" - - def __init__(self, msg): - super(ParseError, self).__init__(msg) - - -class CustomContentError(STIXError, ValueError): - """Custom STIX Content (SDO, Observable, Extension, etc.) detected.""" - - def __init__(self, msg): - super(CustomContentError, self).__init__(msg) - - -class InvalidSelectorError(STIXError, AssertionError): +class InvalidSelectorError(ObjectConfigurationError): """Granular Marking selector violation. The selector must resolve into an existing STIX object property.""" def __init__(self, cls, key): @@ -194,20 +149,7 @@ class InvalidSelectorError(STIXError, AssertionError): return msg.format(self.key, self.cls.__class__.__name__) -class MarkingNotFoundError(STIXError, AssertionError): - """Marking violation. The marking reference must be present in SDO or SRO.""" - - def __init__(self, cls, key): - super(MarkingNotFoundError, self).__init__() - self.cls = cls - self.key = key - - def __str__(self): - msg = "Marking {0} was not found in {1}!" - return msg.format(self.key, self.cls.__class__.__name__) - - -class TLPMarkingDefinitionError(STIXError, AssertionError): +class TLPMarkingDefinitionError(ObjectConfigurationError): """Marking violation. The marking-definition for TLP MUST follow the mandated instances from the spec.""" def __init__(self, user_obj, spec_obj): @@ -218,3 +160,69 @@ class TLPMarkingDefinitionError(STIXError, AssertionError): def __str__(self): msg = "Marking {0} does not match spec marking {1}!" return msg.format(self.user_obj, self.spec_obj) + + +class ImmutableError(STIXError): + """Attempted to modify an object after creation.""" + + def __init__(self, cls, key): + super(ImmutableError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Cannot modify '{0.key}' property in '{0.cls.__name__}' after creation." + return msg.format(self) + + +class UnmodifiablePropertyError(STIXError): + """Attempted to modify an unmodifiable property of object when creating a new version.""" + + def __init__(self, unchangable_properties): + super(UnmodifiablePropertyError, self).__init__() + self.unchangable_properties = unchangable_properties + + def __str__(self): + msg = "These properties cannot be changed when making a new version: {0}." + return msg.format(", ".join(self.unchangable_properties)) + + +class RevokeError(STIXError): + """Attempted an operation on a revoked object.""" + + def __init__(self, called_by): + super(RevokeError, self).__init__() + self.called_by = called_by + + def __str__(self): + if self.called_by == "revoke": + return "Cannot revoke an already revoked object." + else: + return "Cannot create a new version of a revoked object." + + +class ParseError(STIXError): + """Could not parse object.""" + + def __init__(self, msg): + super(ParseError, self).__init__(msg) + + +class CustomContentError(STIXError): + """Custom STIX Content (SDO, Observable, Extension, etc.) detected.""" + + def __init__(self, msg): + super(CustomContentError, self).__init__(msg) + + +class MarkingNotFoundError(STIXError): + """Marking violation. The marking reference must be present in SDO or SRO.""" + + def __init__(self, cls, key): + super(MarkingNotFoundError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Marking {0} was not found in {1}!" + return msg.format(self.key, self.cls.__class__.__name__) diff --git a/stix2/properties.py b/stix2/properties.py index 4e2f5f6..b9a5aff 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -102,7 +102,7 @@ class Property(object): - Return a value that is valid for this property. If ``value`` is not valid for this property, this will attempt to transform it first. If ``value`` is not valid and no such transformation is possible, it - should raise a ValueError. + should raise an exception. - ``def default(self):`` - provide a default value for this property. - ``default()`` can return the special value ``NOW`` to use the current diff --git a/stix2/test/v20/test_bundle.py b/stix2/test/v20/test_bundle.py index 57c189e..df59f89 100644 --- a/stix2/test/v20/test_bundle.py +++ b/stix2/test/v20/test_bundle.py @@ -4,6 +4,7 @@ import pytest import stix2 +from ...exceptions import InvalidValueError from .constants import IDENTITY_ID EXPECTED_BUNDLE = """{ @@ -156,15 +157,15 @@ def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationsh def test_create_bundle_invalid(indicator, malware, relationship): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Bundle(objects=[1]) assert excinfo.value.reason == "This property may only contain a dictionary or object" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Bundle(objects=[{}]) assert excinfo.value.reason == "This property may only contain a non-empty dictionary or object" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Bundle(objects=[{'type': 'bundle'}]) assert excinfo.value.reason == 'This property may not contain a Bundle object' @@ -232,7 +233,7 @@ def test_bundle_with_different_spec_objects(): }, ] - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Bundle(objects=data) assert "Spec version 2.0 bundles don't yet support containing objects of a different spec version." in str(excinfo.value) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 32632b9..8a99b2c 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -2,6 +2,7 @@ import pytest import stix2 +from ...exceptions import InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v20.Identity( @@ -133,7 +134,7 @@ def test_custom_property_dict_in_bundled_object(): 'identity_class': 'individual', 'x_foo': 'bar', } - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(InvalidValueError): stix2.v20.Bundle(custom_identity) bundle = stix2.v20.Bundle(custom_identity, allow_custom=True) @@ -199,7 +200,7 @@ def test_custom_property_object_in_observable_extension(): def test_custom_property_dict_in_observable_extension(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(InvalidValueError): stix2.v20.File( name='test', extensions={ @@ -718,7 +719,7 @@ def test_custom_extension(): def test_custom_extension_wrong_observable_type(): # NewExtension is an extension of DomainName, not File ext = NewExtension(property1='something') - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.File( name="abc.txt", extensions={ @@ -911,7 +912,7 @@ def test_parse_observable_with_custom_extension(): ], ) def test_parse_observable_with_unregistered_custom_extension(data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.parse_observable(data, version='2.0') assert "Can't parse unknown extension type" in str(excinfo.value) diff --git a/stix2/test/v20/test_granular_markings.py b/stix2/test/v20/test_granular_markings.py index b5f2e3d..e912cc1 100644 --- a/stix2/test/v20/test_granular_markings.py +++ b/stix2/test/v20/test_granular_markings.py @@ -2,7 +2,7 @@ import pytest from stix2 import markings -from stix2.exceptions import MarkingNotFoundError +from stix2.exceptions import InvalidSelectorError, MarkingNotFoundError from stix2.v20 import TLP_RED, Malware from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST @@ -179,7 +179,7 @@ def test_add_marking_mark_same_property_same_marking(): ], ) def test_add_marking_bad_selector(data, marking): - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.add_markings(data, marking[0], marking[1]) @@ -299,7 +299,7 @@ def test_get_markings_multiple_selectors(data): ) def test_get_markings_bad_selector(data, selector): """Test bad selectors raise exception""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.get_markings(data, selector) @@ -560,7 +560,7 @@ def test_remove_marking_bad_selector(): before = { "description": "test description", } - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"]) @@ -642,7 +642,7 @@ def test_is_marked_smoke(data): ) def test_is_marked_invalid_selector(data, selector): """Test invalid selector raises an error.""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.is_marked(data, selectors=selector) @@ -836,7 +836,7 @@ def test_is_marked_positional_arguments_combinations(): def test_create_sdo_with_invalid_marking(): - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(InvalidSelectorError) as excinfo: Malware( granular_markings=[ { @@ -974,7 +974,7 @@ def test_set_marking_bad_selector(marking): **MALWARE_KWARGS ) - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): before = markings.set_markings(before, marking[0], marking[1]) assert before == after @@ -1080,7 +1080,7 @@ def test_clear_marking_all_selectors(data): ) def test_clear_marking_bad_selector(data, selector): """Test bad selector raises exception.""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.clear_markings(data, selector) diff --git a/stix2/test/v20/test_malware.py b/stix2/test/v20/test_malware.py index 900a4b9..bd49007 100644 --- a/stix2/test/v20/test_malware.py +++ b/stix2/test/v20/test_malware.py @@ -6,6 +6,7 @@ import pytz import stix2 +from ...exceptions import InvalidValueError from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS EXPECTED_MALWARE = """{ @@ -145,7 +146,7 @@ def test_parse_malware(data): def test_parse_malware_invalid_labels(): data = re.compile('\\[.+\\]', re.DOTALL).sub('1', EXPECTED_MALWARE) - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.parse(data, version="2.0") assert "Invalid value for Malware 'labels'" in str(excinfo.value) diff --git a/stix2/test/v20/test_object_markings.py b/stix2/test/v20/test_object_markings.py index 156c42d..191f33a 100644 --- a/stix2/test/v20/test_object_markings.py +++ b/stix2/test/v20/test_object_markings.py @@ -4,6 +4,7 @@ import pytest from stix2 import exceptions, markings from stix2.v20 import TLP_AMBER, Malware +from ...exceptions import MarkingNotFoundError from .constants import FAKE_TIME, MALWARE_ID from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST from .constants import MARKING_IDS @@ -350,7 +351,7 @@ def test_remove_markings_bad_markings(): object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], **MALWARE_KWARGS ) - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(MarkingNotFoundError) as excinfo: markings.remove_markings(before, [MARKING_IDS[4]], None) assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] diff --git a/stix2/test/v20/test_observed_data.py b/stix2/test/v20/test_observed_data.py index 95daf22..a822efb 100644 --- a/stix2/test/v20/test_observed_data.py +++ b/stix2/test/v20/test_observed_data.py @@ -6,6 +6,7 @@ import pytz import stix2 +from ...exceptions import InvalidValueError from .constants import IDENTITY_ID, OBSERVED_DATA_ID OBJECTS_REGEX = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL) @@ -239,7 +240,7 @@ def test_parse_artifact_valid(data): ) def test_parse_artifact_invalid(data): odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) - with pytest.raises(ValueError): + with pytest.raises(InvalidValueError): stix2.parse(odata_str, version="2.0") @@ -468,11 +469,10 @@ def test_parse_email_message_with_at_least_one_error(data): "4": "artifact", "5": "file", } - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.parse_observable(data, valid_refs, version='2.0') - assert excinfo.value.cls == stix2.v20.EmailMIMEComponent - assert excinfo.value.properties == ["body", "body_raw_ref"] + assert excinfo.value.cls == stix2.v20.EmailMessage assert "At least one of the" in str(excinfo.value) assert "must be populated" in str(excinfo.value) @@ -734,7 +734,7 @@ def test_file_example_with_NTFSExt(): def test_file_example_with_empty_NTFSExt(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.File( name="abc.txt", extensions={ @@ -742,8 +742,7 @@ def test_file_example_with_empty_NTFSExt(): }, ) - assert excinfo.value.cls == stix2.v20.NTFSExt - assert excinfo.value.properties == sorted(list(stix2.NTFSExt._properties.keys())) + assert excinfo.value.cls == stix2.v20.File def test_file_example_with_PDFExt(): @@ -1112,16 +1111,14 @@ def test_process_example_empty_error(): def test_process_example_empty_with_extensions(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Process( extensions={ "windows-process-ext": {}, }, ) - assert excinfo.value.cls == stix2.v20.WindowsProcessExt - properties_of_extension = list(stix2.v20.WindowsProcessExt._properties.keys()) - assert excinfo.value.properties == sorted(properties_of_extension) + assert excinfo.value.cls == stix2.v20.Process def test_process_example_windows_process_ext(): @@ -1144,7 +1141,7 @@ def test_process_example_windows_process_ext(): def test_process_example_windows_process_ext_empty(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v20.Process( pid=1221, name="gedit-bin", @@ -1153,9 +1150,7 @@ def test_process_example_windows_process_ext_empty(): }, ) - assert excinfo.value.cls == stix2.v20.WindowsProcessExt - properties_of_extension = list(stix2.v20.WindowsProcessExt._properties.keys()) - assert excinfo.value.properties == sorted(properties_of_extension) + assert excinfo.value.cls == stix2.v20.Process def test_process_example_extensions_empty(): @@ -1289,7 +1284,7 @@ def test_user_account_unix_account_ext_example(): def test_windows_registry_key_example(): - with pytest.raises(ValueError): + with pytest.raises(InvalidValueError): stix2.v20.WindowsRegistryValueType( name="Foo", data="qwerty", diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 04a26f4..9952eac 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -3,7 +3,9 @@ import uuid import pytest import stix2 -from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError +from stix2.exceptions import ( + AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, +) from stix2.properties import ( BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, @@ -465,23 +467,27 @@ def test_extension_property_valid(): }) -@pytest.mark.parametrize( - "data", [ - 1, - {'foobar-ext': { - 'pe_type': 'exe', - }}, - ], -) -def test_extension_property_invalid(data): +def test_extension_property_invalid1(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') with pytest.raises(ValueError): - ext_prop.clean(data) + ext_prop.clean(1) + + +def test_extension_property_invalid2(): + ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') + with pytest.raises(CustomContentError): + ext_prop.clean( + { + 'foobar-ext': { + 'pe_type': 'exe', + }, + }, + ) def test_extension_property_invalid_type(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='indicator') - with pytest.raises(ValueError) as excinfo: + with pytest.raises(CustomContentError) as excinfo: ext_prop.clean( { 'windows-pebinary-ext': { diff --git a/stix2/test/v20/test_workbench.py b/stix2/test/v20/test_workbench.py deleted file mode 100644 index c254966..0000000 --- a/stix2/test/v20/test_workbench.py +++ /dev/null @@ -1,316 +0,0 @@ -import os - -import stix2 -from stix2.workbench import ( - AttackPattern, Campaign, CourseOfAction, ExternalReference, - FileSystemSource, Filter, Identity, Indicator, IntrusionSet, Malware, - MarkingDefinition, ObservedData, Relationship, Report, StatementMarking, - ThreatActor, Tool, Vulnerability, add_data_source, all_versions, - attack_patterns, campaigns, courses_of_action, create, get, identities, - indicators, intrusion_sets, malware, observed_data, query, reports, save, - set_default_created, set_default_creator, set_default_external_refs, - set_default_object_marking_refs, threat_actors, tools, vulnerabilities, -) - -from .constants import ( - ATTACK_PATTERN_ID, ATTACK_PATTERN_KWARGS, CAMPAIGN_ID, CAMPAIGN_KWARGS, - COURSE_OF_ACTION_ID, COURSE_OF_ACTION_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, - INDICATOR_ID, INDICATOR_KWARGS, INTRUSION_SET_ID, INTRUSION_SET_KWARGS, - MALWARE_ID, MALWARE_KWARGS, OBSERVED_DATA_ID, OBSERVED_DATA_KWARGS, - REPORT_ID, REPORT_KWARGS, THREAT_ACTOR_ID, THREAT_ACTOR_KWARGS, TOOL_ID, - TOOL_KWARGS, VULNERABILITY_ID, VULNERABILITY_KWARGS, -) - - -def test_workbench_environment(): - - # Create a STIX object - ind = create(Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS) - save(ind) - - resp = get(INDICATOR_ID) - assert resp['labels'][0] == 'malicious-activity' - - resp = all_versions(INDICATOR_ID) - assert len(resp) == 1 - - # Search on something other than id - q = [Filter('type', '=', 'vulnerability')] - resp = query(q) - assert len(resp) == 0 - - -def test_workbench_get_all_attack_patterns(): - mal = AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS) - save(mal) - - resp = attack_patterns() - assert len(resp) == 1 - assert resp[0].id == ATTACK_PATTERN_ID - - -def test_workbench_get_all_campaigns(): - cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) - save(cam) - - resp = campaigns() - assert len(resp) == 1 - assert resp[0].id == CAMPAIGN_ID - - -def test_workbench_get_all_courses_of_action(): - coa = CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS) - save(coa) - - resp = courses_of_action() - assert len(resp) == 1 - assert resp[0].id == COURSE_OF_ACTION_ID - - -def test_workbench_get_all_identities(): - idty = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) - save(idty) - - resp = identities() - assert len(resp) == 1 - assert resp[0].id == IDENTITY_ID - - -def test_workbench_get_all_indicators(): - resp = indicators() - assert len(resp) == 1 - assert resp[0].id == INDICATOR_ID - - -def test_workbench_get_all_intrusion_sets(): - ins = IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS) - save(ins) - - resp = intrusion_sets() - assert len(resp) == 1 - assert resp[0].id == INTRUSION_SET_ID - - -def test_workbench_get_all_malware(): - mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) - save(mal) - - resp = malware() - assert len(resp) == 1 - assert resp[0].id == MALWARE_ID - - -def test_workbench_get_all_observed_data(): - od = ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS) - save(od) - - resp = observed_data() - assert len(resp) == 1 - assert resp[0].id == OBSERVED_DATA_ID - - -def test_workbench_get_all_reports(): - rep = Report(id=REPORT_ID, **REPORT_KWARGS) - save(rep) - - resp = reports() - assert len(resp) == 1 - assert resp[0].id == REPORT_ID - - -def test_workbench_get_all_threat_actors(): - thr = ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS) - save(thr) - - resp = threat_actors() - assert len(resp) == 1 - assert resp[0].id == THREAT_ACTOR_ID - - -def test_workbench_get_all_tools(): - tool = Tool(id=TOOL_ID, **TOOL_KWARGS) - save(tool) - - resp = tools() - assert len(resp) == 1 - assert resp[0].id == TOOL_ID - - -def test_workbench_get_all_vulnerabilities(): - vuln = Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS) - save(vuln) - - resp = vulnerabilities() - assert len(resp) == 1 - assert resp[0].id == VULNERABILITY_ID - - -def test_workbench_add_to_bundle(): - vuln = Vulnerability(**VULNERABILITY_KWARGS) - bundle = stix2.v20.Bundle(vuln) - assert bundle.objects[0].name == 'Heartbleed' - - -def test_workbench_relationships(): - rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID) - save(rel) - - ind = get(INDICATOR_ID) - resp = ind.relationships() - assert len(resp) == 1 - assert resp[0].relationship_type == 'indicates' - assert resp[0].source_ref == INDICATOR_ID - assert resp[0].target_ref == MALWARE_ID - - -def test_workbench_created_by(): - intset = IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID) - save(intset) - creator = intset.created_by() - assert creator.id == IDENTITY_ID - - -def test_workbench_related(): - rel1 = Relationship(MALWARE_ID, 'targets', IDENTITY_ID) - rel2 = Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID) - save([rel1, rel2]) - - resp = get(MALWARE_ID).related() - assert len(resp) == 3 - assert any(x['id'] == CAMPAIGN_ID for x in resp) - assert any(x['id'] == INDICATOR_ID for x in resp) - assert any(x['id'] == IDENTITY_ID for x in resp) - - resp = get(MALWARE_ID).related(relationship_type='indicates') - assert len(resp) == 1 - - -def test_workbench_related_with_filters(): - malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID) - rel = Relationship(malware.id, 'variant-of', MALWARE_ID) - save([malware, rel]) - - filters = [Filter('created_by_ref', '=', IDENTITY_ID)] - resp = get(MALWARE_ID).related(filters=filters) - - assert len(resp) == 1 - assert resp[0].name == malware.name - assert resp[0].created_by_ref == IDENTITY_ID - - # filters arg can also be single filter - resp = get(MALWARE_ID).related(filters=filters[0]) - assert len(resp) == 1 - - -def test_add_data_source(): - fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") - fs = FileSystemSource(fs_path) - add_data_source(fs) - - resp = tools() - assert len(resp) == 3 - resp_ids = [tool.id for tool in resp] - assert TOOL_ID in resp_ids - assert 'tool--03342581-f790-4f03-ba41-e82e67392e23' in resp_ids - assert 'tool--242f3da3-4425-4d11-8f5c-b842886da966' in resp_ids - - -def test_additional_filter(): - resp = tools(Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5')) - assert len(resp) == 2 - - -def test_additional_filters_list(): - resp = tools([ - Filter('created_by_ref', '=', 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'), - Filter('name', '=', 'Windows Credential Editor'), - ]) - assert len(resp) == 1 - - -def test_default_creator(): - set_default_creator(IDENTITY_ID) - campaign = Campaign(**CAMPAIGN_KWARGS) - - assert 'created_by_ref' not in CAMPAIGN_KWARGS - assert campaign.created_by_ref == IDENTITY_ID - - -def test_default_created_timestamp(): - timestamp = "2018-03-19T01:02:03.000Z" - set_default_created(timestamp) - campaign = Campaign(**CAMPAIGN_KWARGS) - - assert 'created' not in CAMPAIGN_KWARGS - assert stix2.utils.format_datetime(campaign.created) == timestamp - assert stix2.utils.format_datetime(campaign.modified) == timestamp - - -def test_default_external_refs(): - ext_ref = ExternalReference( - source_name="ACME Threat Intel", - description="Threat report", - ) - set_default_external_refs(ext_ref) - campaign = Campaign(**CAMPAIGN_KWARGS) - - assert campaign.external_references[0].source_name == "ACME Threat Intel" - assert campaign.external_references[0].description == "Threat report" - - -def test_default_object_marking_refs(): - stmt_marking = StatementMarking("Copyright 2016, Example Corp") - mark_def = MarkingDefinition( - definition_type="statement", - definition=stmt_marking, - ) - set_default_object_marking_refs(mark_def) - campaign = Campaign(**CAMPAIGN_KWARGS) - - assert campaign.object_marking_refs[0] == mark_def.id - - -def test_workbench_custom_property_object_in_observable_extension(): - ntfs = stix2.v20.NTFSExt( - allow_custom=True, - sid=1, - x_foo='bar', - ) - artifact = stix2.v20.File( - name='test', - extensions={'ntfs-ext': ntfs}, - ) - observed_data = ObservedData( - allow_custom=True, - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=1, - objects={"0": artifact}, - ) - - assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar" - assert '"x_foo": "bar"' in str(observed_data) - - -def test_workbench_custom_property_dict_in_observable_extension(): - artifact = stix2.v20.File( - allow_custom=True, - name='test', - extensions={ - 'ntfs-ext': { - 'allow_custom': True, - 'sid': 1, - 'x_foo': 'bar', - }, - }, - ) - observed_data = ObservedData( - allow_custom=True, - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=1, - objects={"0": artifact}, - ) - - assert observed_data.objects['0'].extensions['ntfs-ext'].x_foo == "bar" - assert '"x_foo": "bar"' in str(observed_data) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index fd1ff38..c3ce3c0 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -77,7 +77,7 @@ GROUPING_KWARGS = dict( context="suspicious-activity", object_refs=[ "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", - "identity--988145ed-a3b4-4421-b7a7-273376be67ce" + "identity--988145ed-a3b4-4421-b7a7-273376be67ce", ], ) diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 58d3b3f..54ef318 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -4,6 +4,7 @@ import pytest import stix2 +from ...exceptions import InvalidValueError from .constants import IDENTITY_ID EXPECTED_BUNDLE = """{ @@ -164,15 +165,15 @@ def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationsh def test_create_bundle_invalid(indicator, malware, relationship): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v21.Bundle(objects=[1]) assert excinfo.value.reason == "This property may only contain a dictionary or object" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v21.Bundle(objects=[{}]) assert excinfo.value.reason == "This property may only contain a non-empty dictionary or object" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v21.Bundle(objects=[{'type': 'bundle'}]) assert excinfo.value.reason == 'This property may not contain a Bundle object' @@ -183,7 +184,7 @@ def test_parse_bundle(version): assert bundle.type == "bundle" assert bundle.id.startswith("bundle--") - assert type(bundle.objects[0]) is stix2.v21.Indicator + assert isinstance(bundle.objects[0], stix2.v21.Indicator) assert bundle.objects[0].type == 'indicator' assert bundle.objects[1].type == 'malware' assert bundle.objects[2].type == 'relationship' diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index f04e600..25348cd 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -79,7 +79,7 @@ def test_register_object_with_version(): v = 'v21' assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] - assert v in str(bundle.objects[0].__class__) + assert bundle.objects[0].spec_version == "2.1" def test_register_marking_with_version(): diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 6e1e585..a5c9244 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -3,6 +3,7 @@ import pytest import stix2 import stix2.base +from ...exceptions import InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v21.Identity( @@ -97,7 +98,7 @@ def test_identity_custom_property_allowed(): def test_parse_identity_custom_property(data): with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: stix2.parse(data, version="2.1") - assert excinfo.value.cls == stix2.v21.Identity + assert issubclass(excinfo.value.cls, stix2.v21.Identity) assert excinfo.value.properties == ['foo'] assert "Unexpected properties for" in str(excinfo.value) @@ -136,7 +137,7 @@ def test_custom_property_dict_in_bundled_object(): 'identity_class': 'individual', 'x_foo': 'bar', } - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(InvalidValueError): stix2.v21.Bundle(custom_identity) bundle = stix2.v21.Bundle(custom_identity, allow_custom=True) @@ -203,7 +204,7 @@ def test_custom_property_object_in_observable_extension(): def test_custom_property_dict_in_observable_extension(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(InvalidValueError): stix2.v21.File( name='test', extensions={ @@ -722,7 +723,7 @@ def test_custom_extension(): def test_custom_extension_wrong_observable_type(): # NewExtension is an extension of DomainName, not File ext = NewExtension(property1='something') - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.v21.File( name="abc.txt", extensions={ @@ -915,7 +916,7 @@ def test_parse_observable_with_custom_extension(): ], ) def test_parse_observable_with_unregistered_custom_extension(data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.parse_observable(data, version='2.1') assert "Can't parse unknown extension type" in str(excinfo.value) diff --git a/stix2/test/v21/test_granular_markings.py b/stix2/test/v21/test_granular_markings.py index e178f86..1c3194b 100644 --- a/stix2/test/v21/test_granular_markings.py +++ b/stix2/test/v21/test_granular_markings.py @@ -1,7 +1,7 @@ import pytest from stix2 import markings -from stix2.exceptions import MarkingNotFoundError +from stix2.exceptions import InvalidSelectorError, MarkingNotFoundError from stix2.v21 import TLP_RED, Malware from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST @@ -209,7 +209,7 @@ def test_add_marking_mark_same_property_same_marking(): ], ) def test_add_marking_bad_selector(data, marking): - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.add_markings(data, marking[0], marking[1]) @@ -329,7 +329,7 @@ def test_get_markings_multiple_selectors(data): ) def test_get_markings_bad_selector(data, selector): """Test bad selectors raise exception""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.get_markings(data, selector) @@ -714,7 +714,7 @@ def test_remove_marking_bad_selector(): before = { "description": "test description", } - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"]) @@ -805,7 +805,7 @@ def test_is_marked_smoke(data): ) def test_is_marked_invalid_selector(data, selector): """Test invalid selector raises an error.""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.is_marked(data, selectors=selector) @@ -1000,7 +1000,7 @@ def test_is_marked_positional_arguments_combinations(): def test_create_sdo_with_invalid_marking(): - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(InvalidSelectorError) as excinfo: Malware( granular_markings=[ { @@ -1192,7 +1192,7 @@ def test_set_marking_bad_selector(marking): **MALWARE_KWARGS ) - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): before = markings.set_markings(before, marking[0], marking[1]) assert before == after @@ -1298,7 +1298,7 @@ def test_clear_marking_all_selectors(data): ) def test_clear_marking_bad_selector(data, selector): """Test bad selector raises exception.""" - with pytest.raises(AssertionError): + with pytest.raises(InvalidSelectorError): markings.clear_markings(data, selector) diff --git a/stix2/test/v21/test_grouping.py b/stix2/test/v21/test_grouping.py index 449400b..a92a180 100644 --- a/stix2/test/v21/test_grouping.py +++ b/stix2/test/v21/test_grouping.py @@ -124,5 +124,5 @@ def test_parse_grouping(data): assert grp.context == "suspicious-activity" assert grp.object_refs == [ "malware--c8d2fae5-7271-400c-b81d-931a4caf20b9", - "identity--988145ed-a3b4-4421-b7a7-273376be67ce" + "identity--988145ed-a3b4-4421-b7a7-273376be67ce", ] diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index c734334..e8c8597 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -5,6 +5,7 @@ import pytest import pytz import stix2 +import stix2.exceptions from .constants import LOCATION_ID @@ -111,7 +112,7 @@ def test_parse_location(data): ], ) def test_location_bad_latitude(data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.parse(data) assert "Invalid value for Location 'latitude'" in str(excinfo.value) @@ -140,7 +141,7 @@ def test_location_bad_latitude(data): ], ) def test_location_bad_longitude(data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.parse(data) assert "Invalid value for Location 'longitude'" in str(excinfo.value) @@ -190,7 +191,7 @@ def test_location_properties_missing_when_precision_is_present(data): ], ) def test_location_negative_precision(data): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.parse(data) assert "Invalid value for Location 'precision'" in str(excinfo.value) diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index 9ae0ed2..0fc652a 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -6,6 +6,7 @@ import pytz import stix2 +from ...exceptions import InvalidValueError from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS EXPECTED_MALWARE = """{ @@ -136,7 +137,7 @@ def test_parse_malware(data): def test_parse_malware_invalid_labels(): data = re.compile('\\[.+\\]', re.DOTALL).sub('1', EXPECTED_MALWARE) - with pytest.raises(ValueError) as excinfo: + with pytest.raises(InvalidValueError) as excinfo: stix2.parse(data) assert "Invalid value for Malware 'malware_types'" in str(excinfo.value) diff --git a/stix2/test/v21/test_object_markings.py b/stix2/test/v21/test_object_markings.py index 7b19d4f..a21fbf6 100644 --- a/stix2/test/v21/test_object_markings.py +++ b/stix2/test/v21/test_object_markings.py @@ -3,6 +3,7 @@ import pytest from stix2 import exceptions, markings from stix2.v21 import TLP_AMBER, Malware +from ...exceptions import MarkingNotFoundError from .constants import FAKE_TIME, MALWARE_ID from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST from .constants import MARKING_IDS @@ -349,7 +350,7 @@ def test_remove_markings_bad_markings(): object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], **MALWARE_KWARGS ) - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(MarkingNotFoundError) as excinfo: markings.remove_markings(before, [MARKING_IDS[4]], None) assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 09e6a67..0e97ca6 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -305,7 +305,7 @@ def test_parse_artifact_valid(data): ) def test_parse_artifact_invalid(data): odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) - with pytest.raises(ValueError): + with pytest.raises(stix2.exceptions.InvalidValueError): stix2.parse(odata_str, version="2.1") @@ -534,11 +534,10 @@ def test_parse_email_message_with_at_least_one_error(data): "4": "artifact", "5": "file", } - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.parse_observable(data, valid_refs, version='2.1') - assert excinfo.value.cls == stix2.v21.EmailMIMEComponent - assert excinfo.value.properties == ["body", "body_raw_ref"] + assert excinfo.value.cls == stix2.v21.EmailMessage assert "At least one of the" in str(excinfo.value) assert "must be populated" in str(excinfo.value) @@ -788,7 +787,7 @@ def test_file_example_with_NTFSExt(): def test_file_example_with_empty_NTFSExt(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.File( name="abc.txt", extensions={ @@ -796,8 +795,7 @@ def test_file_example_with_empty_NTFSExt(): }, ) - assert excinfo.value.cls == stix2.v21.NTFSExt - assert excinfo.value.properties == sorted(list(stix2.NTFSExt._properties.keys())) + assert excinfo.value.cls == stix2.v21.File def test_file_example_with_PDFExt(): @@ -1152,14 +1150,12 @@ def test_process_example_empty_error(): def test_process_example_empty_with_extensions(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Process(extensions={ "windows-process-ext": {}, }) - assert excinfo.value.cls == stix2.v21.WindowsProcessExt - properties_of_extension = list(stix2.v21.WindowsProcessExt._properties.keys()) - assert excinfo.value.properties == sorted(properties_of_extension) + assert excinfo.value.cls == stix2.v21.Process def test_process_example_windows_process_ext(): @@ -1181,7 +1177,7 @@ def test_process_example_windows_process_ext(): def test_process_example_windows_process_ext_empty(): - with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Process( pid=1221, extensions={ @@ -1189,9 +1185,7 @@ def test_process_example_windows_process_ext_empty(): }, ) - assert excinfo.value.cls == stix2.v21.WindowsProcessExt - properties_of_extension = list(stix2.v21.WindowsProcessExt._properties.keys()) - assert excinfo.value.properties == sorted(properties_of_extension) + assert excinfo.value.cls == stix2.v21.Process def test_process_example_extensions_empty(): @@ -1324,7 +1318,7 @@ def test_user_account_unix_account_ext_example(): def test_windows_registry_key_example(): - with pytest.raises(ValueError): + with pytest.raises(stix2.exceptions.InvalidValueError): stix2.v21.WindowsRegistryValueType( name="Foo", data="qwerty", diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 557e419..fde13d3 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -1,7 +1,9 @@ import pytest import stix2 -from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError +from stix2.exceptions import ( + AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, +) from stix2.properties import ( BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, @@ -474,23 +476,27 @@ def test_extension_property_valid(): }) -@pytest.mark.parametrize( - "data", [ - 1, - {'foobar-ext': { - 'pe_type': 'exe', - }}, - ], -) -def test_extension_property_invalid(data): +def test_extension_property_invalid1(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') with pytest.raises(ValueError): - ext_prop.clean(data) + ext_prop.clean(1) + + +def test_extension_property_invalid2(): + ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') + with pytest.raises(CustomContentError): + ext_prop.clean( + { + 'foobar-ext': { + 'pe_type': 'exe', + }, + }, + ) def test_extension_property_invalid_type(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='indicator') - with pytest.raises(ValueError) as excinfo: + with pytest.raises(CustomContentError) as excinfo: ext_prop.clean( { 'windows-pebinary-ext': { diff --git a/stix2/test/v21/test_workbench.py b/stix2/test/v21/test_workbench.py index 0d84422..3a4a3fd 100644 --- a/stix2/test/v21/test_workbench.py +++ b/stix2/test/v21/test_workbench.py @@ -1,7 +1,5 @@ import os -import pytest - import stix2 from stix2.workbench import ( AttackPattern, Campaign, CourseOfAction, ExternalReference, @@ -24,7 +22,6 @@ from .constants import ( ) -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_environment(): # Create a STIX object @@ -79,7 +76,6 @@ def test_workbench_get_all_identities(): assert resp[0].id == IDENTITY_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_get_all_indicators(): resp = indicators() assert len(resp) == 1 @@ -95,7 +91,6 @@ def test_workbench_get_all_intrusion_sets(): assert resp[0].id == INTRUSION_SET_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_get_all_malware(): mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) save(mal) @@ -114,7 +109,6 @@ def test_workbench_get_all_observed_data(): assert resp[0].id == OBSERVED_DATA_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_get_all_reports(): rep = Report(id=REPORT_ID, **REPORT_KWARGS) save(rep) @@ -124,7 +118,6 @@ def test_workbench_get_all_reports(): assert resp[0].id == REPORT_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_get_all_threat_actors(): thr = ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS) save(thr) @@ -134,7 +127,6 @@ def test_workbench_get_all_threat_actors(): assert resp[0].id == THREAT_ACTOR_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_get_all_tools(): tool = Tool(id=TOOL_ID, **TOOL_KWARGS) save(tool) @@ -159,7 +151,6 @@ def test_workbench_add_to_bundle(): assert bundle.objects[0].name == 'Heartbleed' -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_relationships(): rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID) save(rel) @@ -179,7 +170,6 @@ def test_workbench_created_by(): assert creator.id == IDENTITY_ID -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_related(): rel1 = Relationship(MALWARE_ID, 'targets', IDENTITY_ID) rel2 = Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID) @@ -195,7 +185,6 @@ def test_workbench_related(): assert len(resp) == 1 -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_workbench_related_with_filters(): malware = Malware( malware_types=["ransomware"], name="CryptorBit", @@ -216,7 +205,6 @@ def test_workbench_related_with_filters(): assert len(resp) == 1 -@pytest.mark.xfail(reason='The workbench is not working correctly for 2.1') def test_add_data_source(): fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") fs = FileSystemSource(fs_path) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 66000c1..490de39 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -7,7 +7,7 @@ from six.moves.urllib.parse import quote_plus from ..core import STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import InvalidPropertyConfigurationError +from ..exceptions import PropertyPresenceError from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -76,7 +76,7 @@ class Campaign(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(Campaign, self)._check_object_constraints() first_seen = self.get('first_seen') last_seen = self.get('last_seen') @@ -215,7 +215,7 @@ class Indicator(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(Indicator, self)._check_object_constraints() valid_from = self.get('valid_from') valid_until = self.get('valid_until') @@ -256,7 +256,7 @@ class Infrastructure(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(Infrastructure, self)._check_object_constraints() first_seen = self.get('first_seen') last_seen = self.get('last_seen') @@ -299,7 +299,7 @@ class IntrusionSet(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(IntrusionSet, self)._check_object_constraints() first_seen = self.get('first_seen') last_seen = self.get('last_seen') @@ -344,7 +344,7 @@ class Location(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(Location, self)._check_object_constraints() if self.get('precision') is not None: self._check_properties_dependency(['longitude', 'latitude'], ['precision']) @@ -360,10 +360,10 @@ class Location(STIXDomainObject): and 'longitude' in self ) ): - raise InvalidPropertyConfigurationError( + raise PropertyPresenceError( "Location objects must have the properties 'region', " "'country', or 'latitude' and 'longitude'", - Location + Location, ) def to_maps_url(self, map_engine="Google Maps"): @@ -454,7 +454,7 @@ class Malware(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(Malware, self)._check_object_constraints() first_seen = self.get('first_seen') last_seen = self.get('last_seen') @@ -464,9 +464,9 @@ class Malware(STIXDomainObject): raise ValueError(msg.format(self)) if self.is_family and "name" not in self: - raise InvalidPropertyConfigurationError( + raise PropertyPresenceError( "'name' is a required property for malware families", - Malware + Malware, ) @@ -576,7 +576,7 @@ class ObservedData(STIXDomainObject): super(ObservedData, self).__init__(*args, **kwargs) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(ObservedData, self)._check_object_constraints() first_observed = self.get('first_observed') last_observed = self.get('last_observed') @@ -694,7 +694,7 @@ class ThreatActor(STIXDomainObject): ]) def _check_object_constraints(self): - super(self.__class__, self)._check_object_constraints() + super(ThreatActor, self)._check_object_constraints() first_observed = self.get('first_seen') last_observed = self.get('last_seen') diff --git a/stix2/workbench.py b/stix2/workbench.py index e621073..d43778f 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -20,6 +20,7 @@ """ +import functools import stix2 from . import AttackPattern as _AttackPattern from . import Campaign as _Campaign @@ -122,42 +123,35 @@ def _observed_data_init(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) -def _constructor_wrapper(obj_type): - # Use an intermediate wrapper class so the implicit environment will create objects that have our wrapper functions - class_dict = dict( - created_by=_created_by_wrapper, - relationships=_relationships_wrapper, - related=_related_wrapper, - **obj_type.__dict__ - ) - - # Avoid TypeError about super() in ObservedData - if 'ObservedData' in obj_type.__name__: - class_dict['__init__'] = _observed_data_init - - wrapped_type = type(obj_type.__name__, obj_type.__bases__, class_dict) - - @staticmethod - def new_constructor(cls, *args, **kwargs): - x = _environ.create(wrapped_type, *args, **kwargs) - return x - return new_constructor - - def _setup_workbench(): - # Create wrapper classes whose constructors call the implicit environment's create() for obj_type in STIX_OBJS: - new_class_dict = { - '__new__': _constructor_wrapper(obj_type), - '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS), - } - new_class = type(obj_type.__name__, (), new_class_dict) - # Add our new class to this module's globals and to the library-wide mapping. - # This allows parse() to use the wrapped classes. - globals()[obj_type.__name__] = new_class - stix2.OBJ_MAP[obj_type._type] = new_class - new_class = None + # The idea here was originally to dynamically create subclasses which + # were cleverly customized such that instantiating them would actually + # invoke _environ.create(). This turns out to be impossible, since + # __new__ can never create the class in the normal way, since that + # invokes __new__ again, resulting in infinite recursion. And + # _environ.create() does exactly that. + # + # So instead, we create something "class-like", in that calling it + # produces an instance of the desired class. But these things will + # be functions instead of classes. One might think this trickery will + # have undesirable side-effects, but actually it seems to work. + # So far... + new_class_dict = { + '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS), + 'created_by': _created_by_wrapper, + 'relationships': _relationships_wrapper, + 'related': _related_wrapper, + } + + new_class = type(obj_type.__name__, (obj_type,), new_class_dict) + factory_func = functools.partial(_environ.create, new_class) + + # Add our new "class" to this module's globals and to the library-wide + # mapping. This allows parse() to use the wrapped classes. + globals()[obj_type.__name__] = factory_func + stix2.OBJ_MAP[obj_type._type] = factory_func _setup_workbench() diff --git a/tox.ini b/tox.ini index f3a10fb..5deb4ef 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,7 @@ deps = taxii2-client medallion commands = - pytest --ignore=stix2/test/v20/test_workbench.py --ignore=stix2/test/v21/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing - pytest stix2/test/v20/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append - pytest stix2/test/v21/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append + pytest --cov=stix2 stix2/test/ --cov-report term-missing passenv = CI TRAVIS TRAVIS_* From 823b67a4fc8961f8dbb6d90187e90f3e435519c9 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 19 Jul 2019 15:40:03 -0400 Subject: [PATCH 28/62] Add a few more tests to exercise more complex property presence constraint checking. --- stix2/test/v21/test_location.py | 9 +++++++++ stix2/test/v21/test_malware.py | 23 ++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index e8c8597..7517fdf 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -265,6 +265,15 @@ def test_location_lat_or_lon_dependency_missing(data, msg): assert msg in str(excinfo.value) +def test_location_complex_presence_constraint(): + with pytest.raises(stix2.exceptions.PropertyPresenceError): + stix2.parse({ + "type": "location", + "spec_version": "2.1", + "id": LOCATION_ID, + }) + + def test_google_map_url_long_lat_provided(): expected_url = "https://www.google.com/maps/search/?api=1&query=41.862401%2C-87.616001" diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index 0fc652a..53838c9 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -6,7 +6,7 @@ import pytz import stix2 -from ...exceptions import InvalidValueError +from ...exceptions import InvalidValueError, PropertyPresenceError from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS EXPECTED_MALWARE = """{ @@ -176,3 +176,24 @@ def test_malware_invalid_last_before_first(): stix2.v21.Malware(first_seen="2017-01-01T12:34:56.000Z", last_seen="2017-01-01T12:33:56.000Z", **MALWARE_KWARGS) assert "'last_seen' must be greater than or equal to 'first_seen'" in str(excinfo.value) + + +def test_malware_family_no_name(): + with pytest.raises(PropertyPresenceError): + stix2.parse({ + "type": "malware", + "id": MALWARE_ID, + "spec_version": "2.1", + "is_family": True, + "malware_types": ["a type"], + }) + + +def test_malware_non_family_no_name(): + stix2.parse({ + "type": "malware", + "id": MALWARE_ID, + "spec_version": "2.1", + "is_family": False, + "malware_types": ["something"], + }) From 227383cdcbfc741cb66e5aa2a411c28d5643f461 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 19 Jul 2019 15:58:15 -0400 Subject: [PATCH 29/62] Removed _observed_data_init() from workbench.py, part of the old monkeypatching algorithm. It's no longer needed and I forgot to delete it. --- stix2/workbench.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/stix2/workbench.py b/stix2/workbench.py index d43778f..0d20034 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -117,12 +117,6 @@ def _related_wrapper(self, *args, **kwargs): return _environ.related_to(self, *args, **kwargs) -def _observed_data_init(self, *args, **kwargs): - self.__allow_custom = kwargs.get('allow_custom', False) - self._properties['objects'].allow_custom = kwargs.get('allow_custom', False) - super(self.__class__, self).__init__(*args, **kwargs) - - def _setup_workbench(): for obj_type in STIX_OBJS: From 165d87e1036e71eb4057ca308df7c20fb551dec7 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 22 Jul 2019 16:55:22 -0400 Subject: [PATCH 30/62] Revert the import in the top-level stix2 package, to v20. This additionally required: - Removing the v21 workbench test suite and reinstating the v20 test suite - Fixing up a few v20 unit tests to work with the workbench monkeypatching. - I didn't revert the analogous changes I'd previously made to the v21 unit tests, because I think they make sense even when the workbench monkeypatching isn't happening. --- stix2/__init__.py | 2 +- stix2/test/v20/test_bundle.py | 2 +- stix2/test/v20/test_core.py | 4 +++- stix2/test/v20/test_custom.py | 2 +- stix2/test/{v21 => v20}/test_workbench.py | 15 ++++++--------- 5 files changed, 12 insertions(+), 13 deletions(-) rename stix2/test/{v21 => v20}/test_workbench.py (96%) diff --git a/stix2/__init__.py b/stix2/__init__.py index 68e0264..714bf46 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -54,7 +54,7 @@ from .patterns import ( WithinQualifier, ) from .utils import new_version, revoke -from .v21 import * # This import will always be the latest STIX 2.X version +from .v20 import * # This import will always be the latest STIX 2.X version from .version import __version__ _collect_stix2_mappings() diff --git a/stix2/test/v20/test_bundle.py b/stix2/test/v20/test_bundle.py index df59f89..f53d0cb 100644 --- a/stix2/test/v20/test_bundle.py +++ b/stix2/test/v20/test_bundle.py @@ -176,7 +176,7 @@ def test_parse_bundle(version): assert bundle.type == "bundle" assert bundle.id.startswith("bundle--") - assert type(bundle.objects[0]) is stix2.v20.Indicator + assert isinstance(bundle.objects[0], stix2.v20.Indicator) assert bundle.objects[0].type == 'indicator' assert bundle.objects[1].type == 'malware' assert bundle.objects[2].type == 'relationship' diff --git a/stix2/test/v20/test_core.py b/stix2/test/v20/test_core.py index c2056b8..d2efa22 100644 --- a/stix2/test/v20/test_core.py +++ b/stix2/test/v20/test_core.py @@ -74,7 +74,9 @@ def test_register_object_with_version(): v = 'v20' assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] - assert v in str(bundle.objects[0].__class__) + # spec_version is not in STIX 2.0, and is required in 2.1, so this + # suffices as a test for a STIX 2.0 object. + assert "spec_version" not in bundle.objects[0] def test_register_marking_with_version(): diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 8a99b2c..3cfc23f 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -96,7 +96,7 @@ def test_identity_custom_property_allowed(): def test_parse_identity_custom_property(data): with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: stix2.parse(data, version="2.0") - assert excinfo.value.cls == stix2.v20.Identity + assert issubclass(excinfo.value.cls, stix2.v20.Identity) assert excinfo.value.properties == ['foo'] assert "Unexpected properties for" in str(excinfo.value) diff --git a/stix2/test/v21/test_workbench.py b/stix2/test/v20/test_workbench.py similarity index 96% rename from stix2/test/v21/test_workbench.py rename to stix2/test/v20/test_workbench.py index 3a4a3fd..c254966 100644 --- a/stix2/test/v21/test_workbench.py +++ b/stix2/test/v20/test_workbench.py @@ -29,7 +29,7 @@ def test_workbench_environment(): save(ind) resp = get(INDICATOR_ID) - assert resp['indicator_types'][0] == 'malicious-activity' + assert resp['labels'][0] == 'malicious-activity' resp = all_versions(INDICATOR_ID) assert len(resp) == 1 @@ -147,7 +147,7 @@ def test_workbench_get_all_vulnerabilities(): def test_workbench_add_to_bundle(): vuln = Vulnerability(**VULNERABILITY_KWARGS) - bundle = stix2.v21.Bundle(vuln) + bundle = stix2.v20.Bundle(vuln) assert bundle.objects[0].name == 'Heartbleed' @@ -186,10 +186,7 @@ def test_workbench_related(): def test_workbench_related_with_filters(): - malware = Malware( - malware_types=["ransomware"], name="CryptorBit", - created_by_ref=IDENTITY_ID, is_family=False, - ) + malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID) rel = Relationship(malware.id, 'variant-of', MALWARE_ID) save([malware, rel]) @@ -274,12 +271,12 @@ def test_default_object_marking_refs(): def test_workbench_custom_property_object_in_observable_extension(): - ntfs = stix2.v21.NTFSExt( + ntfs = stix2.v20.NTFSExt( allow_custom=True, sid=1, x_foo='bar', ) - artifact = stix2.v21.File( + artifact = stix2.v20.File( name='test', extensions={'ntfs-ext': ntfs}, ) @@ -296,7 +293,7 @@ def test_workbench_custom_property_object_in_observable_extension(): def test_workbench_custom_property_dict_in_observable_extension(): - artifact = stix2.v21.File( + artifact = stix2.v20.File( allow_custom=True, name='test', extensions={ From d69449706fa28b3336b436fdd4d4b784bdedb17d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 22 Jul 2019 17:01:52 -0400 Subject: [PATCH 31/62] Revert the docstrings generated for the workbench dynamically created subclasses, to mention v20 instead of v21. --- stix2/workbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/workbench.py b/stix2/workbench.py index 0d20034..f0abdf9 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -133,7 +133,7 @@ def _setup_workbench(): # have undesirable side-effects, but actually it seems to work. # So far... new_class_dict = { - '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS), + '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS), 'created_by': _created_by_wrapper, 'relationships': _relationships_wrapper, 'related': _related_wrapper, From 38103ac6c54406c579539fa02246beaccc79d5ee Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jul 2019 15:35:59 -0400 Subject: [PATCH 32/62] Moved test/v20/test_workbench.py up one directory level since it doesn't make sense to have a test per STIX version. The workbench only uses the latest supported STIX version. In order to make this work, the test suite was modified to dynamically compute some settings like where to get demo data, based on the value of stix2.DEFAULT_VERSION. Switched stix2.DEFAULT_VERSION back to "2.0", since I figure it should be sync'd up with the 'from .vxx import *' import statement from the top level package. --- stix2/__init__.py | 2 +- stix2/test/{v20 => }/test_workbench.py | 152 ++++++++++++++----------- 2 files changed, 89 insertions(+), 65 deletions(-) rename stix2/test/{v20 => }/test_workbench.py (59%) diff --git a/stix2/__init__.py b/stix2/__init__.py index 714bf46..c9384a0 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -20,7 +20,7 @@ # flake8: noqa -DEFAULT_VERSION = '2.1' # Default version will always be the latest STIX 2.X version +DEFAULT_VERSION = '2.0' # Default version will always be the latest STIX 2.X version from .confidence import scales from .core import _collect_stix2_mappings, parse, parse_observable diff --git a/stix2/test/v20/test_workbench.py b/stix2/test/test_workbench.py similarity index 59% rename from stix2/test/v20/test_workbench.py rename to stix2/test/test_workbench.py index c254966..a85ab0e 100644 --- a/stix2/test/v20/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -1,3 +1,4 @@ +import importlib import os import stix2 @@ -12,26 +13,32 @@ from stix2.workbench import ( set_default_object_marking_refs, threat_actors, tools, vulnerabilities, ) -from .constants import ( - ATTACK_PATTERN_ID, ATTACK_PATTERN_KWARGS, CAMPAIGN_ID, CAMPAIGN_KWARGS, - COURSE_OF_ACTION_ID, COURSE_OF_ACTION_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, - INDICATOR_ID, INDICATOR_KWARGS, INTRUSION_SET_ID, INTRUSION_SET_KWARGS, - MALWARE_ID, MALWARE_KWARGS, OBSERVED_DATA_ID, OBSERVED_DATA_KWARGS, - REPORT_ID, REPORT_KWARGS, THREAT_ACTOR_ID, THREAT_ACTOR_KWARGS, TOOL_ID, - TOOL_KWARGS, VULNERABILITY_ID, VULNERABILITY_KWARGS, + +# Auto-detect some settings based on the current default STIX version +_STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") +_STIX_DATA_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + _STIX_VID, + "stix2_data" ) +_STIX_CONSTANTS_MODULE = "stix2.test." + _STIX_VID + ".constants" + + +constants = importlib.import_module(_STIX_CONSTANTS_MODULE) def test_workbench_environment(): # Create a STIX object - ind = create(Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS) + ind = create( + Indicator, id=constants.INDICATOR_ID, **constants.INDICATOR_KWARGS + ) save(ind) - resp = get(INDICATOR_ID) + resp = get(constants.INDICATOR_ID) assert resp['labels'][0] == 'malicious-activity' - resp = all_versions(INDICATOR_ID) + resp = all_versions(constants.INDICATOR_ID) assert len(resp) == 1 # Search on something other than id @@ -41,176 +48,193 @@ def test_workbench_environment(): def test_workbench_get_all_attack_patterns(): - mal = AttackPattern(id=ATTACK_PATTERN_ID, **ATTACK_PATTERN_KWARGS) + mal = AttackPattern( + id=constants.ATTACK_PATTERN_ID, **constants.ATTACK_PATTERN_KWARGS + ) save(mal) resp = attack_patterns() assert len(resp) == 1 - assert resp[0].id == ATTACK_PATTERN_ID + assert resp[0].id == constants.ATTACK_PATTERN_ID def test_workbench_get_all_campaigns(): - cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) + cam = Campaign(id=constants.CAMPAIGN_ID, **constants.CAMPAIGN_KWARGS) save(cam) resp = campaigns() assert len(resp) == 1 - assert resp[0].id == CAMPAIGN_ID + assert resp[0].id == constants.CAMPAIGN_ID def test_workbench_get_all_courses_of_action(): - coa = CourseOfAction(id=COURSE_OF_ACTION_ID, **COURSE_OF_ACTION_KWARGS) + coa = CourseOfAction( + id=constants.COURSE_OF_ACTION_ID, **constants.COURSE_OF_ACTION_KWARGS + ) save(coa) resp = courses_of_action() assert len(resp) == 1 - assert resp[0].id == COURSE_OF_ACTION_ID + assert resp[0].id == constants.COURSE_OF_ACTION_ID def test_workbench_get_all_identities(): - idty = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) + idty = Identity(id=constants.IDENTITY_ID, **constants.IDENTITY_KWARGS) save(idty) resp = identities() assert len(resp) == 1 - assert resp[0].id == IDENTITY_ID + assert resp[0].id == constants.IDENTITY_ID def test_workbench_get_all_indicators(): resp = indicators() assert len(resp) == 1 - assert resp[0].id == INDICATOR_ID + assert resp[0].id == constants.INDICATOR_ID def test_workbench_get_all_intrusion_sets(): - ins = IntrusionSet(id=INTRUSION_SET_ID, **INTRUSION_SET_KWARGS) + ins = IntrusionSet( + id=constants.INTRUSION_SET_ID, **constants.INTRUSION_SET_KWARGS + ) save(ins) resp = intrusion_sets() assert len(resp) == 1 - assert resp[0].id == INTRUSION_SET_ID + assert resp[0].id == constants.INTRUSION_SET_ID def test_workbench_get_all_malware(): - mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) + mal = Malware(id=constants.MALWARE_ID, **constants.MALWARE_KWARGS) save(mal) resp = malware() assert len(resp) == 1 - assert resp[0].id == MALWARE_ID + assert resp[0].id == constants.MALWARE_ID def test_workbench_get_all_observed_data(): - od = ObservedData(id=OBSERVED_DATA_ID, **OBSERVED_DATA_KWARGS) + od = ObservedData( + id=constants.OBSERVED_DATA_ID, **constants.OBSERVED_DATA_KWARGS + ) save(od) resp = observed_data() assert len(resp) == 1 - assert resp[0].id == OBSERVED_DATA_ID + assert resp[0].id == constants.OBSERVED_DATA_ID def test_workbench_get_all_reports(): - rep = Report(id=REPORT_ID, **REPORT_KWARGS) + rep = Report(id=constants.REPORT_ID, **constants.REPORT_KWARGS) save(rep) resp = reports() assert len(resp) == 1 - assert resp[0].id == REPORT_ID + assert resp[0].id == constants.REPORT_ID def test_workbench_get_all_threat_actors(): - thr = ThreatActor(id=THREAT_ACTOR_ID, **THREAT_ACTOR_KWARGS) + thr = ThreatActor( + id=constants.THREAT_ACTOR_ID, **constants.THREAT_ACTOR_KWARGS + ) save(thr) resp = threat_actors() assert len(resp) == 1 - assert resp[0].id == THREAT_ACTOR_ID + assert resp[0].id == constants.THREAT_ACTOR_ID def test_workbench_get_all_tools(): - tool = Tool(id=TOOL_ID, **TOOL_KWARGS) + tool = Tool(id=constants.TOOL_ID, **constants.TOOL_KWARGS) save(tool) resp = tools() assert len(resp) == 1 - assert resp[0].id == TOOL_ID + assert resp[0].id == constants.TOOL_ID def test_workbench_get_all_vulnerabilities(): - vuln = Vulnerability(id=VULNERABILITY_ID, **VULNERABILITY_KWARGS) + vuln = Vulnerability( + id=constants.VULNERABILITY_ID, **constants.VULNERABILITY_KWARGS + ) save(vuln) resp = vulnerabilities() assert len(resp) == 1 - assert resp[0].id == VULNERABILITY_ID + assert resp[0].id == constants.VULNERABILITY_ID def test_workbench_add_to_bundle(): - vuln = Vulnerability(**VULNERABILITY_KWARGS) + vuln = Vulnerability(**constants.VULNERABILITY_KWARGS) bundle = stix2.v20.Bundle(vuln) assert bundle.objects[0].name == 'Heartbleed' def test_workbench_relationships(): - rel = Relationship(INDICATOR_ID, 'indicates', MALWARE_ID) + rel = Relationship( + constants.INDICATOR_ID, 'indicates', constants.MALWARE_ID + ) save(rel) - ind = get(INDICATOR_ID) + ind = get(constants.INDICATOR_ID) resp = ind.relationships() assert len(resp) == 1 assert resp[0].relationship_type == 'indicates' - assert resp[0].source_ref == INDICATOR_ID - assert resp[0].target_ref == MALWARE_ID + assert resp[0].source_ref == constants.INDICATOR_ID + assert resp[0].target_ref == constants.MALWARE_ID def test_workbench_created_by(): - intset = IntrusionSet(name="Breach 123", created_by_ref=IDENTITY_ID) + intset = IntrusionSet( + name="Breach 123", created_by_ref=constants.IDENTITY_ID + ) save(intset) creator = intset.created_by() - assert creator.id == IDENTITY_ID + assert creator.id == constants.IDENTITY_ID def test_workbench_related(): - rel1 = Relationship(MALWARE_ID, 'targets', IDENTITY_ID) - rel2 = Relationship(CAMPAIGN_ID, 'uses', MALWARE_ID) + rel1 = Relationship(constants.MALWARE_ID, 'targets', constants.IDENTITY_ID) + rel2 = Relationship(constants.CAMPAIGN_ID, 'uses', constants.MALWARE_ID) save([rel1, rel2]) - resp = get(MALWARE_ID).related() + resp = get(constants.MALWARE_ID).related() assert len(resp) == 3 - assert any(x['id'] == CAMPAIGN_ID for x in resp) - assert any(x['id'] == INDICATOR_ID for x in resp) - assert any(x['id'] == IDENTITY_ID for x in resp) + assert any(x['id'] == constants.CAMPAIGN_ID for x in resp) + assert any(x['id'] == constants.INDICATOR_ID for x in resp) + assert any(x['id'] == constants.IDENTITY_ID for x in resp) - resp = get(MALWARE_ID).related(relationship_type='indicates') + resp = get(constants.MALWARE_ID).related(relationship_type='indicates') assert len(resp) == 1 def test_workbench_related_with_filters(): - malware = Malware(labels=["ransomware"], name="CryptorBit", created_by_ref=IDENTITY_ID) - rel = Relationship(malware.id, 'variant-of', MALWARE_ID) + malware = Malware( + labels=["ransomware"], name="CryptorBit", created_by_ref=constants.IDENTITY_ID + ) + rel = Relationship(malware.id, 'variant-of', constants.MALWARE_ID) save([malware, rel]) - filters = [Filter('created_by_ref', '=', IDENTITY_ID)] - resp = get(MALWARE_ID).related(filters=filters) + filters = [Filter('created_by_ref', '=', constants.IDENTITY_ID)] + resp = get(constants.MALWARE_ID).related(filters=filters) assert len(resp) == 1 assert resp[0].name == malware.name - assert resp[0].created_by_ref == IDENTITY_ID + assert resp[0].created_by_ref == constants.IDENTITY_ID # filters arg can also be single filter - resp = get(MALWARE_ID).related(filters=filters[0]) + resp = get(constants.MALWARE_ID).related(filters=filters[0]) assert len(resp) == 1 def test_add_data_source(): - fs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") - fs = FileSystemSource(fs_path) + fs = FileSystemSource(_STIX_DATA_PATH) add_data_source(fs) resp = tools() assert len(resp) == 3 resp_ids = [tool.id for tool in resp] - assert TOOL_ID in resp_ids + assert constants.TOOL_ID in resp_ids assert 'tool--03342581-f790-4f03-ba41-e82e67392e23' in resp_ids assert 'tool--242f3da3-4425-4d11-8f5c-b842886da966' in resp_ids @@ -229,19 +253,19 @@ def test_additional_filters_list(): def test_default_creator(): - set_default_creator(IDENTITY_ID) - campaign = Campaign(**CAMPAIGN_KWARGS) + set_default_creator(constants.IDENTITY_ID) + campaign = Campaign(**constants.CAMPAIGN_KWARGS) - assert 'created_by_ref' not in CAMPAIGN_KWARGS - assert campaign.created_by_ref == IDENTITY_ID + assert 'created_by_ref' not in constants.CAMPAIGN_KWARGS + assert campaign.created_by_ref == constants.IDENTITY_ID def test_default_created_timestamp(): timestamp = "2018-03-19T01:02:03.000Z" set_default_created(timestamp) - campaign = Campaign(**CAMPAIGN_KWARGS) + campaign = Campaign(**constants.CAMPAIGN_KWARGS) - assert 'created' not in CAMPAIGN_KWARGS + assert 'created' not in constants.CAMPAIGN_KWARGS assert stix2.utils.format_datetime(campaign.created) == timestamp assert stix2.utils.format_datetime(campaign.modified) == timestamp @@ -252,7 +276,7 @@ def test_default_external_refs(): description="Threat report", ) set_default_external_refs(ext_ref) - campaign = Campaign(**CAMPAIGN_KWARGS) + campaign = Campaign(**constants.CAMPAIGN_KWARGS) assert campaign.external_references[0].source_name == "ACME Threat Intel" assert campaign.external_references[0].description == "Threat report" @@ -265,7 +289,7 @@ def test_default_object_marking_refs(): definition=stmt_marking, ) set_default_object_marking_refs(mark_def) - campaign = Campaign(**CAMPAIGN_KWARGS) + campaign = Campaign(**constants.CAMPAIGN_KWARGS) assert campaign.object_marking_refs[0] == mark_def.id From 9d08cadcfdbe6809301259d70b30bc48e63eb4b4 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jul 2019 16:23:19 -0400 Subject: [PATCH 33/62] Turn off the workbench test suite's side effects after each test that turns them on. These have the potential to affect subsequent tests. The side effects include automatically setting property values, and automatically appending additional values to list-valued properties. --- stix2/test/test_workbench.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index a85ab0e..0bd52af 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -259,6 +259,9 @@ def test_default_creator(): assert 'created_by_ref' not in constants.CAMPAIGN_KWARGS assert campaign.created_by_ref == constants.IDENTITY_ID + # turn off side-effects to avoid affecting future tests + set_default_creator(None) + def test_default_created_timestamp(): timestamp = "2018-03-19T01:02:03.000Z" @@ -269,6 +272,9 @@ def test_default_created_timestamp(): assert stix2.utils.format_datetime(campaign.created) == timestamp assert stix2.utils.format_datetime(campaign.modified) == timestamp + # turn off side-effects to avoid affecting future tests + set_default_created(None) + def test_default_external_refs(): ext_ref = ExternalReference( @@ -281,6 +287,9 @@ def test_default_external_refs(): assert campaign.external_references[0].source_name == "ACME Threat Intel" assert campaign.external_references[0].description == "Threat report" + # turn off side-effects to avoid affecting future tests + set_default_external_refs([]) + def test_default_object_marking_refs(): stmt_marking = StatementMarking("Copyright 2016, Example Corp") @@ -293,6 +302,9 @@ def test_default_object_marking_refs(): assert campaign.object_marking_refs[0] == mark_def.id + # turn off side-effects to avoid affecting future tests + set_default_object_marking_refs([]) + def test_workbench_custom_property_object_in_observable_extension(): ntfs = stix2.v20.NTFSExt( From 22face6c1aaa8d16c803747bba352b8fab85660c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jul 2019 16:30:18 -0400 Subject: [PATCH 34/62] Add trailing commas to satisfy pre-commit hooks... --- stix2/test/test_workbench.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index 0bd52af..d1d4328 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -13,13 +13,12 @@ from stix2.workbench import ( set_default_object_marking_refs, threat_actors, tools, vulnerabilities, ) - # Auto-detect some settings based on the current default STIX version _STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") _STIX_DATA_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), _STIX_VID, - "stix2_data" + "stix2_data", ) _STIX_CONSTANTS_MODULE = "stix2.test." + _STIX_VID + ".constants" @@ -172,7 +171,7 @@ def test_workbench_add_to_bundle(): def test_workbench_relationships(): rel = Relationship( - constants.INDICATOR_ID, 'indicates', constants.MALWARE_ID + constants.INDICATOR_ID, 'indicates', constants.MALWARE_ID, ) save(rel) @@ -186,7 +185,7 @@ def test_workbench_relationships(): def test_workbench_created_by(): intset = IntrusionSet( - name="Breach 123", created_by_ref=constants.IDENTITY_ID + name="Breach 123", created_by_ref=constants.IDENTITY_ID, ) save(intset) creator = intset.created_by() @@ -210,7 +209,7 @@ def test_workbench_related(): def test_workbench_related_with_filters(): malware = Malware( - labels=["ransomware"], name="CryptorBit", created_by_ref=constants.IDENTITY_ID + labels=["ransomware"], name="CryptorBit", created_by_ref=constants.IDENTITY_ID, ) rel = Relationship(malware.id, 'variant-of', constants.MALWARE_ID) save([malware, rel]) From b0eb518997ff436b00fc8b110a14befb95197fb2 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jul 2019 17:20:52 -0400 Subject: [PATCH 35/62] Added adaptability to the workbench module, regarding the autogenerated docstrings: v20/v21 is automatically referenced as appropriate, based on stix2.DEFAULT_VERSION. To avoid duplication, I also moved _STIX_VID from test_workbench.py to workbench.py; the former now imports it from the latter. --- stix2/test/test_workbench.py | 3 +-- stix2/workbench.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index d1d4328..fd62703 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -3,7 +3,7 @@ import os import stix2 from stix2.workbench import ( - AttackPattern, Campaign, CourseOfAction, ExternalReference, + _STIX_VID, AttackPattern, Campaign, CourseOfAction, ExternalReference, FileSystemSource, Filter, Identity, Indicator, IntrusionSet, Malware, MarkingDefinition, ObservedData, Relationship, Report, StatementMarking, ThreatActor, Tool, Vulnerability, add_data_source, all_versions, @@ -14,7 +14,6 @@ from stix2.workbench import ( ) # Auto-detect some settings based on the current default STIX version -_STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") _STIX_DATA_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), _STIX_VID, diff --git a/stix2/workbench.py b/stix2/workbench.py index f0abdf9..c5aac6e 100644 --- a/stix2/workbench.py +++ b/stix2/workbench.py @@ -53,6 +53,11 @@ from . import ( # noqa: F401 ) from .datastore.filters import FilterSet + +# Enable some adaptation to the current default supported STIX version. +_STIX_VID = "v" + stix2.DEFAULT_VERSION.replace(".", "") + + # Use an implicit MemoryStore _environ = Environment(store=MemoryStore()) @@ -133,7 +138,11 @@ def _setup_workbench(): # have undesirable side-effects, but actually it seems to work. # So far... new_class_dict = { - '__doc__': 'Workbench wrapper around the `{0} `__ object. {1}'.format(obj_type.__name__, STIX_OBJ_DOCS), + '__doc__': 'Workbench wrapper around the `{0} `__ object. {2}'.format( + obj_type.__name__, + _STIX_VID, + STIX_OBJ_DOCS, + ), 'created_by': _created_by_wrapper, 'relationships': _relationships_wrapper, 'related': _related_wrapper, From 5649559c6d876e4343c266119344dbc0674fa340 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jul 2019 17:39:00 -0400 Subject: [PATCH 36/62] Removed some more hard-codings of v20 in the workbench test suite. --- stix2/test/test_workbench.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index fd62703..d946547 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -3,13 +3,14 @@ import os import stix2 from stix2.workbench import ( - _STIX_VID, AttackPattern, Campaign, CourseOfAction, ExternalReference, - FileSystemSource, Filter, Identity, Indicator, IntrusionSet, Malware, - MarkingDefinition, ObservedData, Relationship, Report, StatementMarking, - ThreatActor, Tool, Vulnerability, add_data_source, all_versions, - attack_patterns, campaigns, courses_of_action, create, get, identities, - indicators, intrusion_sets, malware, observed_data, query, reports, save, - set_default_created, set_default_creator, set_default_external_refs, + _STIX_VID, AttackPattern, Bundle, Campaign, CourseOfAction, + ExternalReference, File, FileSystemSource, Filter, Identity, Indicator, + IntrusionSet, Malware, MarkingDefinition, NTFSExt, ObservedData, + Relationship, Report, StatementMarking, ThreatActor, Tool, Vulnerability, + add_data_source, all_versions, attack_patterns, campaigns, + courses_of_action, create, get, identities, indicators, intrusion_sets, + malware, observed_data, query, reports, save, set_default_created, + set_default_creator, set_default_external_refs, set_default_object_marking_refs, threat_actors, tools, vulnerabilities, ) @@ -164,7 +165,7 @@ def test_workbench_get_all_vulnerabilities(): def test_workbench_add_to_bundle(): vuln = Vulnerability(**constants.VULNERABILITY_KWARGS) - bundle = stix2.v20.Bundle(vuln) + bundle = Bundle(vuln) assert bundle.objects[0].name == 'Heartbleed' @@ -305,12 +306,12 @@ def test_default_object_marking_refs(): def test_workbench_custom_property_object_in_observable_extension(): - ntfs = stix2.v20.NTFSExt( + ntfs = NTFSExt( allow_custom=True, sid=1, x_foo='bar', ) - artifact = stix2.v20.File( + artifact = File( name='test', extensions={'ntfs-ext': ntfs}, ) @@ -327,7 +328,7 @@ def test_workbench_custom_property_object_in_observable_extension(): def test_workbench_custom_property_dict_in_observable_extension(): - artifact = stix2.v20.File( + artifact = File( allow_custom=True, name='test', extensions={ From 8362d80206e8c06adeab95d948462968cd195591 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 25 Jul 2019 16:56:34 -0400 Subject: [PATCH 37/62] Change "object_modified" property of LocationContent to be optional. Add a corresponding unit test. --- stix2/test/v21/test_language_content.py | 15 +++++++++++++++ stix2/v21/common.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/stix2/test/v21/test_language_content.py b/stix2/test/v21/test_language_content.py index 4f541e0..95adebc 100644 --- a/stix2/test/v21/test_language_content.py +++ b/stix2/test/v21/test_language_content.py @@ -71,3 +71,18 @@ def test_language_content_campaign(): # or https://docs.python.org/2/library/json.html#json.dumps assert lc.serialize(pretty=True, ensure_ascii=False) == TEST_LANGUAGE_CONTENT assert lc.modified == camp.modified + + +def test_object_modified_optional(): + """ + object_modified is now optional in STIX 2.1. + """ + + stix2.v21.LanguageContent( + object_ref=CAMPAIGN_ID, + contents={ + "en": { + "name": "the english text", + }, + }, + ) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index f85bf88..70be5dc 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -79,7 +79,7 @@ class LanguageContent(_STIXBase): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('object_ref', ReferenceProperty(spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. - ('object_modified', TimestampProperty(required=True, precision='millisecond')), + ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx ('contents', DictionaryProperty(spec_version='2.1', required=True)), ('revoked', BooleanProperty()), From 423487d65a3b056d8f39d74a4273d96896fd8748 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 25 Jul 2019 16:57:15 -0400 Subject: [PATCH 38/62] Add a unit test for the first/last_seen value co-constraint on ThreatActor. --- stix2/test/v21/test_threat_actor.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/stix2/test/v21/test_threat_actor.py b/stix2/test/v21/test_threat_actor.py index 5468731..ecd7b00 100644 --- a/stix2/test/v21/test_threat_actor.py +++ b/stix2/test/v21/test_threat_actor.py @@ -4,6 +4,7 @@ import pytest import pytz import stix2 +import stix2.v21 from .constants import IDENTITY_ID, THREAT_ACTOR_ID @@ -67,4 +68,27 @@ def test_parse_threat_actor(data): assert actor.name == "Evil Org" assert actor.threat_actor_types == ["crime-syndicate"] + +def test_seen_ordering_constraint(): + """ + Test first_seen/last_seen value co-constraint. + """ + with pytest.raises(ValueError): + stix2.v21.ThreatActor( + name="Bad Person", + threat_actor_types=["bad person", "evil person"], + first_seen="2010-04-21T09:31:11Z", + last_seen="2009-02-06T03:39:31Z", + ) + + # equal timestamps is okay. + stix2.v21.ThreatActor( + name="Bad Person", + threat_actor_types=["bad person", "evil person"], + first_seen="2010-04-21T09:31:11Z", + last_seen="2010-04-21T09:31:11Z", + ) + + + # TODO: Add other examples From 9404cf456006b8b6e2db7fbba3be90a3752d68ed Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 25 Jul 2019 16:58:48 -0400 Subject: [PATCH 39/62] Fix flake8 style error. --- stix2/test/v21/test_threat_actor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/test/v21/test_threat_actor.py b/stix2/test/v21/test_threat_actor.py index ecd7b00..6a782ef 100644 --- a/stix2/test/v21/test_threat_actor.py +++ b/stix2/test/v21/test_threat_actor.py @@ -90,5 +90,4 @@ def test_seen_ordering_constraint(): ) - # TODO: Add other examples From 27beec4060e25dd4daade9ed6f37016d39c32440 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 29 Jul 2019 16:35:38 -0400 Subject: [PATCH 40/62] Add a deprecation warning for the "objects" property of observed-data. Add a unit test to ensure we get the warning. --- stix2/test/v21/test_observed_data.py | 15 +++++++++++++++ stix2/v21/sdo.py | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 0e97ca6..b4e3f31 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1375,3 +1375,18 @@ def test_new_version_with_related_objects(): new_version = data.new_version(last_observed="2017-12-12T12:00:00Z") assert new_version.last_observed.year == 2017 assert new_version.objects['domain'].resolves_to_refs[0] == 'src_ip' + + +def test_objects_deprecation(): + with pytest.deprecated_call(): + stix2.v21.ObservedData( + first_observed="2016-03-12T12:00:00Z", + last_observed="2016-03-12T12:00:00Z", + number_observed=1, + objects={ + "0": { + "type": "file", + "name": "foo", + }, + }, + ) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 490de39..f04ea98 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -2,6 +2,7 @@ from collections import OrderedDict import itertools +import warnings from six.moves.urllib.parse import quote_plus @@ -573,6 +574,13 @@ class ObservedData(STIXDomainObject): self.__allow_custom = kwargs.get('allow_custom', False) self._properties['objects'].allow_custom = kwargs.get('allow_custom', False) + if "objects" in kwargs: + warnings.warn( + "The 'objects' property of observed-data is deprecated in " + "STIX 2.1.", + DeprecationWarning, + ) + super(ObservedData, self).__init__(*args, **kwargs) def _check_object_constraints(self): From 5e5a03c001d72f21abd03758b5c29fdd91504a1a Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 7 Aug 2019 10:16:18 -0400 Subject: [PATCH 41/62] Changed emitted deprecation warnings to a custom DeprecationWarning subclass. Changed the unit test to test for that specific warning category, instead of any DeprecationWarning. --- stix2/exceptions.py | 7 +++++++ stix2/test/v21/test_observed_data.py | 3 ++- stix2/v21/sdo.py | 4 ++-- tox.ini | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index c680774..d2ec3fc 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -226,3 +226,10 @@ class MarkingNotFoundError(STIXError): def __str__(self): msg = "Marking {0} was not found in {1}!" return msg.format(self.key, self.cls.__class__.__name__) + + +class STIXDeprecationWarning(DeprecationWarning): + """ + Represents usage of a deprecated component of a STIX specification. + """ + pass diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index b4e3f31..c2999f8 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1378,7 +1378,8 @@ def test_new_version_with_related_objects(): def test_objects_deprecation(): - with pytest.deprecated_call(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + stix2.v21.ObservedData( first_observed="2016-03-12T12:00:00Z", last_observed="2016-03-12T12:00:00Z", diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index f04ea98..662007c 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -8,7 +8,7 @@ from six.moves.urllib.parse import quote_plus from ..core import STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import PropertyPresenceError +from ..exceptions import PropertyPresenceError, STIXDeprecationWarning from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -578,7 +578,7 @@ class ObservedData(STIXDomainObject): warnings.warn( "The 'objects' property of observed-data is deprecated in " "STIX 2.1.", - DeprecationWarning, + STIXDeprecationWarning, ) super(ObservedData, self).__init__(*args, **kwargs) diff --git a/tox.ini b/tox.ini index 5deb4ef..e075386 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = taxii2-client medallion commands = - pytest --cov=stix2 stix2/test/ --cov-report term-missing + pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning passenv = CI TRAVIS TRAVIS_* From bb4a6b1bcb13188122925431b02d1b57c64e64d8 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 8 Aug 2019 15:22:21 -0400 Subject: [PATCH 42/62] Change how tox invokes pytest, to workaround a seeming tox bug which seems to be related to how it copes with code which emits warnings. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e075386..d4ab42c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps = taxii2-client medallion commands = - pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning + python -m pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning passenv = CI TRAVIS TRAVIS_* From ec55463398ae224fc756b131366681bd981fe091 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 19 Aug 2019 09:39:13 -0400 Subject: [PATCH 43/62] Update SCO stuff to WD 05 --- stix2/base.py | 47 +++ stix2/org/__init__.py | 0 stix2/org/webpki/__init__.py | 0 stix2/org/webpki/json/Canonicalize.py | 503 ++++++++++++++++++++++++++ stix2/org/webpki/json/LICENSE | 254 +++++++++++++ stix2/org/webpki/json/NumberToJson.py | 95 +++++ stix2/org/webpki/json/__init__.py | 0 stix2/test/v21/test_observables.py | 0 stix2/v21/observables.py | 52 ++- 9 files changed, 934 insertions(+), 17 deletions(-) create mode 100644 stix2/org/__init__.py create mode 100644 stix2/org/webpki/__init__.py create mode 100644 stix2/org/webpki/json/Canonicalize.py create mode 100644 stix2/org/webpki/json/LICENSE create mode 100644 stix2/org/webpki/json/NumberToJson.py create mode 100644 stix2/org/webpki/json/__init__.py create mode 100644 stix2/test/v21/test_observables.py diff --git a/stix2/base.py b/stix2/base.py index 9fe1617..ac9d162 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -3,10 +3,13 @@ import collections import copy import datetime as dt +import uuid import simplejson as json import six +from stix2.org.webpki.json.Canonicalize import canonicalize + from .exceptions import ( AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, InvalidObjRefError, @@ -309,6 +312,11 @@ class _Observable(_STIXBase): self.__allow_custom = kwargs.get('allow_custom', False) self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) + if 'id' not in kwargs: + possible_id = self._generate_id(kwargs) + if possible_id is not None: + kwargs['id'] = possible_id + super(_Observable, self).__init__(**kwargs) def _check_ref(self, ref, prop, prop_name): @@ -347,6 +355,45 @@ class _Observable(_STIXBase): for ref in kwargs[prop_name]: self._check_ref(ref, prop, prop_name) + def _generate_id(self, kwargs): + required_prefix = self._type + "--" + namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") + + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_object = {} + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = self._choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_object["hashes"] = possible_hash + for key in kwargs.keys(): + if key in properties_to_use and key != "hashes": + streamlined_object[key] = kwargs[key] + + if streamlined_object: + data = canonicalize(streamlined_object) + else: + return None + + return required_prefix + str(uuid.uuid5(namespace, str(data))) + else: + return None + + def _choose_one_hash(self, hash_dict): + if "MD5" in hash_dict: + return {"MD5": hash_dict["MD5"]} + elif "SHA-1" in hash_dict: + return {"SHA-1": hash_dict["SHA-1"]} + elif "SHA-256" in hash_dict: + return {"SHA-256": hash_dict["SHA-256"]} + elif "SHA-512" in hash_dict: + return {"SHA-512": hash_dict["SHA-512"]} + else: + # Cheesy way to pick the first item in the dictionary, since its not indexable + for (k, v) in hash_dict.items(): + break + return {k: v} + class _Extension(_STIXBase): diff --git a/stix2/org/__init__.py b/stix2/org/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/org/webpki/__init__.py b/stix2/org/webpki/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/org/webpki/json/Canonicalize.py b/stix2/org/webpki/json/Canonicalize.py new file mode 100644 index 0000000..cda810f --- /dev/null +++ b/stix2/org/webpki/json/Canonicalize.py @@ -0,0 +1,503 @@ +############################################################################## +# # +# Copyright 2006-2019 WebPKI.org (http://webpki.org). # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# https://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## + +################################################# +# JCS compatible JSON serializer for Python 3.x # +################################################# + +# This file has been modified to be compatible with Python 2.x as well + +import re + +from stix2.org.webpki.json.NumberToJson import convert2Es6Format + +try: + from _json import encode_basestring_ascii as c_encode_basestring_ascii +except ImportError: + c_encode_basestring_ascii = None +try: + from _json import encode_basestring as c_encode_basestring +except ImportError: + c_encode_basestring = None +try: + from _json import make_encoder as c_make_encoder +except ImportError: + c_make_encoder = None + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(b'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + +INFINITY = float('inf') + + +def py_encode_basestring(s): + """Return a JSON representation of a Python string + + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + + +encode_basestring = (c_encode_basestring or py_encode_basestring) + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u{0:04x}'.format(n) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '"' + ESCAPE_ASCII.sub(replace, s) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii +) + + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str | string | + +-------------------+---------------+ + | int, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + + def __init__( + self, skipkeys=False, ensure_ascii=False, + check_circular=True, allow_nan=True, sort_keys=True, + indent=None, separators=(',', ':'), default=None, + ): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming non-ASCII characters escaped. If + ensure_ascii is false, the output can contain non-ASCII characters. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be an (item_separator, key_separator) + tuple. The default is (', ', ': ') if *indent* is ``None`` and + (',', ': ') otherwise. To get the most compact JSON representation, + you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + elif indent is not None: + self.item_separator = ',' + if default is not None: + self.default = default + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + # Let the base class default method raise the TypeError + return JSONEncoder.default(self, o) + + """ + raise TypeError( + "Object of type '%s' is not JSON serializable" % + o.__class__.__name__, + ) + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from json.encoder import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, str): + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=False) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + return ''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + + def floatstr( + o, allow_nan=self.allow_nan, + _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY, + ): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o), + ) + + return text + + if ( + _one_shot and c_make_encoder is not None + and self.indent is None + ): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, + ) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, + ) + return _iterencode(o, 0) + + +def _make_iterencode( + markers, _default, _encoder, _indent, _floatstr, + _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, + # HACK: hand-optimized bytecode; turn globals into locals + ValueError=ValueError, + dict=dict, + float=float, + id=id, + int=int, + isinstance=isinstance, + list=list, + str=str, + tuple=tuple, + _intstr=int.__str__, +): + + if _indent is not None and not isinstance(_indent, str): + _indent = ' ' * _indent + + def _iterencode_list(lst, _current_indent_level): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + buf = '[' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + _indent * _current_indent_level + separator = _item_separator + newline_indent + buf += newline_indent + else: + newline_indent = None + separator = _item_separator + first = True + for value in lst: + if first: + first = False + else: + buf = separator + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __str__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + convert2Es6Format(value) + elif isinstance(value, float): + # see comment above for int + yield buf + convert2Es6Format(value) + else: + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + # yield from chunks + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + _indent * _current_indent_level + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(dct, _current_indent_level): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + _indent * _current_indent_level + item_separator = _item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = _item_separator + first = True + if _sort_keys: + items = sorted(dct.items(), key=lambda kv: kv[0].encode('utf-16_be')) + else: + items = dct.items() + for key, value in items: + if isinstance(key, str): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + # see comment for int/float in _make_iterencode + key = convert2Es6Format(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif isinstance(key, int): + # see comment for int/float in _make_iterencode + key = convert2Es6Format(key) + elif _skipkeys: + continue + else: + raise TypeError("key " + repr(key) + " is not a string") + if first: + first = False + else: + yield item_separator + yield _encoder(key) + yield _key_separator + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(value) + else: + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + # yield from chunks + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + _indent * _current_indent_level + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(o, _current_indent_level): + if isinstance(o, str): + yield _encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, int): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(o) + elif isinstance(o, float): + # see comment for int/float in _make_iterencode + yield convert2Es6Format(o) + elif isinstance(o, (list, tuple)): + # yield from _iterencode_list(o, _current_indent_level) + for thing in _iterencode_list(o, _current_indent_level): + yield thing + elif isinstance(o, dict): + # yield from _iterencode_dict(o, _current_indent_level) + for thing in _iterencode_dict(o, _current_indent_level): + yield thing + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + o = _default(o) + # yield from _iterencode(o, _current_indent_level) + for thing in _iterencode(o, _current_indent_level): + yield thing + if markers is not None: + del markers[markerid] + return _iterencode + + +def canonicalize(obj, utf8=True): + textVal = JSONEncoder(sort_keys=True).encode(obj) + if utf8: + return textVal.encode() + return textVal + + +def serialize(obj, utf8=True): + textVal = JSONEncoder(sort_keys=False).encode(obj) + if utf8: + return textVal.encode() + return textVal diff --git a/stix2/org/webpki/json/LICENSE b/stix2/org/webpki/json/LICENSE new file mode 100644 index 0000000..1afbedb --- /dev/null +++ b/stix2/org/webpki/json/LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/stix2/org/webpki/json/NumberToJson.py b/stix2/org/webpki/json/NumberToJson.py new file mode 100644 index 0000000..cea54d0 --- /dev/null +++ b/stix2/org/webpki/json/NumberToJson.py @@ -0,0 +1,95 @@ +############################################################################## +# # +# Copyright 2006-2019 WebPKI.org (http://webpki.org). # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# https://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# # +############################################################################## + + +################################################################## +# Convert a Python double/float into an ES6/V8 compatible string # +################################################################## +def convert2Es6Format(value): + # Convert double/float to str using the native Python formatter + fvalue = float(value) + + # Zero is a special case. The following line takes "-0" case as well + if fvalue == 0: + return '0' + + # The rest of the algorithm works on the textual representation only + pyDouble = str(fvalue) + + # The following line catches the "inf" and "nan" values returned by str(fvalue) + if pyDouble.find('n') >= 0: + raise ValueError("Invalid JSON number: " + pyDouble) + + # Save sign separately, it doesn't have any role in the algorithm + pySign = '' + if pyDouble.find('-') == 0: + pySign = '-' + pyDouble = pyDouble[1:] + + # Now we should only have valid non-zero values + pyExpStr = '' + pyExpVal = 0 + q = pyDouble.find('e') + if q > 0: + # Grab the exponent and remove it from the number + pyExpStr = pyDouble[q:] + if pyExpStr[2:3] == '0': + # Supress leading zero on exponents + pyExpStr = pyExpStr[:2] + pyExpStr[3:] + pyDouble = pyDouble[0:q] + pyExpVal = int(pyExpStr[1:]) + + # Split number in pyFirst + pyDot + pyLast + pyFirst = pyDouble + pyDot = '' + pyLast = '' + q = pyDouble.find('.') + if q > 0: + pyDot = '.' + pyFirst = pyDouble[:q] + pyLast = pyDouble[q + 1:] + + # Now the string is split into: pySign + pyFirst + pyDot + pyLast + pyExpStr + if pyLast == '0': + # Always remove trailing .0 + pyDot = '' + pyLast = '' + + if pyExpVal > 0 and pyExpVal < 21: + # Integers are shown as is with up to 21 digits + pyFirst += pyLast + pyLast = '' + pyDot = '' + pyExpStr = '' + q = pyExpVal - len(pyFirst) + while q >= 0: + q -= 1 + pyFirst += '0' + elif pyExpVal < 0 and pyExpVal > -7: + # Small numbers are shown as 0.etc with e-6 as lower limit + pyLast = pyFirst + pyLast + pyFirst = '0' + pyDot = '.' + pyExpStr = '' + q = pyExpVal + while q < -1: + q += 1 + pyLast = '0' + pyLast + + # The resulting sub-strings are concatenated + return pySign + pyFirst + pyDot + pyLast + pyExpStr diff --git a/stix2/org/webpki/json/__init__.py b/stix2/org/webpki/json/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/test/v21/test_observables.py b/stix2/test/v21/test_observables.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index be2b127..438fed2 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -28,7 +28,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), @@ -37,6 +37,7 @@ class Artifact(_Observable): ('decryption_key', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "payload_bin"] def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() @@ -53,12 +54,13 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["number"] class Directory(_Observable): @@ -70,7 +72,7 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself @@ -80,6 +82,7 @@ class Directory(_Observable): ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["path"] class DomainName(_Observable): @@ -91,11 +94,12 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class EmailAddress(_Observable): @@ -107,12 +111,13 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class EmailMIMEComponent(_STIXBase): @@ -142,7 +147,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), @@ -159,6 +164,7 @@ class EmailMessage(_Observable): ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["from_ref", "subject", "body"] def _check_object_constraints(self): super(EmailMessage, self)._check_object_constraints() @@ -329,7 +335,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), ('name', StringProperty()), @@ -345,6 +351,7 @@ class File(_Observable): ('content_ref', ObjectReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "name", "extensions"] def _check_object_constraints(self): super(File, self)._check_object_constraints() @@ -360,12 +367,13 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class IPv6Address(_Observable): @@ -377,12 +385,13 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class MACAddress(_Observable): @@ -394,10 +403,11 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class Mutex(_Observable): @@ -409,10 +419,11 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["name"] class HTTPRequestExt(_Extension): @@ -516,7 +527,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), @@ -536,6 +547,7 @@ class NetworkTraffic(_Observable): ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] def _check_object_constraints(self): super(NetworkTraffic, self)._check_object_constraints() @@ -651,6 +663,7 @@ class Process(_Observable): ('child_refs', ListProperty(ObjectReferenceProperty('process'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = [] def _check_object_constraints(self): # no need to check windows-service-ext, since it has a required property @@ -676,7 +689,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('cpe', StringProperty()), ('languages', ListProperty(StringProperty)), @@ -684,6 +697,7 @@ class Software(_Observable): ('version', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["name", "cpe", "vendor", "version"] class URL(_Observable): @@ -695,10 +709,11 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["value"] class UNIXAccountExt(_Extension): @@ -725,7 +740,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('user_id', StringProperty()), ('credential', StringProperty()), ('account_login', StringProperty()), @@ -742,6 +757,7 @@ class UserAccount(_Observable): ('account_last_login', TimestampProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["account_type", "user_id", "account_login"] class WindowsRegistryValueType(_STIXBase): @@ -783,7 +799,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself @@ -792,6 +808,7 @@ class WindowsRegistryKey(_Observable): ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["key", "values"] @property def values(self): @@ -835,7 +852,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), ('version', StringProperty()), @@ -851,6 +868,7 @@ class X509Certificate(_Observable): ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ]) + _id_contributing_properties = ["hashes", "serial_number"] def CustomObservable(type='x-custom-observable', properties=None): From 46359ead69b49d8e4fcd79499609c468089730a3 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Mon, 19 Aug 2019 13:35:17 -0400 Subject: [PATCH 44/62] Modify a few things --- stix2/base.py | 35 ++- .../json => canonicalization}/Canonicalize.py | 2 +- .../json => canonicalization}/NumberToJson.py | 0 stix2/{org => canonicalization}/__init__.py | 0 stix2/org/webpki/__init__.py | 0 stix2/org/webpki/json/LICENSE | 254 ------------------ stix2/org/webpki/json/__init__.py | 0 stix2/test/v21/test_observables.py | 0 stix2/v21/observables.py | 2 +- 9 files changed, 19 insertions(+), 274 deletions(-) rename stix2/{org/webpki/json => canonicalization}/Canonicalize.py (99%) rename stix2/{org/webpki/json => canonicalization}/NumberToJson.py (100%) rename stix2/{org => canonicalization}/__init__.py (100%) delete mode 100644 stix2/org/webpki/__init__.py delete mode 100644 stix2/org/webpki/json/LICENSE delete mode 100644 stix2/org/webpki/json/__init__.py delete mode 100644 stix2/test/v21/test_observables.py diff --git a/stix2/base.py b/stix2/base.py index ac9d162..580823d 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -8,7 +8,7 @@ import uuid import simplejson as json import six -from stix2.org.webpki.json.Canonicalize import canonicalize +from stix2.canonicalization.Canonicalize import canonicalize from .exceptions import ( AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, @@ -359,24 +359,23 @@ class _Observable(_STIXBase): required_prefix = self._type + "--" namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") - properties_to_use = self._id_contributing_properties - if properties_to_use: - streamlined_object = {} - if "hashes" in kwargs and "hashes" in properties_to_use: - possible_hash = self._choose_one_hash(kwargs["hashes"]) - if possible_hash: - streamlined_object["hashes"] = possible_hash - for key in kwargs.keys(): - if key in properties_to_use and key != "hashes": - streamlined_object[key] = kwargs[key] + try: + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_object = {} + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = self._choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_object["hashes"] = possible_hash + for key in kwargs.keys(): + if key in properties_to_use and key != "hashes": + streamlined_object[key] = kwargs[key] - if streamlined_object: - data = canonicalize(streamlined_object) - else: - return None - - return required_prefix + str(uuid.uuid5(namespace, str(data))) - else: + if streamlined_object: + data = canonicalize(streamlined_object, utf8=False) + return required_prefix + str(uuid.uuid5(namespace, str(data))) + return None + except AttributeError: return None def _choose_one_hash(self, hash_dict): diff --git a/stix2/org/webpki/json/Canonicalize.py b/stix2/canonicalization/Canonicalize.py similarity index 99% rename from stix2/org/webpki/json/Canonicalize.py rename to stix2/canonicalization/Canonicalize.py index cda810f..8bf2529 100644 --- a/stix2/org/webpki/json/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -24,7 +24,7 @@ import re -from stix2.org.webpki.json.NumberToJson import convert2Es6Format +from stix2.canonicalization.NumberToJson import convert2Es6Format try: from _json import encode_basestring_ascii as c_encode_basestring_ascii diff --git a/stix2/org/webpki/json/NumberToJson.py b/stix2/canonicalization/NumberToJson.py similarity index 100% rename from stix2/org/webpki/json/NumberToJson.py rename to stix2/canonicalization/NumberToJson.py diff --git a/stix2/org/__init__.py b/stix2/canonicalization/__init__.py similarity index 100% rename from stix2/org/__init__.py rename to stix2/canonicalization/__init__.py diff --git a/stix2/org/webpki/__init__.py b/stix2/org/webpki/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/stix2/org/webpki/json/LICENSE b/stix2/org/webpki/json/LICENSE deleted file mode 100644 index 1afbedb..0000000 --- a/stix2/org/webpki/json/LICENSE +++ /dev/null @@ -1,254 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations, which became -Zope Corporation. In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization -created specifically to own Python-related Intellectual Property. -Zope Corporation was a sponsoring member of the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All -Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/stix2/org/webpki/json/__init__.py b/stix2/org/webpki/json/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/stix2/test/v21/test_observables.py b/stix2/test/v21/test_observables.py deleted file mode 100644 index e69de29..0000000 diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 438fed2..3d9c1d5 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -648,7 +648,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ ('type', TypeProperty(_type)), - ('id', IDProperty(_type)), + ('id', IDProperty(_type, spec_version='2.1')), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself From bf1b8b567d89a7772c147769a7d939a9deb5d0d3 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 02:00:41 -0400 Subject: [PATCH 45/62] Updates to allow existing tests to pass --- stix2/base.py | 26 ++++++++++++++++++++++-- stix2/canonicalization/Canonicalize.py | 5 +++++ stix2/test/v21/test_datastore_filters.py | 5 +++-- stix2/test/v21/test_observed_data.py | 17 ++++++++++++++++ stix2/v21/observables.py | 2 +- 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 580823d..eb468ca 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -128,6 +128,11 @@ class _STIXBase(collections.Mapping): list_of_properties.remove('type') current_properties = self.properties_populated() list_of_properties_populated = set(list_of_properties).intersection(current_properties) + + if list_of_properties_populated == set(['id']) and isinstance(self, _Observable): + # Do not count the auto-generated id as a user-specified property + list_of_properties_populated = None + if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == set(['extensions'])): raise AtLeastOnePropertyError(self.__class__, list_of_properties) @@ -369,13 +374,24 @@ class _Observable(_STIXBase): streamlined_object["hashes"] = possible_hash for key in kwargs.keys(): if key in properties_to_use and key != "hashes": - streamlined_object[key] = kwargs[key] + if type(kwargs[key]) is dict: + for otherKey in kwargs[key]: + if isinstance(kwargs[key][otherKey], _STIXBase): + streamlined_object[key] = self._embed_obj_to_json(kwargs[key][otherKey]) + else: + streamlined_object[key] = kwargs[key] + else: + if isinstance(kwargs[key], _STIXBase): + streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) + else: + streamlined_object[key] = kwargs[key] if streamlined_object: - data = canonicalize(streamlined_object, utf8=False) + data = canonicalize(str(streamlined_object), utf8=False) return required_prefix + str(uuid.uuid5(namespace, str(data))) return None except AttributeError: + # We ideally end up here if handling a 2.0 SCO return None def _choose_one_hash(self, hash_dict): @@ -393,6 +409,12 @@ class _Observable(_STIXBase): break return {k: v} + def _embed_obj_to_json(self, obj): + tmp_obj = dict(copy.deepcopy(obj)) + for prop_name in obj._defaulted_optional_properties: + del tmp_obj[prop_name] + return tmp_obj + class _Extension(_STIXBase): diff --git a/stix2/canonicalization/Canonicalize.py b/stix2/canonicalization/Canonicalize.py index 8bf2529..daf0616 100644 --- a/stix2/canonicalization/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -360,6 +360,7 @@ def _make_iterencode( chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) + # Below line commented-out for python2 compatibility # yield from chunks for chunk in chunks: yield chunk @@ -441,6 +442,7 @@ def _make_iterencode( chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) + # Below line commented-out for python2 compatibility # yield from chunks for chunk in chunks: yield chunk @@ -467,10 +469,12 @@ def _make_iterencode( # see comment for int/float in _make_iterencode yield convert2Es6Format(o) elif isinstance(o, (list, tuple)): + # Below line commented-out for python2 compatibility # yield from _iterencode_list(o, _current_indent_level) for thing in _iterencode_list(o, _current_indent_level): yield thing elif isinstance(o, dict): + # Below line commented-out for python2 compatibility # yield from _iterencode_dict(o, _current_indent_level) for thing in _iterencode_dict(o, _current_indent_level): yield thing @@ -481,6 +485,7 @@ def _make_iterencode( raise ValueError("Circular reference detected") markers[markerid] = o o = _default(o) + # Below line commented-out for python2 compatibility # yield from _iterencode(o, _current_indent_level) for thing in _iterencode(o, _current_indent_level): yield thing diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index b96aa4d..92d035b 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,6 +86,7 @@ stix_objs = [ "objects": { "0": { "type": "file", + "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec", "name": "HAL 9000.exe", }, }, @@ -109,8 +110,8 @@ filters = [ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe"}), + Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}}), + Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 9916b79..64c9859 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -25,6 +25,7 @@ EXPECTED = """{ "objects": { "0": { "type": "file", + "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", "name": "foo.exe" } } @@ -64,10 +65,12 @@ EXPECTED_WITH_REF = """{ "objects": { "0": { "type": "file", + "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", "name": "foo.exe" }, "1": { "type": "directory", + "id": "directory--ed959127-2df3-5999-99b6-df7614398c1c", "path": "/usr/home", "contains_refs": [ "0" @@ -1391,3 +1394,17 @@ def test_objects_deprecation(): }, }, ) + + +# def test_deterministic_id_same_extra_prop_vals(): +# email_addr_1 = stix2.v21.EmailAddress( +# value="john@example.com", +# display_name="Johnny Doe" +# ) + +# email_addr_2 = stix2.v21.EmailAddress( +# value="john@example.com", +# display_name="Johnny Doe" +# ) + +# assert email_addr_1.id == email_addr_2.id diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 3d9c1d5..db7a3c7 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -41,7 +41,7 @@ class Artifact(_Observable): def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() - self._check_mutually_exclusive_properties(['payload_bin', 'url']) + self._check_mutually_exclusive_properties(['payload_bin', 'url'], at_least_one=False) self._check_properties_dependency(['hashes'], ['url']) From 5e9d6a6a14193db21a9ddf05995dbc4806b82a24 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 08:49:33 -0400 Subject: [PATCH 46/62] Fix small indentation error --- stix2/base.py | 2 +- stix2/test/v21/test_observed_data.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index eb468ca..03d808e 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -382,7 +382,7 @@ class _Observable(_STIXBase): streamlined_object[key] = kwargs[key] else: if isinstance(kwargs[key], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) + streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) else: streamlined_object[key] = kwargs[key] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 64c9859..90a494a 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1396,15 +1396,15 @@ def test_objects_deprecation(): ) -# def test_deterministic_id_same_extra_prop_vals(): -# email_addr_1 = stix2.v21.EmailAddress( -# value="john@example.com", -# display_name="Johnny Doe" -# ) +def test_deterministic_id_same_extra_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) -# email_addr_2 = stix2.v21.EmailAddress( -# value="john@example.com", -# display_name="Johnny Doe" -# ) + email_addr_2 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) -# assert email_addr_1.id == email_addr_2.id + assert email_addr_1.id == email_addr_2.id From 364daec40a220858ca3eb281033475d75f2f0ac6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 09:21:51 -0400 Subject: [PATCH 47/62] Add deterministic ID tests --- stix2/test/v21/test_observed_data.py | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 90a494a..370d1cc 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1,5 +1,6 @@ import datetime as dt import re +import uuid import pytest import pytz @@ -1408,3 +1409,51 @@ def test_deterministic_id_same_extra_prop_vals(): ) assert email_addr_1.id == email_addr_2.id + + +def test_deterministic_id_diff_extra_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + email_addr_2 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Janey Doe", + ) + + assert email_addr_1.id == email_addr_2.id + + +def test_deterministic_id_diff_contributing_prop_vals(): + email_addr_1 = stix2.v21.EmailAddress( + value="john@example.com", + display_name="Johnny Doe", + ) + + email_addr_2 = stix2.v21.EmailAddress( + value="jane@example.com", + display_name="Janey Doe", + ) + + assert email_addr_1.id != email_addr_2.id + + +def test_deterministic_id_no_contributing_props(): + email_msg_1 = stix2.v21.EmailMessage( + is_multipart=False, + ) + + email_msg_2 = stix2.v21.EmailMessage( + is_multipart=False, + ) + + assert email_msg_1.id != email_msg_2.id + + uuid_obj_1 = uuid.UUID(email_msg_1.id) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_msg_2.id) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 From 7c9fc3fd08468bdc64cc8d83e7e78a168222fc7d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 21 Aug 2019 09:33:42 -0400 Subject: [PATCH 48/62] Fix deterministic ID tests --- stix2/test/v21/test_observed_data.py | 32 ++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 370d1cc..dc27058 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -1410,6 +1410,14 @@ def test_deterministic_id_same_extra_prop_vals(): assert email_addr_1.id == email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_diff_extra_prop_vals(): email_addr_1 = stix2.v21.EmailAddress( @@ -1424,6 +1432,14 @@ def test_deterministic_id_diff_extra_prop_vals(): assert email_addr_1.id == email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_diff_contributing_prop_vals(): email_addr_1 = stix2.v21.EmailAddress( @@ -1438,6 +1454,14 @@ def test_deterministic_id_diff_contributing_prop_vals(): assert email_addr_1.id != email_addr_2.id + uuid_obj_1 = uuid.UUID(email_addr_1.id[-36:]) + assert uuid_obj_1.variant == uuid.RFC_4122 + assert uuid_obj_1.version == 5 + + uuid_obj_2 = uuid.UUID(email_addr_2.id[-36:]) + assert uuid_obj_2.variant == uuid.RFC_4122 + assert uuid_obj_2.version == 5 + def test_deterministic_id_no_contributing_props(): email_msg_1 = stix2.v21.EmailMessage( @@ -1450,10 +1474,10 @@ def test_deterministic_id_no_contributing_props(): assert email_msg_1.id != email_msg_2.id - uuid_obj_1 = uuid.UUID(email_msg_1.id) + uuid_obj_1 = uuid.UUID(email_msg_1.id[-36:]) assert uuid_obj_1.variant == uuid.RFC_4122 - assert uuid_obj_1.version == 5 + assert uuid_obj_1.version == 4 - uuid_obj_2 = uuid.UUID(email_msg_2.id) + uuid_obj_2 = uuid.UUID(email_msg_2.id[-36:]) assert uuid_obj_2.variant == uuid.RFC_4122 - assert uuid_obj_2.version == 5 + assert uuid_obj_2.version == 4 From c212c7c6788e55cdde726a20857286c0cd9e33fc Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 26 Aug 2019 17:10:54 -0400 Subject: [PATCH 49/62] Fix handling of custom extensions: make sure when allow_custom=True that you never get a half-cleaned property value. --- stix2/base.py | 15 +++--------- stix2/properties.py | 5 +++- stix2/test/v20/test_custom.py | 44 +++++++++++++++++++++++++++++++++++ stix2/test/v21/test_custom.py | 44 +++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 13 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 9fe1617..3874721 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -8,10 +8,9 @@ import simplejson as json import six from .exceptions import ( - AtLeastOnePropertyError, CustomContentError, DependentPropertiesError, - ExtraPropertiesError, ImmutableError, InvalidObjRefError, - InvalidValueError, MissingPropertiesError, - MutuallyExclusivePropertiesError, + AtLeastOnePropertyError, DependentPropertiesError, ExtraPropertiesError, + ImmutableError, InvalidObjRefError, InvalidValueError, + MissingPropertiesError, MutuallyExclusivePropertiesError, ) from .markings.utils import validate from .utils import NOW, find_property_index, format_datetime, get_timestamp @@ -93,14 +92,6 @@ class _STIXBase(collections.Mapping): # No point in wrapping InvalidValueError in another # InvalidValueError... so let those propagate. raise - except CustomContentError as exc: - if not self.__allow_custom: - six.raise_from( - InvalidValueError( - self.__class__, prop_name, reason=str(exc), - ), - exc, - ) except Exception as exc: six.raise_from( InvalidValueError( diff --git a/stix2/properties.py b/stix2/properties.py index b9a5aff..1095156 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -568,7 +568,10 @@ class ExtensionsProperty(DictionaryProperty): else: raise ValueError("Cannot determine extension type.") else: - raise CustomContentError("Can't parse unknown extension type: {}".format(key)) + if self.allow_custom: + dictified[key] = subvalue + else: + raise CustomContentError("Can't parse unknown extension type: {}".format(key)) return dictified diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 3cfc23f..cb04c17 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -1,6 +1,7 @@ import pytest import stix2 +import stix2.v20 from ...exceptions import InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID @@ -885,6 +886,49 @@ def test_parse_observable_with_custom_extension(): assert parsed.extensions['x-new-ext'].property2 == 12 +def test_custom_and_spec_extension_mix(): + """ + Try to make sure that when allow_custom=True, encountering a custom + extension doesn't result in a partially-cleaned extensions property. + """ + + file_obs = stix2.v20.File( + name="my_file.dat", + extensions={ + "x-custom1": { + "a": 1, + "b": 2, + }, + "ntfs-ext": { + "sid": "S-1-whatever", + }, + "x-custom2": { + "z": 99.9, + "y": False, + }, + "raster-image-ext": { + "image_height": 1024, + "image_width": 768, + "bits_per_pixel": 32, + }, + }, + allow_custom=True, + ) + + assert file_obs.extensions["x-custom1"] == {"a": 1, "b": 2} + assert file_obs.extensions["x-custom2"] == {"y": False, "z": 99.9} + assert file_obs.extensions["ntfs-ext"].sid == "S-1-whatever" + assert file_obs.extensions["raster-image-ext"].image_height == 1024 + + # Both of these should have been converted to objects, not left as dicts. + assert isinstance( + file_obs.extensions["raster-image-ext"], stix2.v20.RasterImageExt, + ) + assert isinstance( + file_obs.extensions["ntfs-ext"], stix2.v20.NTFSExt, + ) + + @pytest.mark.parametrize( "data", [ # URL is not in EXT_MAP diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index a5c9244..9ee982f 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -2,6 +2,7 @@ import pytest import stix2 import stix2.base +import stix2.v21 from ...exceptions import InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID @@ -889,6 +890,49 @@ def test_parse_observable_with_custom_extension(): assert parsed.extensions['x-new-ext'].property2 == 12 +def test_custom_and_spec_extension_mix(): + """ + Try to make sure that when allow_custom=True, encountering a custom + extension doesn't result in a partially-cleaned extensions property. + """ + + file_obs = stix2.v21.File( + name="my_file.dat", + extensions={ + "x-custom1": { + "a": 1, + "b": 2, + }, + "ntfs-ext": { + "sid": "S-1-whatever", + }, + "x-custom2": { + "z": 99.9, + "y": False, + }, + "raster-image-ext": { + "image_height": 1024, + "image_width": 768, + "bits_per_pixel": 32, + }, + }, + allow_custom=True, + ) + + assert file_obs.extensions["x-custom1"] == {"a": 1, "b": 2} + assert file_obs.extensions["x-custom2"] == {"y": False, "z": 99.9} + assert file_obs.extensions["ntfs-ext"].sid == "S-1-whatever" + assert file_obs.extensions["raster-image-ext"].image_height == 1024 + + # Both of these should have been converted to objects, not left as dicts. + assert isinstance( + file_obs.extensions["raster-image-ext"], stix2.v21.RasterImageExt, + ) + assert isinstance( + file_obs.extensions["ntfs-ext"], stix2.v21.NTFSExt, + ) + + @pytest.mark.parametrize( "data", [ # URL is not in EXT_MAP From 94bb76f669c2e9e8558126e7c7def1500100f5c7 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 26 Aug 2019 17:49:55 -0400 Subject: [PATCH 50/62] Fix docstring on the unit tests I added. I'd said "partially cleaned" property, but actually, the cleaning algorithm works on a dict copy, so aborting cleaning partway through doesn't actually affect the object in that way. It would actually cause the extensions property to be completely uncleaned, rather than partially cleaned. --- stix2/test/v20/test_custom.py | 2 +- stix2/test/v21/test_custom.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index cb04c17..6d127f2 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -889,7 +889,7 @@ def test_parse_observable_with_custom_extension(): def test_custom_and_spec_extension_mix(): """ Try to make sure that when allow_custom=True, encountering a custom - extension doesn't result in a partially-cleaned extensions property. + extension doesn't result in a completely uncleaned extensions property. """ file_obs = stix2.v20.File( diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 9ee982f..47d484a 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -893,7 +893,7 @@ def test_parse_observable_with_custom_extension(): def test_custom_and_spec_extension_mix(): """ Try to make sure that when allow_custom=True, encountering a custom - extension doesn't result in a partially-cleaned extensions property. + extension doesn't result in a completely uncleaned extensions property. """ file_obs = stix2.v21.File( From 49077352d7db2f8f132c256c63fe1f6be6a53260 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Tue, 27 Aug 2019 17:36:45 -0400 Subject: [PATCH 51/62] Updates and corrections for SCO WD 05 updates. Temp backup; testing and more fixes coming soon --- stix2/base.py | 111 ++++++++++--------- stix2/properties.py | 13 ++- stix2/test/v20/test_properties.py | 4 +- stix2/test/v21/test_properties.py | 4 +- stix2/v20/common.py | 6 +- stix2/v20/sdo.py | 54 ++++----- stix2/v20/sro.py | 18 +-- stix2/v21/common.py | 12 +- stix2/v21/observables.py | 178 ++++++++++++++++++++++++------ stix2/v21/sdo.py | 88 +++++++-------- stix2/v21/sro.py | 18 +-- 11 files changed, 318 insertions(+), 188 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 03d808e..7eda12c 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -317,10 +317,17 @@ class _Observable(_STIXBase): self.__allow_custom = kwargs.get('allow_custom', False) self._properties['extensions'].allow_custom = kwargs.get('allow_custom', False) - if 'id' not in kwargs: - possible_id = self._generate_id(kwargs) - if possible_id is not None: - kwargs['id'] = possible_id + try: + # Since `spec_version` is optional, this is how we check for a 2.1 SCO + self._id_contributing_properties + + if 'id' not in kwargs: + possible_id = self._generate_id(kwargs) + if possible_id is not None: + kwargs['id'] = possible_id + except AttributeError: + # End up here if handling a 2.0 SCO, and don't need to do anything further + pass super(_Observable, self).__init__(**kwargs) @@ -364,56 +371,28 @@ class _Observable(_STIXBase): required_prefix = self._type + "--" namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") - try: - properties_to_use = self._id_contributing_properties - if properties_to_use: - streamlined_object = {} - if "hashes" in kwargs and "hashes" in properties_to_use: - possible_hash = self._choose_one_hash(kwargs["hashes"]) - if possible_hash: - streamlined_object["hashes"] = possible_hash - for key in kwargs.keys(): - if key in properties_to_use and key != "hashes": - if type(kwargs[key]) is dict: - for otherKey in kwargs[key]: - if isinstance(kwargs[key][otherKey], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key][otherKey]) - else: - streamlined_object[key] = kwargs[key] - else: - if isinstance(kwargs[key], _STIXBase): - streamlined_object[key] = self._embed_obj_to_json(kwargs[key]) - else: - streamlined_object[key] = kwargs[key] + properties_to_use = self._id_contributing_properties + if properties_to_use: + streamlined_obj_vals = [] + if "hashes" in kwargs and "hashes" in properties_to_use: + possible_hash = _choose_one_hash(kwargs["hashes"]) + if possible_hash: + streamlined_obj_vals.append(possible_hash) + for key in properties_to_use: + if key != "hashes" and key in kwargs: + if isinstance(kwargs[key], dict) or isinstance(kwargs[key], _STIXBase): + temp_deep_copy = copy.deepcopy(dict(kwargs[key])) + _recursive_stix_to_dict(temp_deep_copy) + streamlined_obj_vals.append(temp_deep_copy) + else: + streamlined_obj_vals.append(kwargs[key]) - if streamlined_object: - data = canonicalize(str(streamlined_object), utf8=False) - return required_prefix + str(uuid.uuid5(namespace, str(data))) - return None - except AttributeError: - # We ideally end up here if handling a 2.0 SCO - return None + if streamlined_obj_vals: + data = canonicalize(streamlined_obj_vals, utf8=False) + return required_prefix + six.text_type(uuid.uuid5(namespace, data)) - def _choose_one_hash(self, hash_dict): - if "MD5" in hash_dict: - return {"MD5": hash_dict["MD5"]} - elif "SHA-1" in hash_dict: - return {"SHA-1": hash_dict["SHA-1"]} - elif "SHA-256" in hash_dict: - return {"SHA-256": hash_dict["SHA-256"]} - elif "SHA-512" in hash_dict: - return {"SHA-512": hash_dict["SHA-512"]} - else: - # Cheesy way to pick the first item in the dictionary, since its not indexable - for (k, v) in hash_dict.items(): - break - return {k: v} - - def _embed_obj_to_json(self, obj): - tmp_obj = dict(copy.deepcopy(obj)) - for prop_name in obj._defaulted_optional_properties: - del tmp_obj[prop_name] - return tmp_obj + # We return None if there are no values specified for any of the id-contributing-properties + return None class _Extension(_STIXBase): @@ -423,6 +402,34 @@ class _Extension(_STIXBase): self._check_at_least_one_property() +def _choose_one_hash(hash_dict): + if "MD5" in hash_dict: + return {"MD5": hash_dict["MD5"]} + elif "SHA-1" in hash_dict: + return {"SHA-1": hash_dict["SHA-1"]} + elif "SHA-256" in hash_dict: + return {"SHA-256": hash_dict["SHA-256"]} + elif "SHA-512" in hash_dict: + return {"SHA-512": hash_dict["SHA-512"]} + else: + k = next(iter(hash_dict), None) + if k is not None: + return {k: hash_dict[k]} + + def _cls_init(cls, obj, kwargs): if getattr(cls, '__init__', object.__init__) is not object.__init__: cls.__init__(obj, **kwargs) + + +def _recursive_stix_to_dict(input_dict): + for key in input_dict: + if isinstance(input_dict[key], dict): + _recursive_stix_to_dict(input_dict[key]) + elif isinstance(input_dict[key], _STIXBase): + input_dict[key] = dict(input_dict[key]) + + # There may stil be nested _STIXBase objects + _recursive_stix_to_dict(input_dict[key]) + else: + return diff --git a/stix2/properties.py b/stix2/properties.py index b9a5aff..ed73f3a 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -417,12 +417,16 @@ class HexProperty(Property): class ReferenceProperty(Property): - def __init__(self, type=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): + def __init__(self, valid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ - self.required_prefix = type + "--" if type else None self.spec_version = spec_version + + if valid_types and type(valid_types) is not list: + valid_types = [valid_types] + self.valid_types = valid_types + super(ReferenceProperty, self).__init__(**kwargs) def clean(self, value): @@ -430,7 +434,10 @@ class ReferenceProperty(Property): value = value.id value = str(value) - _validate_id(value, self.spec_version, self.required_prefix) + if self.valid_types and value[:value.index('--')] in self.valid_types: + required_prefix = value[:value.index('--') + 2] + + _validate_id(value, self.spec_version, required_prefix) return value diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 9952eac..548e7fe 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -276,7 +276,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(spec_version="2.0") + ref_prop = ReferenceProperty(valid_types=None, spec_version="2.0") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): @@ -288,7 +288,7 @@ def test_reference_property(): def test_reference_property_specific_type(): - ref_prop = ReferenceProperty("my-type", spec_version="2.0") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index fde13d3..41ca3e9 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -271,7 +271,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(spec_version="2.1") + ref_prop = ReferenceProperty(valid_types=None, spec_version="2.1") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): @@ -283,7 +283,7 @@ def test_reference_property(): def test_reference_property_specific_type(): - ref_prop = ReferenceProperty("my-type", spec_version="2.1") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") diff --git a/stix2/v20/common.py b/stix2/v20/common.py index c2dedfb..377d992 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -49,7 +49,7 @@ class GranularMarking(_STIXBase): """ _properties = OrderedDict([ - ('marking_ref', ReferenceProperty(required=True, spec_version='2.0', type='marking-definition')), + ('marking_ref', ReferenceProperty(valid_types='marking-definition', spec_version='2.0', required=True)), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -105,10 +105,10 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 3fd3a84..9288003 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -23,7 +23,7 @@ class AttackPattern(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -32,7 +32,7 @@ class AttackPattern(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -46,7 +46,7 @@ class Campaign(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -58,7 +58,7 @@ class Campaign(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -72,7 +72,7 @@ class CourseOfAction(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -80,7 +80,7 @@ class CourseOfAction(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -94,7 +94,7 @@ class Identity(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -105,7 +105,7 @@ class Identity(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -119,7 +119,7 @@ class Indicator(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -131,7 +131,7 @@ class Indicator(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -145,7 +145,7 @@ class IntrusionSet(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -160,7 +160,7 @@ class IntrusionSet(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -174,7 +174,7 @@ class Malware(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -183,7 +183,7 @@ class Malware(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -197,7 +197,7 @@ class ObservedData(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), @@ -207,7 +207,7 @@ class ObservedData(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -227,17 +227,17 @@ class Report(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -251,7 +251,7 @@ class ThreatActor(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -267,7 +267,7 @@ class ThreatActor(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -281,7 +281,7 @@ class Tool(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -291,7 +291,7 @@ class Tool(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -305,7 +305,7 @@ class Vulnerability(STIXDomainObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -313,7 +313,7 @@ class Vulnerability(STIXDomainObject): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -352,7 +352,7 @@ def CustomObject(type='x-custom-type', properties=None): [ ('type', TypeProperty(type)), ('id', IDProperty(type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -361,7 +361,7 @@ def CustomObject(type='x-custom-type', properties=None): ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index f450003..a21c871 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -20,17 +20,17 @@ class Relationship(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(spec_version='2.0', required=True)), - ('target_ref', ReferenceProperty(spec_version='2.0', required=True)), + ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -59,20 +59,20 @@ class Sighting(STIXRelationshipObject): _properties = OrderedDict([ ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.0')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.0')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(spec_version='2.0', required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.0'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.0'))), + ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.0'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), ]) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 70be5dc..e085985 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -54,7 +54,7 @@ class GranularMarking(_STIXBase): _properties = OrderedDict([ ('lang', StringProperty()), - ('marking_ref', ReferenceProperty(type='marking-definition', spec_version='2.1')), + ('marking_ref', ReferenceProperty(valid_types='marking-definition', spec_version='2.1')), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -74,10 +74,10 @@ class LanguageContent(_STIXBase): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('object_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('object_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx @@ -86,7 +86,7 @@ class LanguageContent(_STIXBase): ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -145,10 +145,10 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type)), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW)), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index db7a3c7..f19d8ad 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -7,16 +7,21 @@ Observable and do not have a ``_type`` attribute. from collections import OrderedDict import itertools +import warnings from ..base import _Extension, _Observable, _STIXBase from ..custom import _custom_extension_builder, _custom_observable_builder -from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError +from ..exceptions import ( + AtLeastOnePropertyError, DependentPropertiesError, STIXDeprecationWarning, +) from ..properties import ( BinaryProperty, BooleanProperty, CallableValues, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, - ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, + ObjectReferenceProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty, ) +from .common import GranularMarking class Artifact(_Observable): @@ -36,6 +41,10 @@ class Artifact(_Observable): ('encryption_algorithm', StringProperty()), ('decryption_key', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "payload_bin"] @@ -59,6 +68,10 @@ class AutonomousSystem(_Observable): ('name', StringProperty()), ('rir', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["number"] @@ -79,8 +92,12 @@ class Directory(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), + ('contains_refs', ListProperty(ReferenceProperty(valid_types=['file', 'directory'], spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["path"] @@ -96,11 +113,23 @@ class DomainName(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'], spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of domain-name is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + class EmailAddress(_Observable): # TODO: Add link @@ -114,8 +143,12 @@ class EmailAddress(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), - ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), + ('belongs_to_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -151,18 +184,23 @@ class EmailMessage(_Observable): ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), - ('from_ref', ObjectReferenceProperty(valid_types='email-addr')), - ('sender_ref', ObjectReferenceProperty(valid_types='email-addr')), - ('to_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), - ('cc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), - ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('from_ref', ReferenceProperty(valid_types='email-addr')), + ('sender_ref', ReferenceProperty(valid_types='email-addr')), + ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('message_id', StringProperty()), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), ('additional_header_fields', DictionaryProperty(spec_version='2.1')), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), - ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), + ('raw_email_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["from_ref", "subject", "body"] @@ -346,10 +384,14 @@ class File(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ObjectReferenceProperty)), - ('content_ref', ObjectReferenceProperty(valid_types='artifact')), + ('parent_directory_ref', ReferenceProperty(valid_types='directory')), + ('contains_refs', ListProperty(ReferenceProperty())), + ('content_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "name", "extensions"] @@ -369,12 +411,31 @@ class IPv4Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of ipv4-addr is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + + if self.get('belongs_to_refs'): + warnings.warn( + "The 'belongs_to_refs' property of ipv4-addr is deprecated in " + "STIX 2.1. Use the 'belongs-to' relationship type instead", + STIXDeprecationWarning, + ) + class IPv6Address(_Observable): # TODO: Add link @@ -387,12 +448,31 @@ class IPv6Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] + def _check_object_constraints(self): + if self.get('resolves_to_refs'): + warnings.warn( + "The 'resolves_to_refs' property of ipv6-addr is deprecated in " + "STIX 2.1. Use the 'resolves-to' relationship type instead", + STIXDeprecationWarning, + ) + + if self.get('belongs_to_refs'): + warnings.warn( + "The 'belongs_to_refs' property of ipv6-addr is deprecated in " + "STIX 2.1. Use the 'belongs-to' relationship type instead", + STIXDeprecationWarning, + ) + class MACAddress(_Observable): # TODO: Add link @@ -406,6 +486,10 @@ class MACAddress(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -422,6 +506,10 @@ class Mutex(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["name"] @@ -531,8 +619,8 @@ class NetworkTraffic(_Observable): ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), - ('src_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), - ('dst_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('src_ref', ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'], spec_version='2.1')), + ('dst_ref', ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'], spec_version='2.1')), ('src_port', IntegerProperty(min=0, max=65535)), ('dst_port', IntegerProperty(min=0, max=65535)), ('protocols', ListProperty(StringProperty, required=True)), @@ -541,11 +629,15 @@ class NetworkTraffic(_Observable): ('src_packets', IntegerProperty(min=0)), ('dst_packets', IntegerProperty(min=0)), ('ipfix', DictionaryProperty(spec_version='2.1')), - ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), - ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), - ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), - ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), + ('src_payload_ref', ReferenceProperty(valid_types='artifact')), + ('dst_payload_ref', ReferenceProperty(valid_types='artifact')), + ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), + ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"] @@ -652,16 +744,20 @@ class Process(_Observable): ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), # this is not the created timestamps of the object itself - ('created', TimestampProperty()), + ('created_time', TimestampProperty()), ('cwd', StringProperty()), ('command_line', StringProperty()), ('environment_variables', DictionaryProperty(spec_version='2.1')), - ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), - ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), - ('image_ref', ObjectReferenceProperty(valid_types='file')), - ('parent_ref', ObjectReferenceProperty(valid_types='process')), - ('child_refs', ListProperty(ObjectReferenceProperty('process'))), + ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), + ('creator_user_ref', ReferenceProperty(valid_types='user-account')), + ('image_ref', ReferenceProperty(valid_types='file')), + ('parent_ref', ReferenceProperty(valid_types='process')), + ('child_refs', ListProperty(ReferenceProperty(valid_types='process'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = [] @@ -696,6 +792,10 @@ class Software(_Observable): ('vendor', StringProperty()), ('version', StringProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["name", "cpe", "vendor", "version"] @@ -712,6 +812,10 @@ class URL(_Observable): ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["value"] @@ -756,6 +860,10 @@ class UserAccount(_Observable): ('account_first_login', TimestampProperty()), ('account_last_login', TimestampProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["account_type", "user_id", "account_login"] @@ -804,9 +912,13 @@ class WindowsRegistryKey(_Observable): ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself ('modified_time', TimestampProperty()), - ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('creator_user_ref', ReferenceProperty(valid_types='user-account')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["key", "values"] @@ -867,6 +979,10 @@ class X509Certificate(_Observable): ('subject_public_key_exponent', IntegerProperty()), ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), + ('spec_version', StringProperty(fixed='2.1')), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), + ('granular_markings', ListProperty(GranularMarking)), + ('defanged', BooleanProperty(default=lambda: False)), ]) _id_contributing_properties = ["hashes", "serial_number"] diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 662007c..e9714e7 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -30,7 +30,7 @@ class AttackPattern(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -42,7 +42,7 @@ class AttackPattern(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -58,7 +58,7 @@ class Campaign(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -72,7 +72,7 @@ class Campaign(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -98,7 +98,7 @@ class CourseOfAction(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -112,7 +112,7 @@ class CourseOfAction(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -138,13 +138,13 @@ class Grouping(STIXDomainObject): ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('name', StringProperty()), ('description', StringProperty()), @@ -164,7 +164,7 @@ class Identity(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -178,7 +178,7 @@ class Identity(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -194,7 +194,7 @@ class Indicator(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -211,7 +211,7 @@ class Indicator(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -237,7 +237,7 @@ class Infrastructure(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('revoked', BooleanProperty(default=lambda: False)), @@ -245,7 +245,7 @@ class Infrastructure(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('name', StringProperty(required=True)), ('description', StringProperty()), @@ -278,7 +278,7 @@ class IntrusionSet(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -295,7 +295,7 @@ class IntrusionSet(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -321,7 +321,7 @@ class Location(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -340,7 +340,7 @@ class Location(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -429,7 +429,7 @@ class Malware(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty()), @@ -444,13 +444,13 @@ class Malware(STIXDomainObject): ('architecture_execution_envs', ListProperty(StringProperty)), ('implementation_languages', ListProperty(StringProperty)), ('capabilities', ListProperty(StringProperty)), - ('sample_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), + ('sample_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -484,19 +484,19 @@ class MalwareAnalysis(STIXDomainObject): ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ('product', StringProperty(required=True)), ('version', StringProperty()), - ('host_vm_ref', ReferenceProperty(type='software', spec_version='2.1')), - ('operating_system_ref', ReferenceProperty(type='software', spec_version='2.1')), - ('installed_software_refs', ListProperty(ReferenceProperty(type='software', spec_version='2.1'))), + ('host_vm_ref', ReferenceProperty(valid_types='software', spec_version='2.1')), + ('operating_system_ref', ReferenceProperty(valid_types='software', spec_version='2.1')), + ('installed_software_refs', ListProperty(ReferenceProperty(valid_types='software', spec_version='2.1'))), ('configuration_version', StringProperty()), ('modules', ListProperty(StringProperty)), ('analysis_engine_version', StringProperty()), @@ -505,7 +505,7 @@ class MalwareAnalysis(STIXDomainObject): ('analysis_started', TimestampProperty()), ('analysis_ended', TimestampProperty()), ('av_result', StringProperty()), - ('analysis_sco_refs', ListProperty(ReferenceProperty(spec_version='2.1'))), + ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), ]) def _check_object_constraints(self): @@ -525,7 +525,7 @@ class Note(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('abstract', StringProperty()), @@ -537,7 +537,7 @@ class Note(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -553,20 +553,20 @@ class ObservedData(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), ('objects', ObservableProperty(spec_version='2.1')), - ('object_refs', ListProperty(ReferenceProperty(spec_version="2.1"))), + ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version="2.1"))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -609,7 +609,7 @@ class Opinion(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('explanation', StringProperty()), @@ -631,7 +631,7 @@ class Opinion(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -647,7 +647,7 @@ class Report(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -660,7 +660,7 @@ class Report(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -676,7 +676,7 @@ class ThreatActor(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -697,7 +697,7 @@ class ThreatActor(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -723,7 +723,7 @@ class Tool(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -737,7 +737,7 @@ class Tool(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -753,7 +753,7 @@ class Vulnerability(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), @@ -763,7 +763,7 @@ class Vulnerability(STIXDomainObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -803,7 +803,7 @@ def CustomObject(type='x-custom-type', properties=None): ('type', TypeProperty(type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ], @@ -814,7 +814,7 @@ def CustomObject(type='x-custom-type', properties=None): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 149094e..37f0024 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -22,13 +22,13 @@ class Relationship(STIXRelationshipObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -36,7 +36,7 @@ class Relationship(STIXRelationshipObject): ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) @@ -77,23 +77,23 @@ class Sighting(STIXRelationshipObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created_by_ref', ReferenceProperty(type='identity', spec_version='2.1')), + ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('description', StringProperty()), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(spec_version='2.1', required=True)), - ('observed_data_refs', ListProperty(ReferenceProperty(type='observed-data', spec_version='2.1'))), - ('where_sighted_refs', ListProperty(ReferenceProperty(type='identity', spec_version='2.1'))), + ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), + ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), ('lang', StringProperty()), ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type='marking-definition', spec_version='2.1'))), + ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), ]) From f69b13a006e63c0cc36fb82a3ecb8af9c1e79f52 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 29 Aug 2019 17:15:51 -0400 Subject: [PATCH 52/62] Some more updates, primarily to ReferenceProperty (and related code) --- stix2/properties.py | 33 +++++++++++++++++++++++++++---- stix2/test/v20/test_properties.py | 2 +- stix2/test/v21/test_properties.py | 2 +- stix2/v20/sdo.py | 2 +- stix2/v20/sro.py | 8 +++++--- stix2/v21/common.py | 2 +- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 2 +- stix2/v21/sro.py | 8 +++++--- 9 files changed, 45 insertions(+), 16 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 4eb5014..a217950 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -15,7 +15,10 @@ import stix2 from .base import _Observable, _STIXBase from .core import STIX2_OBJ_MAPS, parse, parse_observable -from .exceptions import CustomContentError, DictionaryKeyError +from .exceptions import ( + CustomContentError, DictionaryKeyError, MissingPropertiesError, + MutuallyExclusivePropertiesError, +) from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime ERROR_INVALID_ID = ( @@ -417,15 +420,27 @@ class HexProperty(Property): class ReferenceProperty(Property): - def __init__(self, valid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): + def __init__(self, valid_types=None, invalid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ self.spec_version = spec_version + # These checks need to be done prior to the STIX object finishing construction + # and thus we can't use base.py's _check_mutually_exclusive_properties() + # in the typical location of _check_object_constraints() in sdo.py + if valid_types and invalid_types: + raise MutuallyExclusivePropertiesError(self.__class__, ['invalid_types', 'valid_types']) + elif valid_types is None and invalid_types is None: + raise MissingPropertiesError(self.__class__, ['invalid_types', 'valid_types']) + if valid_types and type(valid_types) is not list: valid_types = [valid_types] + elif invalid_types and type(invalid_types) is not list: + invalid_types = [invalid_types] + self.valid_types = valid_types + self.invalid_types = invalid_types super(ReferenceProperty, self).__init__(**kwargs) @@ -434,8 +449,18 @@ class ReferenceProperty(Property): value = value.id value = str(value) - if self.valid_types and value[:value.index('--')] in self.valid_types: - required_prefix = value[:value.index('--') + 2] + possible_prefix = value[:value.index('--') + 2] + + if self.valid_types: + if possible_prefix in self.valid_types: + required_prefix = possible_prefix + else: + raise ValueError("The type-specifying prefix for this identifier is invalid") + elif self.invalid_types: + if possible_prefix not in self.invalid_types: + required_prefix = possible_prefix + else: + raise ValueError("The type-specifying prefix for this identifier is invalid") _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 548e7fe..f71d829 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -276,7 +276,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(valid_types=None, spec_version="2.0") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 41ca3e9..1fb3cc4 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -271,7 +271,7 @@ def test_boolean_property_invalid(value): def test_reference_property(): - ref_prop = ReferenceProperty(valid_types=None, spec_version="2.1") + ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") with pytest.raises(ValueError): diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 9288003..899ccd8 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -233,7 +233,7 @@ class Report(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index a21c871..3b1e582 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -16,6 +16,8 @@ class Relationship(STIXRelationshipObject): `the STIX 2.0 specification `__. """ + _invalid_source_target_types = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting'] + _type = 'relationship' _properties = OrderedDict([ ('type', TypeProperty(_type)), @@ -25,8 +27,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), - ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('source_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.0', required=True)), + ('target_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.0', required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), @@ -65,7 +67,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.0', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.0', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), diff --git a/stix2/v21/common.py b/stix2/v21/common.py index e085985..44d3675 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -77,7 +77,7 @@ class LanguageContent(_STIXBase): ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('object_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('object_ref', ReferenceProperty(invalid_types=[""], spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index f19d8ad..fbce8fd 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -385,7 +385,7 @@ class File(_Observable): ('mtime', TimestampProperty()), ('atime', TimestampProperty()), ('parent_directory_ref', ReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ReferenceProperty())), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""]))), ('content_ref', ReferenceProperty(valid_types='artifact')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index e9714e7..cb7eff3 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -444,7 +444,7 @@ class Malware(STIXDomainObject): ('architecture_execution_envs', ListProperty(StringProperty)), ('implementation_languages', ListProperty(StringProperty)), ('capabilities', ListProperty(StringProperty)), - ('sample_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), + ('sample_refs', ListProperty(ReferenceProperty(valid_types=['artifact', 'file'], spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 37f0024..31f9c4c 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -17,6 +17,8 @@ class Relationship(STIXRelationshipObject): `the STIX 2.1 specification `__. """ + _invalid_source_target_types = ['bundle', 'language-content', 'marking-definition', 'relationship', 'sighting'] + _type = 'relationship' _properties = OrderedDict([ ('type', TypeProperty(_type)), @@ -27,8 +29,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -84,7 +86,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types=None, spec_version='2.1', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.1', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), From 44ebd64a1626674c1f677eac3f62d20742f69470 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 30 Aug 2019 03:47:47 -0400 Subject: [PATCH 53/62] Some test fixes. More coming soon --- stix2/base.py | 3 ++- stix2/properties.py | 15 +++++++++++---- stix2/test/v21/test_utils.py | 6 ++++-- stix2/v20/sro.py | 2 +- stix2/v21/sdo.py | 26 ++++++++++++++++++-------- stix2/v21/sro.py | 6 +++--- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 229f78f..bae1f2c 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -327,7 +327,8 @@ class _Observable(_STIXBase): return # don't check if refs are valid if ref not in self._STIXBase__valid_refs: - raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) + if ref[:ref.index('--') + 2] not in self._STIXBase__valid_refs: + raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) try: allowed_types = prop.contained.valid_types diff --git a/stix2/properties.py b/stix2/properties.py index a217950..e8c8442 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -452,15 +452,22 @@ class ReferenceProperty(Property): possible_prefix = value[:value.index('--') + 2] if self.valid_types: - if possible_prefix in self.valid_types: + if self.valid_types == ["only_SDO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['objects'].keys() + elif self.valid_types == ["only_SCO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + elif self.valid_types == ["only_SCO_&_SRO"]: + self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + ['relationship', 'sighting'] + + if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is invalid") + raise ValueError("The type-specifying prefix for this identifier is not valid") elif self.invalid_types: - if possible_prefix not in self.invalid_types: + if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is invalid") + raise ValueError("An invalid type-specifying prefix was specified for this identifier") _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v21/test_utils.py b/stix2/test/v21/test_utils.py index dec3294..5cf88e4 100644 --- a/stix2/test/v21/test_utils.py +++ b/stix2/test/v21/test_utils.py @@ -135,14 +135,16 @@ def test_deduplicate(stix_objs1): "0": { "name": "foo.exe", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", }, "1": { "type": "ipv4-addr", "value": "198.51.100.3", + "id": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5", }, "2": { "type": "network-traffic", - "src_ref": "1", + "src_ref": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5", "protocols": [ "tcp", "http", @@ -161,7 +163,7 @@ def test_deduplicate(stix_objs1): }, }, }, - ), ('1', {"type": "ipv4-addr", "value": "198.51.100.3"}), 1, + ), ('1', {"type": "ipv4-addr", "value": "198.51.100.3", "id": "ipv4-addr--1f8f4d63-9f33-5353-a3e3-e1b84c83a7b5"}), 1, ), ( { diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index 3b1e582..3f561b8 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -67,7 +67,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.0', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="only_SDO", spec_version='2.0', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.0'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.0'))), ('summary', BooleanProperty(default=lambda: False)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index cb7eff3..3062659 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -6,9 +6,11 @@ import warnings from six.moves.urllib.parse import quote_plus -from ..core import STIXDomainObject +from ..core import STIX2_OBJ_MAPS, STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import PropertyPresenceError, STIXDeprecationWarning +from ..exceptions import ( + InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, +) from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -149,7 +151,7 @@ class Grouping(STIXDomainObject): ('name', StringProperty()), ('description', StringProperty()), ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ]) @@ -505,7 +507,7 @@ class MalwareAnalysis(STIXDomainObject): ('analysis_started', TimestampProperty()), ('analysis_ended', TimestampProperty()), ('av_result', StringProperty()), - ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version='2.1'))), + ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types="only_SCO", spec_version='2.1'))), ]) def _check_object_constraints(self): @@ -531,7 +533,7 @@ class Note(STIXDomainObject): ('abstract', StringProperty()), ('content', StringProperty(required=True)), ('authors', ListProperty(StringProperty)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -560,7 +562,7 @@ class ObservedData(STIXDomainObject): ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), ('objects', ObservableProperty(spec_version='2.1')), - ('object_refs', ListProperty(ReferenceProperty(valid_types=None, spec_version="2.1"))), + ('object_refs', ListProperty(ReferenceProperty(valid_types="only_SCO_&_SRO", spec_version="2.1"))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -597,6 +599,14 @@ class ObservedData(STIXDomainObject): ["objects", "object_refs"], ) + if self.get('object_refs'): + for identifier in self.get('object_refs'): + identifier_prefix = identifier[:identifier.index('--') + 2] + if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): + break + else: + raise InvalidValueError(self.__class__, 'object_refs', "At least one identifier must be of a SCO type if this property specified") + class Opinion(STIXDomainObject): # TODO: Add link @@ -625,7 +635,7 @@ class Opinion(STIXDomainObject): ], required=True, ), ), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -654,7 +664,7 @@ class Report(STIXDomainObject): ('description', StringProperty()), ('report_types', ListProperty(StringProperty, required=True)), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 31f9c4c..57c7719 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -29,8 +29,8 @@ class Relationship(STIXRelationshipObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), - ('source_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), - ('target_ref', ReferenceProperty(valid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('source_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.1', required=True)), + ('target_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.1', required=True)), ('start_time', TimestampProperty()), ('stop_time', TimestampProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -86,7 +86,7 @@ class Sighting(STIXRelationshipObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('count', IntegerProperty(min=0, max=999999999)), - ('sighting_of_ref', ReferenceProperty(valid_types="Left to user", spec_version='2.1', required=True)), + ('sighting_of_ref', ReferenceProperty(valid_types="only_SDO", spec_version='2.1', required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(valid_types='observed-data', spec_version='2.1'))), ('where_sighted_refs', ListProperty(ReferenceProperty(valid_types='identity', spec_version='2.1'))), ('summary', BooleanProperty()), From abf29803367b5727149c7f37c4661ceea86d45f7 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 4 Sep 2019 19:08:34 -0400 Subject: [PATCH 54/62] Fix tests and ReferenceProperty --- stix2/base.py | 32 +++-- stix2/properties.py | 4 +- stix2/test/v20/test_utils.py | 4 +- stix2/test/v21/test_core.py | 4 +- stix2/test/v21/test_datastore_filters.py | 6 +- stix2/test/v21/test_observed_data.py | 156 ++++++++++++----------- stix2/v20/sdo.py | 2 +- stix2/v21/observables.py | 46 +++---- stix2/v21/sdo.py | 10 +- 9 files changed, 138 insertions(+), 126 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index bae1f2c..5bc3d10 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -115,15 +115,15 @@ class _STIXBase(collections.Mapping): def _check_at_least_one_property(self, list_of_properties=None): if not list_of_properties: list_of_properties = sorted(list(self.__class__._properties.keys())) - if 'type' in list_of_properties: - list_of_properties.remove('type') + if isinstance(self, _Observable): + props_to_remove = ["type", "id", "defanged", "spec_version"] + else: + props_to_remove = ["type"] + + list_of_properties = [prop for prop in list_of_properties if prop not in props_to_remove] current_properties = self.properties_populated() list_of_properties_populated = set(list_of_properties).intersection(current_properties) - if list_of_properties_populated == set(['id']) and isinstance(self, _Observable): - # Do not count the auto-generated id as a user-specified property - list_of_properties_populated = None - if list_of_properties and (not list_of_properties_populated or list_of_properties_populated == set(['extensions'])): raise AtLeastOnePropertyError(self.__class__, list_of_properties) @@ -327,8 +327,7 @@ class _Observable(_STIXBase): return # don't check if refs are valid if ref not in self._STIXBase__valid_refs: - if ref[:ref.index('--') + 2] not in self._STIXBase__valid_refs: - raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) + raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref) try: allowed_types = prop.contained.valid_types @@ -352,12 +351,14 @@ class _Observable(_STIXBase): if prop_name not in kwargs: return - if prop_name.endswith('_ref'): - ref = kwargs[prop_name] - self._check_ref(ref, prop, prop_name) - elif prop_name.endswith('_refs'): - for ref in kwargs[prop_name]: + from .properties import ObjectReferenceProperty + if isinstance(prop, ObjectReferenceProperty): + if prop_name.endswith('_ref'): + ref = kwargs[prop_name] self._check_ref(ref, prop, prop_name) + elif prop_name.endswith('_refs'): + for ref in kwargs[prop_name]: + self._check_ref(ref, prop, prop_name) def _generate_id(self, kwargs): required_prefix = self._type + "--" @@ -376,6 +377,11 @@ class _Observable(_STIXBase): temp_deep_copy = copy.deepcopy(dict(kwargs[key])) _recursive_stix_to_dict(temp_deep_copy) streamlined_obj_vals.append(temp_deep_copy) + elif isinstance(kwargs[key], list) and isinstance(kwargs[key][0], _STIXBase): + for obj in kwargs[key]: + temp_deep_copy = copy.deepcopy(dict(obj)) + _recursive_stix_to_dict(temp_deep_copy) + streamlined_obj_vals.append(temp_deep_copy) else: streamlined_obj_vals.append(kwargs[key]) diff --git a/stix2/properties.py b/stix2/properties.py index e8c8442..99dc0a1 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -462,12 +462,12 @@ class ReferenceProperty(Property): if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix for this identifier is not valid") + raise ValueError("The type-specifying prefix '%s' for the identifier '%s' is not valid" % (possible_prefix, value)) elif self.invalid_types: if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("An invalid type-specifying prefix was specified for this identifier") + raise ValueError("An invalid type-specifying prefix '%s' was specified for the identifier '%s'" % (possible_prefix, value)) _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v20/test_utils.py b/stix2/test/v20/test_utils.py index ee011c1..0433fd5 100644 --- a/stix2/test/v20/test_utils.py +++ b/stix2/test/v20/test_utils.py @@ -144,8 +144,8 @@ def test_deduplicate(stix_objs1): "type": "network-traffic", "src_ref": "1", "protocols": [ - "tcp", - "http", + "tcp", + "http", ], "extensions": { "http-request-ext": { diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index 25348cd..2018395 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -127,7 +127,7 @@ def test_register_observable_with_default_version(): "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], }, }, ) @@ -165,7 +165,7 @@ def test_register_observable_extension_with_default_version(): "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], }, }, ) diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 92d035b..9908fb3 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,7 +86,7 @@ stix_objs = [ "objects": { "0": { "type": "file", - "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec", + "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", }, }, @@ -110,8 +110,8 @@ filters = [ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--fa1b868c-5fe2-5c85-8197-9674548379ec"}), + Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}}), + Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index dc27058..951943a 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -26,7 +26,7 @@ EXPECTED = """{ "objects": { "0": { "type": "file", - "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe" } } @@ -45,6 +45,7 @@ def test_observed_data_example(): objects={ "0": { "name": "foo.exe", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", }, }, @@ -66,15 +67,15 @@ EXPECTED_WITH_REF = """{ "objects": { "0": { "type": "file", - "id": "file--500d9a03-9d03-5c31-82b2-2be8aacec481", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe" }, "1": { "type": "directory", - "id": "directory--ed959127-2df3-5999-99b6-df7614398c1c", + "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", "path": "/usr/home", "contains_refs": [ - "0" + "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f" ] } } @@ -93,12 +94,14 @@ def test_observed_data_example_with_refs(): objects={ "0": { "name": "foo.exe", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", }, "1": { "type": "directory", + "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", "path": "/usr/home", - "contains_refs": ["0"], + "contains_refs": ["file--5956efbb-a7b0-566d-a7f9-a202eb05c70f"], }, }, ) @@ -117,9 +120,9 @@ EXPECTED_OBJECT_REFS = """{ "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "object_refs": [ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c" + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c" ] }""" @@ -134,9 +137,9 @@ def test_observed_data_example_with_object_refs(): last_observed="2015-12-21T19:00:00Z", number_observed=50, object_refs=[ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", ], ) @@ -160,15 +163,15 @@ def test_observed_data_object_constraint(): }, }, object_refs=[ - "foo--758bf2c0-a6f1-56d1-872e-6b727467739a", - "bar--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", - "baz--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", + "file--758bf2c0-a6f1-56d1-872e-6b727467739a", + "url--d97ed5c4-3f33-46d9-b25b-c3d7b94d1457", + "mutex--eca0b3ba-8d76-11e9-a1fd-34415dabec0c", ], ) def test_observed_data_example_with_bad_refs(): - with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: stix2.v21.ObservedData( id=OBSERVED_DATA_ID, created_by_ref=IDENTITY_ID, @@ -180,19 +183,20 @@ def test_observed_data_example_with_bad_refs(): objects={ "0": { "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "name": "foo.exe", }, "1": { "type": "directory", "path": "/usr/home", - "contains_refs": ["2"], + "contains_refs": ["monkey--5956efbb-a7b0-566d-a7f9-a202eb05c70f"], }, }, ) - assert excinfo.value.cls == stix2.v21.ObservedData - assert excinfo.value.prop_name == "objects" - assert excinfo.value.reason == "Invalid object reference for 'Directory:contains_refs': '2' is not a valid object in local scope" + assert excinfo.value.cls == stix2.v21.Directory + assert excinfo.value.prop_name == "contains_refs" + assert "The type-specifying prefix 'monkey--' for the identifier" in excinfo.value.reason def test_observed_data_example_with_non_dictionary(): @@ -248,6 +252,7 @@ def test_observed_data_example_with_empty_dictionary(): "0": { "name": "foo.exe", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", }, }, }, @@ -640,16 +645,18 @@ def test_observed_data_with_process_example(): objects={ "0": { "type": "file", + "id": "file--0d16c8d3-c177-5f5d-a022-b1bdac329bea", "hashes": { "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f", }, }, "1": { "type": "process", + "id": "process--f6c4a02c-23e1-4a6d-a0d7-d862e893817a", "pid": 1221, - "created": "2016-01-20T14:11:25.55Z", + "created_time": "2016-01-20T14:11:25.55Z", "command_line": "./gedit-bin --new-window", - "image_ref": "0", + "image_ref": "file--0d16c8d3-c177-5f5d-a022-b1bdac329bea", }, }, ) @@ -693,31 +700,33 @@ def test_artifact_mutual_exclusion_error(): def test_directory_example(): - dir = stix2.v21.Directory( - _valid_refs={"1": "file"}, + f = stix2.v21.File( + name="penguin.exe", + ) + + dir1 = stix2.v21.Directory( path='/usr/lib', ctime="2015-12-21T19:00:00Z", mtime="2015-12-24T19:00:00Z", atime="2015-12-21T20:00:00Z", - contains_refs=["1"], + contains_refs=[str(f.id)], ) - assert dir.path == '/usr/lib' - assert dir.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) - assert dir.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) - assert dir.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) - assert dir.contains_refs == ["1"] + assert dir1.path == '/usr/lib' + assert dir1.ctime == dt.datetime(2015, 12, 21, 19, 0, 0, tzinfo=pytz.utc) + assert dir1.mtime == dt.datetime(2015, 12, 24, 19, 0, 0, tzinfo=pytz.utc) + assert dir1.atime == dt.datetime(2015, 12, 21, 20, 0, 0, tzinfo=pytz.utc) + assert dir1.contains_refs == ["file--9d050a3b-72cd-5b57-bf18-024e74e1e5eb"] def test_directory_example_ref_error(): - with pytest.raises(stix2.exceptions.InvalidObjRefError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Directory( - _valid_refs=[], path='/usr/lib', ctime="2015-12-21T19:00:00Z", mtime="2015-12-24T19:00:00Z", atime="2015-12-21T20:00:00Z", - contains_refs=["1"], + contains_refs=["domain-name--02af94ea-7e38-5718-87c3-5cc023e3d49d"], ) assert excinfo.value.cls == stix2.v21.Directory @@ -725,22 +734,24 @@ def test_directory_example_ref_error(): def test_domain_name_example(): - dn = stix2.v21.DomainName( - _valid_refs={"1": 'domain-name'}, - value="example.com", - resolves_to_refs=["1"], + dn1 = stix2.v21.DomainName( + value="mitre.org", ) - assert dn.value == "example.com" - assert dn.resolves_to_refs == ["1"] + dn2 = stix2.v21.DomainName( + value="example.com", + resolves_to_refs=[str(dn1.id)], + ) + + assert dn2.value == "example.com" + assert dn2.resolves_to_refs == ["domain-name--02af94ea-7e38-5718-87c3-5cc023e3d49d"] def test_domain_name_example_invalid_ref_type(): - with pytest.raises(stix2.exceptions.InvalidObjRefError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.DomainName( - _valid_refs={"1": "file"}, value="example.com", - resolves_to_refs=["1"], + resolves_to_refs=["file--44a431e6-764b-5556-a3f5-bf655930a581"], ) assert excinfo.value.cls == stix2.v21.DomainName @@ -882,6 +893,7 @@ RASTER_IMAGE_EXT = """{ "objects": { "0": { "type": "file", + "id": "file--44a431e6-764b-5556-a3f5-bf655930a581", "name": "picture.jpg", "hashes": { "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" @@ -993,18 +1005,17 @@ def test_file_example_encryption_error(): assert "At least one of the (hashes, name)" in str(excinfo.value) -def test_ip4_address_example(): +def test_ipv4_address_example(): ip4 = stix2.v21.IPv4Address( - _valid_refs={"4": "mac-addr", "5": "mac-addr"}, value="198.51.100.3", - resolves_to_refs=["4", "5"], + resolves_to_refs=["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"], ) assert ip4.value == "198.51.100.3" - assert ip4.resolves_to_refs == ["4", "5"] + assert ip4.resolves_to_refs == ["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"] -def test_ip4_address_valid_refs(): +def test_ipv4_address_valid_refs(): mac1 = stix2.v21.MACAddress( value="a1:b2:c3:d4:e5:f6", ) @@ -1013,22 +1024,21 @@ def test_ip4_address_valid_refs(): ) ip4 = stix2.v21.IPv4Address( - _valid_refs={"1": mac1, "2": mac2}, value="177.60.40.7", - resolves_to_refs=["1", "2"], + resolves_to_refs=[str(mac1.id), str(mac2.id)], ) assert ip4.value == "177.60.40.7" - assert ip4.resolves_to_refs == ["1", "2"] + assert ip4.resolves_to_refs == ["mac-addr--a85820f7-d9b7-567a-a3a6-dedc34139342", "mac-addr--9a59b496-fdeb-510f-97b5-7137210bc699"] -def test_ip4_address_example_cidr(): +def test_ipv4_address_example_cidr(): ip4 = stix2.v21.IPv4Address(value="198.51.100.0/24") assert ip4.value == "198.51.100.0/24" -def test_ip6_address_example(): +def test_ipv6_address_example(): ip6 = stix2.v21.IPv6Address(value="2001:0db8:85a3:0000:0000:8a2e:0370:7334") assert ip6.value == "2001:0db8:85a3:0000:0000:8a2e:0370:7334" @@ -1042,14 +1052,13 @@ def test_mac_address_example(): def test_network_traffic_example(): nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr", "1": "ipv4-addr"}, - protocols="tcp", - src_ref="0", - dst_ref="1", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", + dst_ref="ipv4-addr--6d39dd0b-1f74-5faf-8d76-d8762c2a57cb", ) assert nt.protocols == ["tcp"] - assert nt.src_ref == "0" - assert nt.dst_ref == "1" + assert nt.src_ref == "ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88" + assert nt.dst_ref == "ipv4-addr--6d39dd0b-1f74-5faf-8d76-d8762c2a57cb" def test_network_traffic_http_request_example(): @@ -1064,9 +1073,8 @@ def test_network_traffic_http_request_example(): }, ) nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'http-request-ext': h}, ) assert nt.extensions['http-request-ext'].request_method == "get" @@ -1080,9 +1088,8 @@ def test_network_traffic_http_request_example(): def test_network_traffic_icmp_example(): h = stix2.v21.ICMPExt(icmp_type_hex="08", icmp_code_hex="00") nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'icmp-ext': h}, ) assert nt.extensions['icmp-ext'].icmp_type_hex == "08" @@ -1097,9 +1104,8 @@ def test_network_traffic_socket_example(): socket_type="SOCK_STREAM", ) nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'socket-ext': h}, ) assert nt.extensions['socket-ext'].is_listening @@ -1111,9 +1117,8 @@ def test_network_traffic_socket_example(): def test_network_traffic_tcp_example(): h = stix2.v21.TCPExt(src_flags_hex="00000002") nt = stix2.v21.NetworkTraffic( - _valid_refs={"0": "ipv4-addr"}, - protocols="tcp", - src_ref="0", + protocols=["tcp"], + src_ref="ipv4-addr--29a591d9-533a-5ecd-a5a1-cadee4411e88", extensions={'tcp-ext': h}, ) assert nt.extensions['tcp-ext'].src_flags_hex == "00000002" @@ -1127,11 +1132,10 @@ def test_mutex_example(): def test_process_example(): p = stix2.v21.Process( - _valid_refs={"0": "file"}, pid=1221, - created="2016-01-20T14:11:25.55Z", + created_time="2016-01-20T14:11:25.55Z", command_line="./gedit-bin --new-window", - image_ref="0", + image_ref="file--ea587d87-5ed2-5625-a9ac-01fd64161fd8", ) assert p.command_line == "./gedit-bin --new-window" @@ -1143,7 +1147,7 @@ def test_process_example_empty_error(): assert excinfo.value.cls == stix2.v21.Process properties_of_process = list(stix2.v21.Process._properties.keys()) - properties_of_process.remove("type") + properties_of_process = [prop for prop in properties_of_process if prop not in ["type", "id", "defanged", "spec_version"]] assert excinfo.value.properties == sorted(properties_of_process) msg = "At least one of the ({1}) properties for {0} must be populated." msg = msg.format( @@ -1367,18 +1371,20 @@ def test_new_version_with_related_objects(): objects={ 'src_ip': { 'type': 'ipv4-addr', + 'id': 'ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f', 'value': '127.0.0.1/32', }, 'domain': { 'type': 'domain-name', + 'id': 'domain-name--220a2699-5ebf-5b57-bf02-424964bb19c0', 'value': 'example.com', - 'resolves_to_refs': ['src_ip'], + 'resolves_to_refs': ['ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f'], }, }, ) new_version = data.new_version(last_observed="2017-12-12T12:00:00Z") assert new_version.last_observed.year == 2017 - assert new_version.objects['domain'].resolves_to_refs[0] == 'src_ip' + assert new_version.objects['domain'].resolves_to_refs[0] == 'ipv4-addr--2b94bc65-17d4-54f6-9ffe-7d103551bb9f' def test_objects_deprecation(): diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 899ccd8..19c53d8 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -233,7 +233,7 @@ class Report(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index fbce8fd..5719e42 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -184,18 +184,18 @@ class EmailMessage(_Observable): ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), - ('from_ref', ReferenceProperty(valid_types='email-addr')), - ('sender_ref', ReferenceProperty(valid_types='email-addr')), - ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), - ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), - ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr'))), + ('from_ref', ReferenceProperty(valid_types='email-addr', spec_version='2.1')), + ('sender_ref', ReferenceProperty(valid_types='email-addr', spec_version='2.1')), + ('to_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), + ('cc_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), + ('bcc_refs', ListProperty(ReferenceProperty(valid_types='email-addr', spec_version='2.1'))), ('message_id', StringProperty()), ('subject', StringProperty()), ('received_lines', ListProperty(StringProperty)), ('additional_header_fields', DictionaryProperty(spec_version='2.1')), ('body', StringProperty()), ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), - ('raw_email_ref', ReferenceProperty(valid_types='artifact')), + ('raw_email_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -384,9 +384,9 @@ class File(_Observable): ('ctime', TimestampProperty()), ('mtime', TimestampProperty()), ('atime', TimestampProperty()), - ('parent_directory_ref', ReferenceProperty(valid_types='directory')), - ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""]))), - ('content_ref', ReferenceProperty(valid_types='artifact')), + ('parent_directory_ref', ReferenceProperty(valid_types='directory', spec_version='2.1')), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'))), + ('content_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -411,8 +411,8 @@ class IPv4Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -448,8 +448,8 @@ class IPv6Address(_Observable): ('type', TypeProperty(_type)), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), - ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr'))), - ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system'))), + ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), + ('belongs_to_refs', ListProperty(ReferenceProperty(valid_types='autonomous-system', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -629,10 +629,10 @@ class NetworkTraffic(_Observable): ('src_packets', IntegerProperty(min=0)), ('dst_packets', IntegerProperty(min=0)), ('ipfix', DictionaryProperty(spec_version='2.1')), - ('src_payload_ref', ReferenceProperty(valid_types='artifact')), - ('dst_payload_ref', ReferenceProperty(valid_types='artifact')), - ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), - ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic')), + ('src_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), + ('dst_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), + ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))), + ('encapsulated_by_ref', ReferenceProperty(valid_types='network-traffic', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -748,11 +748,11 @@ class Process(_Observable): ('cwd', StringProperty()), ('command_line', StringProperty()), ('environment_variables', DictionaryProperty(spec_version='2.1')), - ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic'))), - ('creator_user_ref', ReferenceProperty(valid_types='user-account')), - ('image_ref', ReferenceProperty(valid_types='file')), - ('parent_ref', ReferenceProperty(valid_types='process')), - ('child_refs', ListProperty(ReferenceProperty(valid_types='process'))), + ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))), + ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), + ('image_ref', ReferenceProperty(valid_types='file', spec_version='2.1')), + ('parent_ref', ReferenceProperty(valid_types='process', spec_version='2.1')), + ('child_refs', ListProperty(ReferenceProperty(valid_types='process', spec_version='2.1'))), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -912,7 +912,7 @@ class WindowsRegistryKey(_Observable): ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself ('modified_time', TimestampProperty()), - ('creator_user_ref', ReferenceProperty(valid_types='user-account')), + ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')), ('number_of_subkeys', IntegerProperty()), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 3062659..4cae0e0 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -151,7 +151,7 @@ class Grouping(STIXDomainObject): ('name', StringProperty()), ('description', StringProperty()), ('context', StringProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ]) @@ -533,7 +533,7 @@ class Note(STIXDomainObject): ('abstract', StringProperty()), ('content', StringProperty(required=True)), ('authors', ListProperty(StringProperty)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -601,7 +601,7 @@ class ObservedData(STIXDomainObject): if self.get('object_refs'): for identifier in self.get('object_refs'): - identifier_prefix = identifier[:identifier.index('--') + 2] + identifier_prefix = identifier[:identifier.index('--')] if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): break else: @@ -635,7 +635,7 @@ class Opinion(STIXDomainObject): ], required=True, ), ), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -664,7 +664,7 @@ class Report(STIXDomainObject): ('description', StringProperty()), ('report_types', ListProperty(StringProperty, required=True)), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""]), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), From 8f773fd556d08b07dfa896f0d0a0ea6bd2b384a6 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 6 Sep 2019 00:25:42 -0400 Subject: [PATCH 55/62] Temp backup of some code changes. More coming soon --- stix2/base.py | 5 ++++ stix2/datastore/filters.py | 16 ++++++++++++ stix2/properties.py | 4 +-- stix2/test/v21/test_datastore_filters.py | 5 ++-- stix2/test/v21/test_observed_data.py | 31 +++++++++++++++++------- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 5bc3d10..09ad884 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -323,6 +323,11 @@ class _Observable(_STIXBase): super(_Observable, self).__init__(**kwargs) def _check_ref(self, ref, prop, prop_name): + """ + Only for checking `*_ref` or `*_refs` properties in spec_version 2.0 + STIX Cyber Observables (SCOs) + """ + if '*' in self._STIXBase__valid_refs: return # don't check if refs are valid diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index 4f72b82..af21b10 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -90,6 +90,14 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): filter_value = self.value if self.op == "=": + boolA = stix_obj_property == filter_value + if boolA is False: + print ('$$$$$') + print (stix_obj_property) + print ('\n') + print (filter_value) + print ('\n') + print ('$$$$$') return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -97,6 +105,14 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): return stix_obj_property in filter_value elif self.op == "contains": if isinstance(filter_value, dict): + boolB = filter_value in stix_obj_property.values() + if boolB is False: + print ('$$$$$') + print (filter_value) + print ('\n') + print (stix_obj_property.values()) + print ('\n') + print ('$$$$$') return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/properties.py b/stix2/properties.py index 99dc0a1..0661820 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -462,12 +462,12 @@ class ReferenceProperty(Property): if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix else: - raise ValueError("The type-specifying prefix '%s' for the identifier '%s' is not valid" % (possible_prefix, value)) + raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (possible_prefix)) elif self.invalid_types: if possible_prefix[:-2] not in self.invalid_types: required_prefix = possible_prefix else: - raise ValueError("An invalid type-specifying prefix '%s' was specified for the identifier '%s'" % (possible_prefix, value)) + raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix, value)) _validate_id(value, self.spec_version, required_prefix) diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 9908fb3..fd76ae4 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -86,6 +86,7 @@ stix_objs = [ "objects": { "0": { "type": "file", + "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", }, @@ -110,8 +111,8 @@ filters = [ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}}), - Filter("objects", "contains", {"type": "file", "name": "HAL 9000.exe", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876"}), + Filter("objects", "=", {"0": {"type": "file", "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe"}}), + Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 951943a..969f62d 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -27,6 +27,7 @@ EXPECTED = """{ "0": { "type": "file", "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "spec_version": "2.1", "name": "foo.exe" } } @@ -44,14 +45,19 @@ def test_observed_data_example(): number_observed=50, objects={ "0": { - "name": "foo.exe", - "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "name": "foo.exe", }, }, ) - assert str(observed_data) == EXPECTED + assert observed_data.id == "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" + assert observed_data.created_by_ref == "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" + assert observed_data.created == observed_data.modified == dt.datetime(2016, 4, 6, 19, 58, 16, tzinfo=pytz.utc) + assert observed_data.first_observed == observed_data.last_observed == dt.datetime(2015, 12, 21, 19, 00, 00, tzinfo=pytz.utc) + assert observed_data.number_observed == 50 + assert observed_data.objects['0'] == stix2.v21.File(name="foo.exe") EXPECTED_WITH_REF = """{ @@ -68,11 +74,13 @@ EXPECTED_WITH_REF = """{ "0": { "type": "file", "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "spec_version": "2.1", "name": "foo.exe" }, "1": { "type": "directory", "id": "directory--536a61a4-0934-516b-9aad-fcbb75e0583a", + "spec_version": "2.1", "path": "/usr/home", "contains_refs": [ "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f" @@ -93,9 +101,9 @@ def test_observed_data_example_with_refs(): number_observed=50, objects={ "0": { - "name": "foo.exe", - "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", "type": "file", + "id": "file--5956efbb-a7b0-566d-a7f9-a202eb05c70f", + "name": "foo.exe", }, "1": { "type": "directory", @@ -105,8 +113,13 @@ def test_observed_data_example_with_refs(): }, }, ) - - assert str(observed_data) == EXPECTED_WITH_REF + assert observed_data.id == "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" + assert observed_data.created_by_ref == "identity--311b2d2d-f010-4473-83ec-1edf84858f4c" + assert observed_data.created == observed_data.modified == dt.datetime(2016, 4, 6, 19, 58, 16, tzinfo=pytz.utc) + assert observed_data.first_observed == observed_data.last_observed == dt.datetime(2015, 12, 21, 19, 00, 00, tzinfo=pytz.utc) + assert observed_data.number_observed == 50 + assert observed_data.objects['0'] == stix2.v21.File(name="foo.exe") + assert observed_data.objects['1'] == stix2.v21.Directory(path="/usr/home", contains_refs=["file--5956efbb-a7b0-566d-a7f9-a202eb05c70f"]) EXPECTED_OBJECT_REFS = """{ @@ -171,7 +184,7 @@ def test_observed_data_object_constraint(): def test_observed_data_example_with_bad_refs(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.ObservedData( id=OBSERVED_DATA_ID, created_by_ref=IDENTITY_ID, @@ -196,7 +209,7 @@ def test_observed_data_example_with_bad_refs(): assert excinfo.value.cls == stix2.v21.Directory assert excinfo.value.prop_name == "contains_refs" - assert "The type-specifying prefix 'monkey--' for the identifier" in excinfo.value.reason + assert "The type-specifying prefix 'monkey--' for this property is not valid" in excinfo.value.reason def test_observed_data_example_with_non_dictionary(): From 5b6592e2dc9ba212063fe39d237d78786354d72e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 6 Sep 2019 18:08:27 -0400 Subject: [PATCH 56/62] Some changes. More fixes coming soon, hopefully --- stix2/base.py | 14 ++-- stix2/datastore/filters.py | 22 +++++- stix2/properties.py | 2 +- stix2/test/v21/test_datastore_filters.py | 2 +- stix2/test/v21/test_observed_data.py | 92 +++++++++++------------- 5 files changed, 73 insertions(+), 59 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 09ad884..f933d3b 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -357,11 +357,12 @@ class _Observable(_STIXBase): return from .properties import ObjectReferenceProperty - if isinstance(prop, ObjectReferenceProperty): - if prop_name.endswith('_ref'): + if prop_name.endswith('_ref'): + if isinstance(prop, ObjectReferenceProperty): ref = kwargs[prop_name] self._check_ref(ref, prop, prop_name) - elif prop_name.endswith('_refs'): + elif prop_name.endswith('_refs'): + if isinstance(prop.contained, ObjectReferenceProperty): for ref in kwargs[prop_name]: self._check_ref(ref, prop, prop_name) @@ -392,7 +393,12 @@ class _Observable(_STIXBase): if streamlined_obj_vals: data = canonicalize(streamlined_obj_vals, utf8=False) - return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + # print (str(type(data))) + try: + return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + except UnicodeDecodeError: + return required_prefix + six.text_type(uuid.uuid5(namespace, six.binary_type(data))) + # return required_prefix + six.text_type(uuid.uuid5(namespace, data)) # We return None if there are no values specified for any of the id-contributing-properties return None diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index af21b10..806f672 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -98,6 +98,15 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): print (filter_value) print ('\n') print ('$$$$$') + pass + else: + # print ('222222222') + # print (stix_obj_property) + # print ('\n') + # print (filter_value) + # print ('\n') + # print ('222222222') + pass return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -107,12 +116,21 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): if isinstance(filter_value, dict): boolB = filter_value in stix_obj_property.values() if boolB is False: - print ('$$$$$') + print ('@@@@@@') print (filter_value) print ('\n') print (stix_obj_property.values()) print ('\n') - print ('$$$$$') + print ('@@@@@@@') + pass + else: + # print ('55555555555') + # print (filter_value) + # print ('\n') + # print (stix_obj_property.values()) + # print ('\n') + # print ('55555555555') + pass return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/properties.py b/stix2/properties.py index 0661820..c956a08 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -457,7 +457,7 @@ class ReferenceProperty(Property): elif self.valid_types == ["only_SCO"]: self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() elif self.valid_types == ["only_SCO_&_SRO"]: - self.valid_types = STIX2_OBJ_MAPS['v21']['observables'].keys() + ['relationship', 'sighting'] + self.valid_types = list(STIX2_OBJ_MAPS['v21']['observables'].keys()) + ['relationship', 'sighting'] if possible_prefix[:-2] in self.valid_types: required_prefix = possible_prefix diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index fd76ae4..96ac17f 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -111,7 +111,7 @@ filters = [ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe"}}), + Filter("objects", "=", {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}}), Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), Filter("labels", "contains", "heartbleed"), ] diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index 969f62d..fbebdbd 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -364,17 +364,21 @@ def test_parse_autonomous_system_valid(data): "type": "email-addr", "value": "john@example.com", "display_name": "John Doe", - "belongs_to_ref": "0" + "belongs_to_ref": "user-account--fc07c1af-6b11-41f8-97a4-47920d866a91" }""", ], ) def test_parse_email_address(data): - odata = stix2.parse_observable(data, {"0": "user-account"}, version='2.1') + odata = stix2.parse_observable(data, version='2.1') assert odata.type == "email-addr" - odata_str = re.compile('"belongs_to_ref": "0"', re.DOTALL).sub('"belongs_to_ref": "3"', data) - with pytest.raises(stix2.exceptions.InvalidObjRefError): - stix2.parse_observable(odata_str, {"0": "user-account"}, version='2.1') + odata_str = re.compile( + '"belongs_to_ref": "user-account--fc07c1af-6b11-41f8-97a4-47920d866a91"', re.DOTALL, + ).sub( + '"belongs_to_ref": "mutex--9be6365f-b89c-48c0-9340-6953f6595718"', data, + ) + with pytest.raises(stix2.exceptions.InvalidValueError): + stix2.parse_observable(odata_str, version='2.1') @pytest.mark.parametrize( @@ -385,12 +389,12 @@ def test_parse_email_address(data): "is_multipart": true, "content_type": "multipart/mixed", "date": "2016-06-19T14:20:40.000Z", - "from_ref": "1", + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", "to_refs": [ - "2" + "email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b" ], "cc_refs": [ - "3" + "email-addr--1766f860-5cf3-5697-8789-35f1242663d5" ], "subject": "Check out this picture of a cat!", "additional_header_fields": { @@ -407,12 +411,12 @@ def test_parse_email_address(data): { "content_type": "image/png", "content_disposition": "attachment; filename=\\"tabby.png\\"", - "body_raw_ref": "4" + "body_raw_ref": "artifact--80b04ad8-db52-464b-a85a-a44a5f3a60c5" }, { "content_type": "application/zip", "content_disposition": "attachment; filename=\\"tabby_pics.zip\\"", - "body_raw_ref": "5" + "body_raw_ref": "file--e63474fc-b386-5630-a003-1b555e22f99b" } ] } @@ -420,15 +424,7 @@ def test_parse_email_address(data): ], ) def test_parse_email_message(data): - valid_refs = { - "0": "email-message", - "1": "email-addr", - "2": "email-addr", - "3": "email-addr", - "4": "artifact", - "5": "file", - } - odata = stix2.parse_observable(data, valid_refs, version='2.1') + odata = stix2.parse_observable(data, version='2.1') assert odata.type == "email-message" assert odata.body_multipart[0].content_disposition == "inline" @@ -438,8 +434,8 @@ def test_parse_email_message(data): """ { "type": "email-message", - "from_ref": "0", - "to_refs": ["1"], + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", + "to_refs": ["email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b"], "is_multipart": true, "date": "1997-11-21T15:55:06.000Z", "subject": "Saying Hello", @@ -449,12 +445,8 @@ def test_parse_email_message(data): ], ) def test_parse_email_message_not_multipart(data): - valid_refs = { - "0": "email-addr", - "1": "email-addr", - } with pytest.raises(stix2.exceptions.DependentPropertiesError) as excinfo: - stix2.parse_observable(data, valid_refs, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.EmailMessage assert excinfo.value.dependencies == [("is_multipart", "body")] @@ -464,18 +456,21 @@ def test_parse_email_message_not_multipart(data): "data", [ """"0": { "type": "file", + "id": "file--ecd47d73-15e4-5250-afda-ef8897b22340", "hashes": { "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a" } }, "1": { "type": "file", + "id": "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", "hashes": { "SHA-256": "19c549ec2628b989382f6b280cbd7bb836a0b461332c0fe53511ce7d584b89d3" } }, "2": { "type": "file", + "id": "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", "hashes": { "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038" } @@ -490,9 +485,9 @@ def test_parse_email_message_not_multipart(data): "extensions": { "archive-ext": { "contains_refs": [ - "0", - "1", - "2" + "file--ecd47d73-15e4-5250-afda-ef8897b22340", + "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", + "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac" ] } } @@ -503,7 +498,11 @@ def test_parse_file_archive(data): odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str, version="2.1") assert all(x in odata.objects["3"].extensions['archive-ext'].contains_refs - for x in ["0", "1", "2"]) + for x in [ + "file--ecd47d73-15e4-5250-afda-ef8897b22340", + "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", + "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", + ]) @pytest.mark.parametrize( @@ -514,12 +513,12 @@ def test_parse_file_archive(data): "is_multipart": true, "content_type": "multipart/mixed", "date": "2016-06-19T14:20:40.000Z", - "from_ref": "1", + "from_ref": "email-addr--d4ef7e1f-086d-5ff4-bce4-312ddc3eae76", "to_refs": [ - "2" + "email-addr--8b0eb924-208c-5efd-80e5-84e2d610e54b" ], "cc_refs": [ - "3" + "email-addr--1766f860-5cf3-5697-8789-35f1242663d5" ], "subject": "Check out this picture of a cat!", "additional_header_fields": { @@ -540,7 +539,7 @@ def test_parse_file_archive(data): { "content_type": "application/zip", "content_disposition": "attachment; filename=\\"tabby_pics.zip\\"", - "body_raw_ref": "5" + "body_raw_ref": "file--e63474fc-b386-5630-a003-1b555e22f99b" } ] } @@ -548,16 +547,8 @@ def test_parse_file_archive(data): ], ) def test_parse_email_message_with_at_least_one_error(data): - valid_refs = { - "0": "email-message", - "1": "email-addr", - "2": "email-addr", - "3": "email-addr", - "4": "artifact", - "5": "file", - } with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: - stix2.parse_observable(data, valid_refs, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.EmailMessage assert "At least one of the" in str(excinfo.value) @@ -569,8 +560,8 @@ def test_parse_email_message_with_at_least_one_error(data): """ { "type": "network-traffic", - "src_ref": "0", - "dst_ref": "1", + "src_ref": "ipv4-addr--e535b017-cc1c-566b-a3e2-f69f92ed9c4c", + "dst_ref": "ipv4-addr--78327430-9ad9-5632-ae3d-8e2fce8f5483", "protocols": [ "tcp" ] @@ -580,13 +571,12 @@ def test_parse_email_message_with_at_least_one_error(data): ) def test_parse_basic_tcp_traffic(data): odata = stix2.parse_observable( - data, {"0": "ipv4-addr", "1": "ipv4-addr"}, - version='2.1', + data, version='2.1', ) assert odata.type == "network-traffic" - assert odata.src_ref == "0" - assert odata.dst_ref == "1" + assert odata.src_ref == "ipv4-addr--e535b017-cc1c-566b-a3e2-f69f92ed9c4c" + assert odata.dst_ref == "ipv4-addr--78327430-9ad9-5632-ae3d-8e2fce8f5483" assert odata.protocols == ["tcp"] @@ -604,7 +594,7 @@ def test_parse_basic_tcp_traffic(data): "src_byte_count": 35779, "dst_byte_count": 935750, "encapsulates_refs": [ - "4" + "network-traffic--016914c3-b680-5df2-81c4-bb9ccf8dc8b0" ] } """, @@ -612,7 +602,7 @@ def test_parse_basic_tcp_traffic(data): ) def test_parse_basic_tcp_traffic_with_error(data): with pytest.raises(stix2.exceptions.AtLeastOnePropertyError) as excinfo: - stix2.parse_observable(data, {"4": "network-traffic"}, version='2.1') + stix2.parse_observable(data, version='2.1') assert excinfo.value.cls == stix2.v21.NetworkTraffic assert excinfo.value.properties == ["dst_ref", "src_ref"] From d828e41c78932aaf290371d95281021af10bfdae Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 10:44:14 -0400 Subject: [PATCH 57/62] End of changes --- stix2/base.py | 4 +-- stix2/canonicalization/Canonicalize.py | 8 ++++-- stix2/datastore/filters.py | 34 ------------------------ stix2/test/v21/test_datastore_filters.py | 11 ++++++-- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index f933d3b..4d4b080 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -393,12 +393,12 @@ class _Observable(_STIXBase): if streamlined_obj_vals: data = canonicalize(streamlined_obj_vals, utf8=False) - # print (str(type(data))) + + # try/except here to enable python 2 compatibility try: return required_prefix + six.text_type(uuid.uuid5(namespace, data)) except UnicodeDecodeError: return required_prefix + six.text_type(uuid.uuid5(namespace, six.binary_type(data))) - # return required_prefix + six.text_type(uuid.uuid5(namespace, data)) # We return None if there are no values specified for any of the id-contributing-properties return None diff --git a/stix2/canonicalization/Canonicalize.py b/stix2/canonicalization/Canonicalize.py index daf0616..78145be 100644 --- a/stix2/canonicalization/Canonicalize.py +++ b/stix2/canonicalization/Canonicalize.py @@ -24,6 +24,8 @@ import re +import six + from stix2.canonicalization.NumberToJson import convert2Es6Format try: @@ -395,7 +397,8 @@ def _make_iterencode( else: items = dct.items() for key, value in items: - if isinstance(key, str): + # Replaced isinstance(key, str) with below to enable simultaneous python 2 & 3 compatibility + if isinstance(key, six.string_types) or isinstance(key, six.binary_type): pass # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. @@ -454,7 +457,8 @@ def _make_iterencode( del markers[markerid] def _iterencode(o, _current_indent_level): - if isinstance(o, str): + # Replaced isinstance(o, str) with below to enable simultaneous python 2 & 3 compatibility + if isinstance(o, six.string_types) or isinstance(o, six.binary_type): yield _encoder(o) elif o is None: yield 'null' diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index 806f672..4f72b82 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -90,23 +90,6 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): filter_value = self.value if self.op == "=": - boolA = stix_obj_property == filter_value - if boolA is False: - print ('$$$$$') - print (stix_obj_property) - print ('\n') - print (filter_value) - print ('\n') - print ('$$$$$') - pass - else: - # print ('222222222') - # print (stix_obj_property) - # print ('\n') - # print (filter_value) - # print ('\n') - # print ('222222222') - pass return stix_obj_property == filter_value elif self.op == "!=": return stix_obj_property != filter_value @@ -114,23 +97,6 @@ class Filter(collections.namedtuple('Filter', ['property', 'op', 'value'])): return stix_obj_property in filter_value elif self.op == "contains": if isinstance(filter_value, dict): - boolB = filter_value in stix_obj_property.values() - if boolB is False: - print ('@@@@@@') - print (filter_value) - print ('\n') - print (stix_obj_property.values()) - print ('\n') - print ('@@@@@@@') - pass - else: - # print ('55555555555') - # print (filter_value) - # print ('\n') - # print (stix_obj_property.values()) - # print ('\n') - # print ('55555555555') - pass return filter_value in stix_obj_property.values() else: return filter_value in stix_obj_property diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index 96ac17f..b7b41a0 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -89,6 +89,7 @@ stix_objs = [ "spec_version": "2.1", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", + "defanged": False, }, }, @@ -111,8 +112,14 @@ filters = [ Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-4000-8000-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "description"), Filter("external_references.source_name", "=", "CVE"), - Filter("objects", "=", {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}}), - Filter("objects", "contains", {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1"}), + Filter( + "objects", "=", + {"0": {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1", "defanged": False}}, + ), + Filter( + "objects", "contains", + {"type": "file", "id": "file--42a7175a-42cc-508f-8fa7-23b330aff876", "name": "HAL 9000.exe", "spec_version": "2.1", "defanged": False}, + ), Filter("labels", "contains", "heartbleed"), ] From 9c7128d074a74f82421fd8f4e97b8c32c335a29d Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 10:49:11 -0400 Subject: [PATCH 58/62] Fix indentation issue --- stix2/base.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 4d4b080..dbe6461 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -412,18 +412,18 @@ class _Extension(_STIXBase): def _choose_one_hash(hash_dict): - if "MD5" in hash_dict: - return {"MD5": hash_dict["MD5"]} - elif "SHA-1" in hash_dict: - return {"SHA-1": hash_dict["SHA-1"]} - elif "SHA-256" in hash_dict: - return {"SHA-256": hash_dict["SHA-256"]} - elif "SHA-512" in hash_dict: - return {"SHA-512": hash_dict["SHA-512"]} - else: - k = next(iter(hash_dict), None) - if k is not None: - return {k: hash_dict[k]} + if "MD5" in hash_dict: + return {"MD5": hash_dict["MD5"]} + elif "SHA-1" in hash_dict: + return {"SHA-1": hash_dict["SHA-1"]} + elif "SHA-256" in hash_dict: + return {"SHA-256": hash_dict["SHA-256"]} + elif "SHA-512" in hash_dict: + return {"SHA-512": hash_dict["SHA-512"]} + else: + k = next(iter(hash_dict), None) + if k is not None: + return {k: hash_dict[k]} def _cls_init(cls, obj, kwargs): From 8447c9fcd97eaf7a9a1151305b119fecdda00845 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 11 Sep 2019 14:21:41 -0400 Subject: [PATCH 59/62] Add few tests to improve some code coverage --- docs/guide/datastore.ipynb | 18 ++++++++---- stix2/test/v21/test_observed_data.py | 42 ++++++++++++++++++++++++++-- stix2/v21/observables.py | 2 +- stix2/v21/sdo.py | 15 ++-------- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 1c88cf1..1ea05ee 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -450,6 +450,14 @@ "mem.source.filters.add([f1,f2])" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Note: The `defanged` property is now always included (implicitly) for STIX 2.1 Cyber Observable Objects (SCOs)\n", + "This is important to remember if you are writing a filter that involves checking the `objects` property of a STIX 2.1 `ObservedData` object. If any of the objects associated with the `objects` property are STIX 2.1 SCOs, then your filter must include the `defanged` property. For an example, refer to `filters[14]` & `filters[15]` in stix2/test/v21/test_datastore_filters.py " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -726,21 +734,21 @@ ], "metadata": { "kernelspec": { - "display_name": "cti-python-stix2", + "display_name": "Python 3", "language": "python", - "name": "cti-python-stix2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.7" } }, "nbformat": 4, diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index fbebdbd..0074bf7 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -465,21 +465,21 @@ def test_parse_email_message_not_multipart(data): "type": "file", "id": "file--65f2873d-38c2-56b4-bfa5-e3ef21e8a3c3", "hashes": { - "SHA-256": "19c549ec2628b989382f6b280cbd7bb836a0b461332c0fe53511ce7d584b89d3" + "SHA-1": "6e71b3cac15d32fe2d36c270887df9479c25c640" } }, "2": { "type": "file", "id": "file--ef2d6dca-ec7d-5ab7-8dd9-ec9c0dee0eac", "hashes": { - "SHA-256": "0969de02ecf8a5f003e3f6d063d848c8a193aada092623f8ce408c15bcb5f038" + "SHA-512": "b7e98c78c24fb4c2c7b175e90474b21eae0ccf1b5ea4708b4e0f2d2940004419edc7161c18a1e71b2565df099ba017bcaa67a248e2989b6268ce078b88f2e210" } }, "3": { "type": "file", "name": "foo.zip", "hashes": { - "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + "SHA3-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" }, "mime_type": "application/zip", "extensions": { @@ -1490,3 +1490,39 @@ def test_deterministic_id_no_contributing_props(): uuid_obj_2 = uuid.UUID(email_msg_2.id[-36:]) assert uuid_obj_2.variant == uuid.RFC_4122 assert uuid_obj_2.version == 4 + + +def test_ipv4_resolves_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv4Address( + value="26.09.19.70", + resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], + ) + + +def test_ipv4_belongs_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv4Address( + value="21.12.19.64", + belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], + ) + + +def test_ipv6_resolves_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv6Address( + value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", + resolves_to_refs=["mac-addr--08900593-0265-52fc-93c0-5b4a942f5887"], + ) + + +def test_ipv6_belongs_to_refs_deprecation(): + with pytest.warns(stix2.exceptions.STIXDeprecationWarning): + + stix2.v21.IPv6Address( + value="2001:0db8:85a3:0000:0000:8a2e:0370:7334", + belongs_to_refs=["autonomous-system--52e0a49d-d683-5801-a7b8-145765a1e116"], + ) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 5719e42..9fffeed 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -385,7 +385,7 @@ class File(_Observable): ('mtime', TimestampProperty()), ('atime', TimestampProperty()), ('parent_directory_ref', ReferenceProperty(valid_types='directory', spec_version='2.1')), - ('contains_refs', ListProperty(ReferenceProperty(invalid_types=[""], spec_version='2.1'))), + ('contains_refs', ListProperty(ReferenceProperty(invalid_types="", spec_version='2.1'))), ('content_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), ('spec_version', StringProperty(fixed='2.1')), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 4cae0e0..8bf76b8 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -6,11 +6,9 @@ import warnings from six.moves.urllib.parse import quote_plus -from ..core import STIX2_OBJ_MAPS, STIXDomainObject +from ..core import STIXDomainObject from ..custom import _custom_object_builder -from ..exceptions import ( - InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, -) +from ..exceptions import PropertyPresenceError, STIXDeprecationWarning from ..properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, ListProperty, @@ -597,16 +595,9 @@ class ObservedData(STIXDomainObject): self._check_mutually_exclusive_properties( ["objects", "object_refs"], + at_least_one=False, ) - if self.get('object_refs'): - for identifier in self.get('object_refs'): - identifier_prefix = identifier[:identifier.index('--')] - if identifier_prefix in STIX2_OBJ_MAPS['v21']['observables'].keys(): - break - else: - raise InvalidValueError(self.__class__, 'object_refs', "At least one identifier must be of a SCO type if this property specified") - class Opinion(STIXDomainObject): # TODO: Add link From 3b1c922ba63a1517b43dd4ed07a511d8e380c331 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 18 Sep 2019 10:29:07 -0400 Subject: [PATCH 60/62] Fix observed data property check for at least one property existing --- stix2/v21/sdo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 8bf76b8..7cd7891 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -595,7 +595,6 @@ class ObservedData(STIXDomainObject): self._check_mutually_exclusive_properties( ["objects", "object_refs"], - at_least_one=False, ) From f241ed5c6c878a224b80a397e6b5d22fb52c6742 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Wed, 18 Sep 2019 10:56:42 -0400 Subject: [PATCH 61/62] Remove at_least_one=False from Artifact SCO --- stix2/v21/observables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 9fffeed..0d27bb4 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -50,7 +50,7 @@ class Artifact(_Observable): def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() - self._check_mutually_exclusive_properties(['payload_bin', 'url'], at_least_one=False) + self._check_mutually_exclusive_properties(['payload_bin', 'url']) self._check_properties_dependency(['hashes'], ['url']) From 113d481e84ef2a3ea426413f610eede1bf3c0e18 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Thu, 19 Sep 2019 10:31:14 -0400 Subject: [PATCH 62/62] Make SCO deterministic ID namespace a global var for better software hygiene --- stix2/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index dbe6461..a13cb98 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -23,6 +23,7 @@ from .utils import revoke as _revoke __all__ = ['STIXJSONEncoder', '_STIXBase'] DEFAULT_ERROR = "{type} must have {property}='{expected}'." +SCO_DET_ID_NAMESPACE = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") class STIXJSONEncoder(json.JSONEncoder): @@ -368,7 +369,6 @@ class _Observable(_STIXBase): def _generate_id(self, kwargs): required_prefix = self._type + "--" - namespace = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") properties_to_use = self._id_contributing_properties if properties_to_use: @@ -396,9 +396,9 @@ class _Observable(_STIXBase): # try/except here to enable python 2 compatibility try: - return required_prefix + six.text_type(uuid.uuid5(namespace, data)) + return required_prefix + six.text_type(uuid.uuid5(SCO_DET_ID_NAMESPACE, data)) except UnicodeDecodeError: - return required_prefix + six.text_type(uuid.uuid5(namespace, six.binary_type(data))) + return required_prefix + six.text_type(uuid.uuid5(SCO_DET_ID_NAMESPACE, six.binary_type(data))) # We return None if there are no values specified for any of the id-contributing-properties return None