From 14dce03616d2d50c902cb3e825f878a8029e10bf Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 16 Apr 2018 14:37:07 -0400 Subject: [PATCH] Provide default for revoked, sighting:summary. This allows filter on un-revoked objects. Changes default JSONEncoder to drop optional properties with default values in the spec if set to the default value. They can be included by passing include_optional_defaults=True to serialize(). --- stix2/base.py | 54 ++++++++++++++++++++++++++++++--- stix2/test/test_bundle.py | 12 ++++---- stix2/test/test_datastore.py | 4 +-- stix2/test/test_indicator.py | 1 + stix2/test/test_relationship.py | 8 ++--- stix2/test/test_sighting.py | 2 +- stix2/test/test_tool.py | 27 +++++++++++++++++ stix2/v20/sdo.py | 26 ++++++++-------- stix2/v20/sro.py | 6 ++-- 9 files changed, 105 insertions(+), 35 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 3219007..e128d48 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -22,6 +22,33 @@ DEFAULT_ERROR = "{type} must have {property}='{expected}'." class STIXJSONEncoder(json.JSONEncoder): + """Custom JSONEncoder subclass for serializing Python ``stix2`` objects. + + If an optional property with a default value specified in the STIX 2 spec + is set to that default value, it will be left out of the serialized output. + + An example of this type of property include the ``revoked`` common property. + """ + + def default(self, obj): + if isinstance(obj, (dt.date, dt.datetime)): + return format_datetime(obj) + elif isinstance(obj, _STIXBase): + tmp_obj = dict(copy.deepcopy(obj)) + for prop_name in obj._defaulted_optional_properties: + del tmp_obj[prop_name] + return tmp_obj + else: + return super(STIXJSONEncoder, self).default(obj) + + +class STIXJSONIncludeOptionalDefaultsEncoder(json.JSONEncoder): + """Custom JSONEncoder subclass for serializing Python ``stix2`` objects. + + Differs from ``STIXJSONEncoder`` in that if an optional property with a default + value specified in the STIX 2 spec is set to that default value, it will be + included in the serialized output. + """ def default(self, obj): if isinstance(obj, (dt.date, dt.datetime)): @@ -122,14 +149,25 @@ class _STIXBase(collections.Mapping): setting_kwargs[prop_name] = prop_value # Detect any missing required properties - required_properties = get_required_properties(cls._properties) - missing_kwargs = set(required_properties) - set(setting_kwargs) + required_properties = set(get_required_properties(cls._properties)) + missing_kwargs = required_properties - set(setting_kwargs) if missing_kwargs: raise MissingPropertiesError(cls, missing_kwargs) for prop_name, prop_metadata in cls._properties.items(): self._check_property(prop_name, prop_metadata, setting_kwargs) + # Cache defaulted optional properties for serialization + defaulted = [] + for name, prop in cls._properties.items(): + try: + if (not prop.required and not hasattr(prop, '_fixed_value') and + prop.default() == setting_kwargs[name]): + defaulted.append(name) + except (AttributeError, KeyError): + continue + self._defaulted_optional_properties = defaulted + self._inner = setting_kwargs self._check_object_constraints() @@ -151,7 +189,7 @@ class _STIXBase(collections.Mapping): (self.__class__.__name__, name)) def __setattr__(self, name, value): - if name != '_inner' and not name.startswith("_STIXBase__"): + if not name.startswith("_"): raise ImmutableError(self.__class__, name) super(_STIXBase, self).__setattr__(name, value) @@ -170,6 +208,7 @@ class _STIXBase(collections.Mapping): if isinstance(self, _Observable): # Assume: valid references in the original object are still valid in the new version new_inner['_valid_refs'] = {'*': '*'} + new_inner['allow_custom'] = self.__allow_custom return cls(**new_inner) def properties_populated(self): @@ -183,7 +222,7 @@ class _STIXBase(collections.Mapping): def revoke(self): return _revoke(self) - def serialize(self, pretty=False, **kwargs): + def serialize(self, pretty=False, include_optional_defaults=False, **kwargs): """ Serialize a STIX object. @@ -191,6 +230,8 @@ class _STIXBase(collections.Mapping): pretty (bool): If True, output properties following the STIX specs formatting. This includes indentation. Refer to notes for more details. (Default: ``False``) + include_optional_defaults (bool): Determines whether to include + optional properties set to the default value defined in the spec. **kwargs: The arguments for a json.dumps() call. Returns: @@ -213,7 +254,10 @@ class _STIXBase(collections.Mapping): kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by}) - return json.dumps(self, cls=STIXJSONEncoder, **kwargs) + if include_optional_defaults: + return json.dumps(self, cls=STIXJSONIncludeOptionalDefaultsEncoder, **kwargs) + else: + return json.dumps(self, cls=STIXJSONEncoder, **kwargs) class _Observable(_STIXBase): diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 8b14172..2d14654 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -6,7 +6,7 @@ import stix2 EXPECTED_BUNDLE = """{ "type": "bundle", - "id": "bundle--00000000-0000-0000-0000-000000000004", + "id": "bundle--00000000-0000-0000-0000-000000000007", "spec_version": "2.0", "objects": [ { @@ -22,7 +22,7 @@ EXPECTED_BUNDLE = """{ }, { "type": "malware", - "id": "malware--00000000-0000-0000-0000-000000000002", + "id": "malware--00000000-0000-0000-0000-000000000003", "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "name": "Cryptolocker", @@ -32,7 +32,7 @@ EXPECTED_BUNDLE = """{ }, { "type": "relationship", - "id": "relationship--00000000-0000-0000-0000-000000000003", + "id": "relationship--00000000-0000-0000-0000-000000000005", "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "relationship_type": "indicates", @@ -44,7 +44,7 @@ EXPECTED_BUNDLE = """{ EXPECTED_BUNDLE_DICT = { "type": "bundle", - "id": "bundle--00000000-0000-0000-0000-000000000004", + "id": "bundle--00000000-0000-0000-0000-000000000007", "spec_version": "2.0", "objects": [ { @@ -60,7 +60,7 @@ EXPECTED_BUNDLE_DICT = { }, { "type": "malware", - "id": "malware--00000000-0000-0000-0000-000000000002", + "id": "malware--00000000-0000-0000-0000-000000000003", "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "name": "Cryptolocker", @@ -70,7 +70,7 @@ EXPECTED_BUNDLE_DICT = { }, { "type": "relationship", - "id": "relationship--00000000-0000-0000-0000-000000000003", + "id": "relationship--00000000-0000-0000-0000-000000000005", "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "relationship_type": "indicates", diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py index 1c7fa43..315aff5 100644 --- a/stix2/test/test_datastore.py +++ b/stix2/test/test_datastore.py @@ -405,13 +405,11 @@ def test_apply_common_filters4(): def test_apply_common_filters5(): # "Return any object whose not revoked" - # Note that if 'revoked' property is not present in object. - # Currently we can't use such an expression to filter for... :( resp = list(apply_common_filters(stix_objs, [filters[5]])) assert len(resp) == 0 resp = list(apply_common_filters(real_stix_objs, [filters[5]])) - assert len(resp) == 0 + assert len(resp) == 4 def test_apply_common_filters6(): diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index 78d1bf2..c9b6e56 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -45,6 +45,7 @@ def test_indicator_with_all_required_properties(): labels=['malicious-activity'], ) + assert ind.revoked is False assert str(ind) == EXPECTED_INDICATOR rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(ind)) assert rep == EXPECTED_INDICATOR_REPR diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py index c6e2d6f..32ddf91 100644 --- a/stix2/test/test_relationship.py +++ b/stix2/test/test_relationship.py @@ -123,8 +123,8 @@ def test_create_relationship_from_objects_rather_than_ids(indicator, malware): assert rel.relationship_type == 'indicates' assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001' - assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002' - assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003' + assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003' + assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005' def test_create_relationship_with_positional_args(indicator, malware): @@ -132,8 +132,8 @@ def test_create_relationship_with_positional_args(indicator, malware): assert rel.relationship_type == 'indicates' assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001' - assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002' - assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003' + assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003' + assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005' @pytest.mark.parametrize("data", [ diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py index ce1fab9..06f96b8 100644 --- a/stix2/test/test_sighting.py +++ b/stix2/test/test_sighting.py @@ -86,7 +86,7 @@ def test_create_sighting_from_objects_rather_than_ids(malware): # noqa: F811 rel = stix2.Sighting(sighting_of_ref=malware) assert rel.sighting_of_ref == 'malware--00000000-0000-0000-0000-000000000001' - assert rel.id == 'sighting--00000000-0000-0000-0000-000000000002' + assert rel.id == 'sighting--00000000-0000-0000-0000-000000000003' @pytest.mark.parametrize("data", [ diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py index ce99fb8..9fc2c22 100644 --- a/stix2/test/test_tool.py +++ b/stix2/test/test_tool.py @@ -19,6 +19,19 @@ EXPECTED = """{ ] }""" +EXPECTED_WITH_REVOKED = """{ + "type": "tool", + "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", + "modified": "2016-04-06T20:03:48.000Z", + "name": "VNC", + "revoked": false, + "labels": [ + "remote-access" + ] +}""" + def test_tool_example(): tool = stix2.Tool( @@ -64,4 +77,18 @@ def test_tool_no_workbench_wrappers(): with pytest.raises(AttributeError): tool.created_by() + +def test_tool_serialize_with_defaults(): + tool = stix2.Tool( + id="tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T20:03:48.000Z", + modified="2016-04-06T20:03:48.000Z", + name="VNC", + labels=["remote-access"], + ) + + assert tool.serialize(pretty=True, include_optional_defaults=True) == EXPECTED_WITH_REVOKED + + # TODO: Add other examples diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index ff024f5..d7e9954 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -36,7 +36,7 @@ class AttackPattern(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -63,7 +63,7 @@ class Campaign(STIXDomainObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('objective', StringProperty()), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -86,7 +86,7 @@ class CourseOfAction(STIXDomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -112,7 +112,7 @@ class Identity(STIXDomainObject): ('identity_class', StringProperty(required=True)), ('sectors', ListProperty(StringProperty)), ('contact_information', StringProperty()), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -139,7 +139,7 @@ class Indicator(STIXDomainObject): ('valid_from', TimestampProperty(default=lambda: NOW)), ('valid_until', TimestampProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -169,7 +169,7 @@ class IntrusionSet(STIXDomainObject): ('resource_level', StringProperty()), ('primary_motivation', StringProperty()), ('secondary_motivations', ListProperty(StringProperty)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -193,7 +193,7 @@ class Malware(STIXDomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -218,7 +218,7 @@ class ObservedData(STIXDomainObject): ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(required=True)), ('objects', ObservableProperty(required=True)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -243,7 +243,7 @@ class Report(STIXDomainObject): ('description', StringProperty()), ('published', TimestampProperty(required=True)), ('object_refs', ListProperty(ReferenceProperty, required=True)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -274,7 +274,7 @@ class ThreatActor(STIXDomainObject): ('primary_motivation', StringProperty()), ('secondary_motivations', ListProperty(StringProperty)), ('personal_motivations', ListProperty(StringProperty)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -299,7 +299,7 @@ class Tool(STIXDomainObject): ('description', StringProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), ('tool_version', StringProperty()), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -322,7 +322,7 @@ class Vulnerability(STIXDomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -382,7 +382,7 @@ def CustomObject(type='x-custom-type', properties=None): # This is to follow the general properties structure. _properties.update([ - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index 7d7d3ae..e488229 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -32,7 +32,7 @@ class Relationship(STIXRelationshipObject): ('description', StringProperty()), ('source_ref', ReferenceProperty(required=True)), ('target_ref', ReferenceProperty(required=True)), - ('revoked', BooleanProperty()), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), @@ -72,8 +72,8 @@ class Sighting(STIXRelationshipObject): ('sighting_of_ref', ReferenceProperty(required=True)), ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))), ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))), - ('summary', BooleanProperty()), - ('revoked', BooleanProperty()), + ('summary', BooleanProperty(default=lambda: False)), + ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),