Merge pull request #168 from oasis-open/123-default-revoked

Provide default for revoked, sighting:summary.

stix2/base.py

@@ -22,6 +22,33 @@ DEFAULT_ERROR = "{type} must have {property}='{expected}'."
 
 
 class STIXJSONEncoder(json.JSONEncoder):
+    """Custom JSONEncoder subclass for serializing Python ``stix2`` objects.
+
+    If an optional property with a default value specified in the STIX 2 spec
+    is set to that default value, it will be left out of the serialized output.
+
+    An example of this type of property include the ``revoked`` common property.
+    """
+
+    def default(self, obj):
+        if isinstance(obj, (dt.date, dt.datetime)):
+            return format_datetime(obj)
+        elif isinstance(obj, _STIXBase):
+            tmp_obj = dict(copy.deepcopy(obj))
+            for prop_name in obj._defaulted_optional_properties:
+                del tmp_obj[prop_name]
+            return tmp_obj
+        else:
+            return super(STIXJSONEncoder, self).default(obj)
+
+
+class STIXJSONIncludeOptionalDefaultsEncoder(json.JSONEncoder):
+    """Custom JSONEncoder subclass for serializing Python ``stix2`` objects.
+
+    Differs from ``STIXJSONEncoder`` in that if an optional property with a default
+    value specified in the STIX 2 spec is set to that default value, it will be
+    included in the serialized output.
+    """
 
     def default(self, obj):
         if isinstance(obj, (dt.date, dt.datetime)):
@@ -122,14 +149,25 @@ class _STIXBase(collections.Mapping):
                 setting_kwargs[prop_name] = prop_value
 
         # Detect any missing required properties
-        required_properties = get_required_properties(cls._properties)
+        required_properties = set(get_required_properties(cls._properties))
-        missing_kwargs = set(required_properties) - set(setting_kwargs)
+        missing_kwargs = required_properties - set(setting_kwargs)
         if missing_kwargs:
             raise MissingPropertiesError(cls, missing_kwargs)
 
         for prop_name, prop_metadata in cls._properties.items():
             self._check_property(prop_name, prop_metadata, setting_kwargs)
 
+        # Cache defaulted optional properties for serialization
+        defaulted = []
+        for name, prop in cls._properties.items():
+            try:
+                if (not prop.required and not hasattr(prop, '_fixed_value') and
+                        prop.default() == setting_kwargs[name]):
+                    defaulted.append(name)
+            except (AttributeError, KeyError):
+                continue
+        self._defaulted_optional_properties = defaulted
+
         self._inner = setting_kwargs
 
         self._check_object_constraints()
@@ -151,7 +189,7 @@ class _STIXBase(collections.Mapping):
                              (self.__class__.__name__, name))
 
     def __setattr__(self, name, value):
-        if name != '_inner' and not name.startswith("_STIXBase__"):
+        if not name.startswith("_"):
             raise ImmutableError(self.__class__, name)
         super(_STIXBase, self).__setattr__(name, value)
 
@@ -170,6 +208,7 @@ class _STIXBase(collections.Mapping):
         if isinstance(self, _Observable):
             # Assume: valid references in the original object are still valid in the new version
             new_inner['_valid_refs'] = {'*': '*'}
+        new_inner['allow_custom'] = self.__allow_custom
         return cls(**new_inner)
 
     def properties_populated(self):
@@ -183,7 +222,7 @@ class _STIXBase(collections.Mapping):
     def revoke(self):
         return _revoke(self)
 
-    def serialize(self, pretty=False, **kwargs):
+    def serialize(self, pretty=False, include_optional_defaults=False, **kwargs):
         """
         Serialize a STIX object.
 
@@ -191,6 +230,8 @@ class _STIXBase(collections.Mapping):
             pretty (bool): If True, output properties following the STIX specs
                 formatting. This includes indentation. Refer to notes for more
                 details. (Default: ``False``)
+            include_optional_defaults (bool): Determines whether to include
+                optional properties set to the default value defined in the spec.
             **kwargs: The arguments for a json.dumps() call.
 
         Returns:
@@ -213,7 +254,10 @@ class _STIXBase(collections.Mapping):
 
             kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by})
 
-        return json.dumps(self, cls=STIXJSONEncoder, **kwargs)
+        if include_optional_defaults:
+            return json.dumps(self, cls=STIXJSONIncludeOptionalDefaultsEncoder, **kwargs)
+        else:
+            return json.dumps(self, cls=STIXJSONEncoder, **kwargs)
 
 
 class _Observable(_STIXBase):

stix2/test/test_bundle.py

@@ -6,7 +6,7 @@ import stix2
 
 EXPECTED_BUNDLE = """{
     "type": "bundle",
-    "id": "bundle--00000000-0000-0000-0000-000000000004",
+    "id": "bundle--00000000-0000-0000-0000-000000000007",
     "spec_version": "2.0",
     "objects": [
         {
@@ -22,7 +22,7 @@ EXPECTED_BUNDLE = """{
         },
         {
             "type": "malware",
-            "id": "malware--00000000-0000-0000-0000-000000000002",
+            "id": "malware--00000000-0000-0000-0000-000000000003",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "name": "Cryptolocker",
@@ -32,7 +32,7 @@ EXPECTED_BUNDLE = """{
         },
         {
             "type": "relationship",
-            "id": "relationship--00000000-0000-0000-0000-000000000003",
+            "id": "relationship--00000000-0000-0000-0000-000000000005",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "relationship_type": "indicates",
@@ -44,7 +44,7 @@ EXPECTED_BUNDLE = """{
 
 EXPECTED_BUNDLE_DICT = {
     "type": "bundle",
-    "id": "bundle--00000000-0000-0000-0000-000000000004",
+    "id": "bundle--00000000-0000-0000-0000-000000000007",
     "spec_version": "2.0",
     "objects": [
         {
@@ -60,7 +60,7 @@ EXPECTED_BUNDLE_DICT = {
         },
         {
             "type": "malware",
-            "id": "malware--00000000-0000-0000-0000-000000000002",
+            "id": "malware--00000000-0000-0000-0000-000000000003",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "name": "Cryptolocker",
@@ -70,7 +70,7 @@ EXPECTED_BUNDLE_DICT = {
         },
         {
             "type": "relationship",
-            "id": "relationship--00000000-0000-0000-0000-000000000003",
+            "id": "relationship--00000000-0000-0000-0000-000000000005",
             "created": "2017-01-01T12:34:56.000Z",
             "modified": "2017-01-01T12:34:56.000Z",
             "relationship_type": "indicates",

stix2/test/test_datastore.py

@@ -405,13 +405,11 @@ def test_apply_common_filters4():
 
 def test_apply_common_filters5():
     # "Return any object whose not revoked"
-    # Note that if 'revoked' property is not present in object.
-    # Currently we can't use such an expression to filter for... :(
     resp = list(apply_common_filters(stix_objs, [filters[5]]))
     assert len(resp) == 0
 
     resp = list(apply_common_filters(real_stix_objs, [filters[5]]))
-    assert len(resp) == 0
+    assert len(resp) == 4
 
 
 def test_apply_common_filters6():

stix2/test/test_indicator.py

@@ -45,6 +45,7 @@ def test_indicator_with_all_required_properties():
         labels=['malicious-activity'],
     )
 
+    assert ind.revoked is False
     assert str(ind) == EXPECTED_INDICATOR
     rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(ind))
     assert rep == EXPECTED_INDICATOR_REPR

stix2/test/test_relationship.py

@@ -123,8 +123,8 @@ def test_create_relationship_from_objects_rather_than_ids(indicator, malware):
 
     assert rel.relationship_type == 'indicates'
     assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001'
-    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002'
+    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003'
-    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003'
+    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005'
 
 
 def test_create_relationship_with_positional_args(indicator, malware):
@@ -132,8 +132,8 @@ def test_create_relationship_with_positional_args(indicator, malware):
 
     assert rel.relationship_type == 'indicates'
     assert rel.source_ref == 'indicator--00000000-0000-0000-0000-000000000001'
-    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000002'
+    assert rel.target_ref == 'malware--00000000-0000-0000-0000-000000000003'
-    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000003'
+    assert rel.id == 'relationship--00000000-0000-0000-0000-000000000005'
 
 
 @pytest.mark.parametrize("data", [

stix2/test/test_sighting.py

@@ -86,7 +86,7 @@ def test_create_sighting_from_objects_rather_than_ids(malware):  # noqa: F811
     rel = stix2.Sighting(sighting_of_ref=malware)
 
     assert rel.sighting_of_ref == 'malware--00000000-0000-0000-0000-000000000001'
-    assert rel.id == 'sighting--00000000-0000-0000-0000-000000000002'
+    assert rel.id == 'sighting--00000000-0000-0000-0000-000000000003'
 
 
 @pytest.mark.parametrize("data", [

stix2/test/test_tool.py

@@ -19,6 +19,19 @@ EXPECTED = """{
     ]
 }"""
 
+EXPECTED_WITH_REVOKED = """{
+    "type": "tool",
+    "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+    "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+    "created": "2016-04-06T20:03:48.000Z",
+    "modified": "2016-04-06T20:03:48.000Z",
+    "name": "VNC",
+    "revoked": false,
+    "labels": [
+        "remote-access"
+    ]
+}"""
+
 
 def test_tool_example():
     tool = stix2.Tool(
@@ -64,4 +77,18 @@ def test_tool_no_workbench_wrappers():
     with pytest.raises(AttributeError):
         tool.created_by()
 
+
+def test_tool_serialize_with_defaults():
+    tool = stix2.Tool(
+        id="tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+        created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        created="2016-04-06T20:03:48.000Z",
+        modified="2016-04-06T20:03:48.000Z",
+        name="VNC",
+        labels=["remote-access"],
+    )
+
+    assert tool.serialize(pretty=True, include_optional_defaults=True) == EXPECTED_WITH_REVOKED
+
+
 # TODO: Add other examples

stix2/v20/sdo.py

@@ -36,7 +36,7 @@ class AttackPattern(STIXDomainObject):
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -63,7 +63,7 @@ class Campaign(STIXDomainObject):
         ('first_seen', TimestampProperty()),
         ('last_seen', TimestampProperty()),
         ('objective', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -86,7 +86,7 @@ class CourseOfAction(STIXDomainObject):
         ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')),
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -112,7 +112,7 @@ class Identity(STIXDomainObject):
         ('identity_class', StringProperty(required=True)),
         ('sectors', ListProperty(StringProperty)),
         ('contact_information', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -139,7 +139,7 @@ class Indicator(STIXDomainObject):
         ('valid_from', TimestampProperty(default=lambda: NOW)),
         ('valid_until', TimestampProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -169,7 +169,7 @@ class IntrusionSet(STIXDomainObject):
         ('resource_level', StringProperty()),
         ('primary_motivation', StringProperty()),
         ('secondary_motivations', ListProperty(StringProperty)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -193,7 +193,7 @@ class Malware(STIXDomainObject):
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -218,7 +218,7 @@ class ObservedData(STIXDomainObject):
         ('last_observed', TimestampProperty(required=True)),
         ('number_observed', IntegerProperty(required=True)),
         ('objects', ObservableProperty(required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -243,7 +243,7 @@ class Report(STIXDomainObject):
         ('description', StringProperty()),
         ('published', TimestampProperty(required=True)),
         ('object_refs', ListProperty(ReferenceProperty, required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -274,7 +274,7 @@ class ThreatActor(STIXDomainObject):
         ('primary_motivation', StringProperty()),
         ('secondary_motivations', ListProperty(StringProperty)),
         ('personal_motivations', ListProperty(StringProperty)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -299,7 +299,7 @@ class Tool(STIXDomainObject):
         ('description', StringProperty()),
         ('kill_chain_phases', ListProperty(KillChainPhase)),
         ('tool_version', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty, required=True)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -322,7 +322,7 @@ class Vulnerability(STIXDomainObject):
         ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')),
         ('name', StringProperty(required=True)),
         ('description', StringProperty()),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -382,7 +382,7 @@ def CustomObject(type='x-custom-type', properties=None):
 
             # This is to follow the general properties structure.
             _properties.update([
-                ('revoked', BooleanProperty()),
+                ('revoked', BooleanProperty(default=lambda: False)),
                 ('labels', ListProperty(StringProperty)),
                 ('external_references', ListProperty(ExternalReference)),
                 ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),

stix2/v20/sro.py

@@ -32,7 +32,7 @@ class Relationship(STIXRelationshipObject):
         ('description', StringProperty()),
         ('source_ref', ReferenceProperty(required=True)),
         ('target_ref', ReferenceProperty(required=True)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),
@@ -72,8 +72,8 @@ class Sighting(STIXRelationshipObject):
         ('sighting_of_ref', ReferenceProperty(required=True)),
         ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))),
         ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))),
-        ('summary', BooleanProperty()),
+        ('summary', BooleanProperty(default=lambda: False)),
-        ('revoked', BooleanProperty()),
+        ('revoked', BooleanProperty(default=lambda: False)),
         ('labels', ListProperty(StringProperty)),
         ('external_references', ListProperty(ExternalReference)),
         ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))),