Add Sighting object and data markings

- Update ReferenceProperty to allow specifying a particular object type
- Update ListProperty and add SelectorProperty
- Add description to Relationship

.gitignore

@@ -57,3 +57,6 @@ docs/_build/
 # PyBuilder
 target/
 
+#pycharm stuff
+.idea/
+

stix2/__init__.py

@@ -7,4 +7,5 @@ from .common import ExternalReference, KillChainPhase
 from .sdo import AttackPattern, Campaign, CourseOfAction, Identity, Indicator, \
     IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, \
     Vulnerability
-from .sro import Relationship
+from .sro import Relationship, Sighting
+from .markings import MarkingDefinition, GranularMarking, StatementMarking, TLPMarking

stix2/base.py

@@ -70,6 +70,11 @@ class _STIXBase(collections.Mapping):
 
         self._inner = kwargs
 
+        if self.granular_markings:
+            for m in self.granular_markings:
+                # TODO: check selectors
+                pass
+
     def __getitem__(self, key):
         return self._inner[key]
 

stix2/common.py

@@ -1,7 +1,7 @@
 """STIX 2 Common Data Types and Properties"""
 
 from .base import _STIXBase
-from .properties import Property, BooleanProperty, ReferenceProperty
+from .properties import Property, BooleanProperty, ReferenceProperty, ListProperty
 from .utils import NOW
 
 COMMON_PROPERTIES = {
@@ -10,10 +10,9 @@ COMMON_PROPERTIES = {
     'modified': Property(default=lambda: NOW),
     'external_references': Property(),
     'revoked': BooleanProperty(),
-    'created_by_ref': ReferenceProperty(),
+    'created_by_ref': ReferenceProperty(type="identity"),
-    # TODO:
+    'object_marking_refs': ListProperty(ReferenceProperty, element_type="marking-definition"),
-    # - object_marking_refs
+    'granular_markings': ListProperty(Property)
-    # - granular_markings
 }
 
 

stix2/markings.py

@@ -0,0 +1,76 @@
+"""STIX 2.0 Marking Objects"""
+
+from .base import _STIXBase
+from .properties import IDProperty, TypeProperty, ListProperty, ReferenceProperty, Property, SelectorProperty
+from .utils import NOW
+
+
+class MarkingDefinition(_STIXBase):
+    _type = 'marking-definition'
+    _properties = {
+        'created': Property(default=lambda: NOW),
+        'external_references': Property(),
+        'created_by_ref': ReferenceProperty(type="identity"),
+        'object_marking_refs': ListProperty(ReferenceProperty, element_type="marking-definition"),
+        'granular_marking': ListProperty(Property, element_type="granular-marking"),
+        'type': TypeProperty(_type),
+        'id': IDProperty(_type),
+        'definition_type': Property(),
+        'definition': Property(),
+    }
+
+
+class GranularMarking(_STIXBase):
+    _properties = {
+        'marking_ref': ReferenceProperty(required=True, type="marking-definition"),
+        'selectors': ListProperty(SelectorProperty, required=True),
+    }
+
+
+class TLPMarking(_STIXBase):
+    # TODO: don't allow the creation of any other TLPMarkings than the ones below
+    _properties = {
+        'tlp': Property(required=True)
+    }
+
+
+class StatementMarking(_STIXBase):
+    _properties = {
+        'statement': Property(required=True)
+    }
+
+    def __init__(self, statement=None, **kwargs):
+        # Allow statement as positional args.
+        if statement and not kwargs.get('statement'):
+            kwargs['statement'] = statement
+
+        super(StatementMarking, self).__init__(**kwargs)
+
+
+TLP_WHITE = MarkingDefinition(
+                id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+                created="2017-01-20T00:00:00.000Z",
+                definition_type="tlp",
+                definition=TLPMarking(tlp="white")
+)
+
+TLP_GREEN = MarkingDefinition(
+                id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da",
+                created="2017-01-20T00:00:00.000Z",
+                definition_type="tlp",
+                definition=TLPMarking(tlp="green")
+)
+
+TLP_AMBER = MarkingDefinition(
+                id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82",
+                created="2017-01-20T00:00:00.000Z",
+                definition_type="tlp",
+                definition=TLPMarking(tlp="amber")
+)
+
+TLP_RED = MarkingDefinition(
+                id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed",
+                created="2017-01-20T00:00:00.000Z",
+                definition_type="tlp",
+                definition=TLPMarking(tlp="red")
+)

stix2/properties.py

@@ -1,5 +1,6 @@
 import re
 import uuid
+from .base import _STIXBase
 
 
 class Property(object):
@@ -50,8 +51,9 @@ class Property(object):
             raise ValueError("must equal '{0}'.".format(self._fixed_value))
         return value
 
-    def __init__(self, required=False, fixed=None, clean=None, default=None):
+    def __init__(self, required=False, fixed=None, clean=None, default=None, type=None):
         self.required = required
+        self.type = type
         if fixed:
             self._fixed_value = fixed
             self.validate = self._default_validate
@@ -72,25 +74,28 @@ class Property(object):
         return value
 
 
-class List(Property):
+class ListProperty(Property):
 
-    def __init__(self, contained):
+    def __init__(self, contained, required=False, element_type=None):
         """
         contained should be a type whose constructor creates an object from the value
         """
         self.contained = contained
+        self.element_type = element_type
+        super(ListProperty, self).__init__(required)
 
     def validate(self, value):
         # TODO: ensure iterable
+        result = []
         for item in value:
-            self.contained.validate(item)
+            result.append(self.contained(type=self.element_type).validate(item))
+        return result
 
     def clean(self, value):
         return [self.contained(x) for x in value]
 
 
 class TypeProperty(Property):
-
     def __init__(self, type):
         super(TypeProperty, self).__init__(fixed=type)
 
@@ -125,9 +130,33 @@ REF_REGEX = re.compile("^[a-z][a-z-]+[a-z]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}"
 
 
 class ReferenceProperty(Property):
-    # TODO: support references that must be to a specific object type
+    def __init__(self, required=False, type=None):
+        """
+        references sometimes must be to a specific object type
+        """
+        self.type = type
+        super(ReferenceProperty, self).__init__(required, type=type)
 
     def validate(self, value):
+        if isinstance(value, _STIXBase):
+            value = value.id
+        if self.type:
+            if not value.startswith(self.type):
+                raise ValueError("must start with '{0}'.".format(self.type))
         if not REF_REGEX.match(value):
             raise ValueError("must match <object-type>--<guid>.")
         return value
+
+
+SELECTOR_REGEX = re.compile("^[a-z0-9_-]{3,250}(\\.(\\[\\d+\\]|[a-z0-9_-]{1,250}))*$")
+
+
+class SelectorProperty(Property):
+    def __init__(self, type=None):
+        # ignore type
+        super(SelectorProperty, self).__init__()
+
+    def validate(self, value):
+        if not SELECTOR_REGEX.match(value):
+            raise ValueError("values must adhere to selector syntax")
+        return value

stix2/sdo.py

@@ -2,7 +2,7 @@
 
 from .base import _STIXBase
 from .common import COMMON_PROPERTIES
-from .properties import IDProperty, TypeProperty, Property
+from .properties import IDProperty, TypeProperty, ListProperty, ReferenceProperty, Property
 from .utils import NOW
 
 
@@ -137,7 +137,7 @@ class Report(_STIXBase):
         'name': Property(required=True),
         'description': Property(),
         'published': Property(),
-        'object_refs': Property(),
+        'object_refs': ListProperty(ReferenceProperty),
     })
 
 

stix2/sro.py

@@ -2,7 +2,7 @@
 
 from .base import _STIXBase
 from .common import COMMON_PROPERTIES
-from .properties import IDProperty, TypeProperty, Property
+from .properties import IDProperty, TypeProperty, ReferenceProperty, ListProperty, Property
 
 
 class Relationship(_STIXBase):
@@ -13,8 +13,9 @@ class Relationship(_STIXBase):
         'id': IDProperty(_type),
         'type': TypeProperty(_type),
         'relationship_type': Property(required=True),
-        'source_ref': Property(required=True),
+        'description': Property(),
-        'target_ref': Property(required=True),
+        'source_ref': ReferenceProperty(required=True),
+        'target_ref': ReferenceProperty(required=True),
     })
 
     # Explicitly define the first three kwargs to make readable Relationship declarations.
@@ -31,12 +32,31 @@ class Relationship(_STIXBase):
         if target_ref and not kwargs.get('target_ref'):
             kwargs['target_ref'] = target_ref
 
-        # If actual STIX objects (vs. just the IDs) are passed in, extract the
-        # ID values to use in the Relationship object.
-        if kwargs.get('source_ref') and isinstance(kwargs['source_ref'], _STIXBase):
-            kwargs['source_ref'] = kwargs['source_ref'].id
-
-        if kwargs.get('target_ref') and isinstance(kwargs['target_ref'], _STIXBase):
-            kwargs['target_ref'] = kwargs['target_ref'].id
-
         super(Relationship, self).__init__(**kwargs)
+
+
+class Sighting(_STIXBase):
+    _type = 'sighting'
+    _properties = COMMON_PROPERTIES.copy()
+    _properties.update({
+        'id': IDProperty(_type),
+        'type': TypeProperty(_type),
+        'first_seen': Property(),
+        'last_seen': Property(),
+        'count': Property(),
+        'sighting_of_ref': ReferenceProperty(required=True),
+        'observed_data_refs': ListProperty(ReferenceProperty, element_type="observed-data"),
+        'where_sighted_refs': ListProperty(ReferenceProperty, element_type="identity"),
+        'summary': Property(),
+    })
+
+    # Explicitly define the first kwargs to make readable Sighting declarations.
+    def __init__(self, sighting_of_ref=None, **kwargs):
+        # TODO:
+        # - description
+
+        # Allow sighting_of_ref as a positional arg.
+        if sighting_of_ref and not kwargs.get('sighting_of_ref'):
+            kwargs['sighting_of_ref'] = sighting_of_ref
+
+        super(Sighting, self).__init__(**kwargs)

stix2/test/constants.py

@@ -8,6 +8,7 @@ INDICATOR_ID = "indicator--01234567-89ab-cdef-0123-456789abcdef"
 MALWARE_ID = "malware--fedcba98-7654-3210-fedc-ba9876543210"
 RELATIONSHIP_ID = "relationship--00000000-1111-2222-3333-444444444444"
 IDENTITY_ID = "identity--d4d765ce-cff7-40e8-b7a6-e205d005ac2c"
+SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb"
 
 # Minimum required args for an Indicator instance
 INDICATOR_KWARGS = dict(
@@ -27,3 +28,7 @@ RELATIONSHIP_KWARGS = dict(
     source_ref=INDICATOR_ID,
     target_ref=MALWARE_ID,
 )
+
+SIGHTING_KWARGS = dict(
+    sighting_of_ref=INDICATOR_ID,
+)

stix2/test/test_indicator.py

@@ -95,7 +95,7 @@ def test_indicator_required_field_pattern():
 def test_indicator_created_ref_invalid_format():
     with pytest.raises(ValueError) as excinfo:
         stix2.Indicator(created_by_ref='myprefix--12345678', **INDICATOR_KWARGS)
-    assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must match <object-type>--<guid>."
+    assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must start with 'identity'."
 
 
 def test_indicator_revoked_invalid():

stix2/test/test_markings.py

@@ -0,0 +1,115 @@
+import stix2
+from stix2.markings import TLP_WHITE
+import pytest
+
+EXPECTED_TLP_MARKING_DEFINITION = """{
+    "created": "2017-01-20T00:00:00.000Z",
+    "definition": {
+        "tlp": "white"
+    },
+    "definition_type": "tlp",
+    "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+    "type": "marking-definition"
+}"""
+
+EXPECTED_STATEMENT_MARKING_DEFINITION = """{
+    "created": "2017-01-20T00:00:00.000Z",
+    "definition": {
+        "statement": "Copyright 2016, Example Corp"
+    },
+    "definition_type": "statement",
+    "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+    "type": "marking-definition"
+}"""
+
+EXPECTED_GRANULAR_MARKING = """{
+    "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+    "selectors": [
+        "abc",
+        "abc.[23]",
+        "abc.def",
+        "abc.[2].efg"
+    ]
+}"""
+
+EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{
+    "created": "2016-04-06T20:03:00.000Z",
+    "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+    "description": "Campaign by Green Group against a series of targets in the financial services sector.",
+    "granular_markings": [
+        {
+            "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+            "selectors": [
+                "description"
+            ]
+        }
+    ],
+    "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+    "modified": "2016-04-06T20:03:00.000Z",
+    "name": "Green Group Attacks Against Finance",
+    "type": "campaign"
+}"""
+
+
+def test_marking_def_example_with_tlp():
+    assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION
+
+
+def test_marking_def_example_with_statement():
+    marking_definition = stix2.MarkingDefinition(
+        id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+        created="2017-01-20T00:00:00.000Z",
+        definition_type="statement",
+        definition=stix2.StatementMarking(statement="Copyright 2016, Example Corp")
+    )
+
+    assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION
+
+
+def test_marking_def_example_with_positional_statement():
+    marking_definition = stix2.MarkingDefinition(
+        id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+        created="2017-01-20T00:00:00.000Z",
+        definition_type="statement",
+        definition=stix2.StatementMarking("Copyright 2016, Example Corp")
+    )
+
+    assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION
+
+
+def test_granular_example():
+    granular_marking = stix2.GranularMarking(
+        marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+        selectors=["abc", "abc.[23]", "abc.def", "abc.[2].efg"]
+    )
+
+    assert str(granular_marking) == EXPECTED_GRANULAR_MARKING
+
+
+def test_granular_example_with_bad_selector():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.GranularMarking(
+            marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+            selectors=["abc[0]"]   # missing "."
+        )
+
+    assert str(excinfo.value) == "Invalid value for GranularMarking 'selectors': values must adhere to selector syntax"
+
+
+def test_campaign_with_granular_markings_example():
+    campaign = stix2.Campaign(
+        id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+        created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+        created="2016-04-06T20:03:00.000Z",
+        modified="2016-04-06T20:03:00.000Z",
+        name="Green Group Attacks Against Finance",
+        description="Campaign by Green Group against a series of targets in the financial services sector.",
+        granular_markings=[
+            stix2.GranularMarking(
+                marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
+                selectors=["description"])
+            ])
+    print(str(campaign))
+    assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS
+
+# TODO: Add other examples

stix2/test/test_report.py

@@ -1,4 +1,6 @@
 import stix2
+import pytest
+from .constants import INDICATOR_KWARGS
 
 EXPECTED = """{
     "created": "2015-12-21T19:59:11.000Z",
@@ -39,4 +41,45 @@ def test_report_example():
 
     assert str(report) == EXPECTED
 
+
+def test_report_example_objects_in_object_refs():
+    report = stix2.Report(
+        id="report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3",
+        created_by_ref="identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283",
+        created="2015-12-21T19:59:11.000Z",
+        modified="2015-12-21T19:59:11.000Z",
+        name="The Black Vine Cyberespionage Group",
+        description="A simple report with an indicator and campaign",
+        published="2016-01-201T17:00:00Z",
+        labels=["campaign"],
+        object_refs=[
+            stix2.Indicator(id="indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", **INDICATOR_KWARGS),
+            "campaign--83422c77-904c-4dc1-aff5-5c38f3a2c55c",
+            "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a"
+        ],
+    )
+
+    assert str(report) == EXPECTED
+
+
+def test_report_example_objects_in_object_refs_with_bad_id():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.Report(
+            id="report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3",
+            created_by_ref="identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283",
+            created="2015-12-21T19:59:11.000Z",
+            modified="2015-12-21T19:59:11.000Z",
+            name="The Black Vine Cyberespionage Group",
+            description="A simple report with an indicator and campaign",
+            published="2016-01-201T17:00:00Z",
+            labels=["campaign"],
+            object_refs=[
+                stix2.Indicator(id="indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", **INDICATOR_KWARGS),
+                "campaign-83422c77-904c-4dc1-aff5-5c38f3a2c55c",   # the "bad" id, missing a "-"
+                "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a"
+            ],
+        )
+
+    assert str(excinfo.value) == "Invalid value for Report 'object_refs': must match <object-type>--<guid>."
+
 # TODO: Add other examples

stix2/test/test_sighting.py

@@ -0,0 +1,81 @@
+import datetime as dt
+
+import pytest
+import pytz
+
+import stix2
+
+from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS
+
+
+EXPECTED_SIGHTING = """{
+    "created": "2016-04-06T20:06:37Z",
+    "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb",
+    "modified": "2016-04-06T20:06:37Z",
+    "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef",
+    "type": "sighting",
+    "where_sighted_refs": [
+        "identity--8cc7afd6-5455-4d2b-a736-e614ee631d99"
+    ]
+}"""
+
+BAD_SIGHTING = """{
+    "created": "2016-04-06T20:06:37Z",
+    "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb",
+    "modified": "2016-04-06T20:06:37Z",
+    "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef",
+    "type": "sighting",
+    "where_sighted_refs": [
+        "malware--8cc7afd6-5455-4d2b-a736-e614ee631d99"
+    ]
+}"""
+
+
+def test_sighting_all_required_fields():
+    now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc)
+
+    s = stix2.Sighting(
+        type='sighting',
+        id=SIGHTING_ID,
+        created=now,
+        modified=now,
+        sighting_of_ref=INDICATOR_ID,
+        where_sighted_refs=["identity--8cc7afd6-5455-4d2b-a736-e614ee631d99"]
+    )
+    assert str(s) == EXPECTED_SIGHTING
+
+
+def test_sighting_bad_where_sighted_refs():
+    now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc)
+
+    with pytest.raises(ValueError) as excinfo:
+        stix2.Sighting(
+            type='sighting',
+            id=SIGHTING_ID,
+            created=now,
+            modified=now,
+            sighting_of_ref=INDICATOR_ID,
+            where_sighted_refs=["malware--8cc7afd6-5455-4d2b-a736-e614ee631d99"]
+        )
+
+    assert str(excinfo.value) == "Invalid value for Sighting 'where_sighted_refs': must start with 'identity'."
+
+
+def test_sighting_type_must_be_sightings():
+    with pytest.raises(ValueError) as excinfo:
+        stix2.Sighting(type='xxx', **SIGHTING_KWARGS)
+
+    assert str(excinfo.value) == "Invalid value for Sighting 'type': must equal 'sighting'."
+
+
+def test_invalid_kwarg_to_sighting():
+    with pytest.raises(TypeError) as excinfo:
+        stix2.Sighting(my_custom_property="foo", **SIGHTING_KWARGS)
+    assert str(excinfo.value) == "unexpected keyword arguments: ['my_custom_property']" in str(excinfo)
+
+
+def test_create_sighting_from_objects_rather_than_ids(malware):  # noqa: F811
+    rel = stix2.Sighting(sighting_of_ref=malware)
+
+    assert rel.sighting_of_ref == 'malware--00000000-0000-0000-0000-000000000001'
+    assert rel.id == 'sighting--00000000-0000-0000-0000-000000000002'