From 3c17c9259c1d36e70a48e646e4381b051470d9ef Mon Sep 17 00:00:00 2001 From: Richard Piazza Date: Fri, 31 Mar 2017 15:52:27 -0400 Subject: [PATCH] Add Sighting object and data markings - Update ReferenceProperty to allow specifying a particular object type - Update ListProperty and add SelectorProperty - Add description to Relationship --- .gitignore | 3 + stix2/__init__.py | 3 +- stix2/base.py | 5 ++ stix2/common.py | 9 ++- stix2/markings.py | 76 +++++++++++++++++++++++ stix2/properties.py | 41 +++++++++++-- stix2/sdo.py | 4 +- stix2/sro.py | 42 +++++++++---- stix2/test/constants.py | 5 ++ stix2/test/test_indicator.py | 2 +- stix2/test/test_markings.py | 115 +++++++++++++++++++++++++++++++++++ stix2/test/test_report.py | 43 +++++++++++++ stix2/test/test_sighting.py | 81 ++++++++++++++++++++++++ 13 files changed, 403 insertions(+), 26 deletions(-) create mode 100644 stix2/markings.py create mode 100644 stix2/test/test_markings.py create mode 100644 stix2/test/test_sighting.py diff --git a/.gitignore b/.gitignore index e50675e..1824e34 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ docs/_build/ # PyBuilder target/ +#pycharm stuff +.idea/ + diff --git a/stix2/__init__.py b/stix2/__init__.py index 7762890..78a6b69 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -7,4 +7,5 @@ from .common import ExternalReference, KillChainPhase from .sdo import AttackPattern, Campaign, CourseOfAction, Identity, Indicator, \ IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, \ Vulnerability -from .sro import Relationship +from .sro import Relationship, Sighting +from .markings import MarkingDefinition, GranularMarking, StatementMarking, TLPMarking diff --git a/stix2/base.py b/stix2/base.py index a44d879..58c455a 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -70,6 +70,11 @@ class _STIXBase(collections.Mapping): self._inner = kwargs + if self.granular_markings: + for m in self.granular_markings: + # TODO: check selectors + pass + def __getitem__(self, key): return self._inner[key] diff --git a/stix2/common.py b/stix2/common.py index 7b4e3ea..6ad7052 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,7 +1,7 @@ """STIX 2 Common Data Types and Properties""" from .base import _STIXBase -from .properties import Property, BooleanProperty, ReferenceProperty +from .properties import Property, BooleanProperty, ReferenceProperty, ListProperty from .utils import NOW COMMON_PROPERTIES = { @@ -10,10 +10,9 @@ COMMON_PROPERTIES = { 'modified': Property(default=lambda: NOW), 'external_references': Property(), 'revoked': BooleanProperty(), - 'created_by_ref': ReferenceProperty(), - # TODO: - # - object_marking_refs - # - granular_markings + 'created_by_ref': ReferenceProperty(type="identity"), + 'object_marking_refs': ListProperty(ReferenceProperty, element_type="marking-definition"), + 'granular_markings': ListProperty(Property) } diff --git a/stix2/markings.py b/stix2/markings.py new file mode 100644 index 0000000..aea0caa --- /dev/null +++ b/stix2/markings.py @@ -0,0 +1,76 @@ +"""STIX 2.0 Marking Objects""" + +from .base import _STIXBase +from .properties import IDProperty, TypeProperty, ListProperty, ReferenceProperty, Property, SelectorProperty +from .utils import NOW + + +class MarkingDefinition(_STIXBase): + _type = 'marking-definition' + _properties = { + 'created': Property(default=lambda: NOW), + 'external_references': Property(), + 'created_by_ref': ReferenceProperty(type="identity"), + 'object_marking_refs': ListProperty(ReferenceProperty, element_type="marking-definition"), + 'granular_marking': ListProperty(Property, element_type="granular-marking"), + 'type': TypeProperty(_type), + 'id': IDProperty(_type), + 'definition_type': Property(), + 'definition': Property(), + } + + +class GranularMarking(_STIXBase): + _properties = { + 'marking_ref': ReferenceProperty(required=True, type="marking-definition"), + 'selectors': ListProperty(SelectorProperty, required=True), + } + + +class TLPMarking(_STIXBase): + # TODO: don't allow the creation of any other TLPMarkings than the ones below + _properties = { + 'tlp': Property(required=True) + } + + +class StatementMarking(_STIXBase): + _properties = { + 'statement': Property(required=True) + } + + def __init__(self, statement=None, **kwargs): + # Allow statement as positional args. + if statement and not kwargs.get('statement'): + kwargs['statement'] = statement + + super(StatementMarking, self).__init__(**kwargs) + + +TLP_WHITE = MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="white") +) + +TLP_GREEN = MarkingDefinition( + id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="green") +) + +TLP_AMBER = MarkingDefinition( + id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="amber") +) + +TLP_RED = MarkingDefinition( + id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="red") +) diff --git a/stix2/properties.py b/stix2/properties.py index 3bb2633..03bae9c 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -1,5 +1,6 @@ import re import uuid +from .base import _STIXBase class Property(object): @@ -50,8 +51,9 @@ class Property(object): raise ValueError("must equal '{0}'.".format(self._fixed_value)) return value - def __init__(self, required=False, fixed=None, clean=None, default=None): + def __init__(self, required=False, fixed=None, clean=None, default=None, type=None): self.required = required + self.type = type if fixed: self._fixed_value = fixed self.validate = self._default_validate @@ -72,25 +74,28 @@ class Property(object): return value -class List(Property): +class ListProperty(Property): - def __init__(self, contained): + def __init__(self, contained, required=False, element_type=None): """ contained should be a type whose constructor creates an object from the value """ self.contained = contained + self.element_type = element_type + super(ListProperty, self).__init__(required) def validate(self, value): # TODO: ensure iterable + result = [] for item in value: - self.contained.validate(item) + result.append(self.contained(type=self.element_type).validate(item)) + return result def clean(self, value): return [self.contained(x) for x in value] class TypeProperty(Property): - def __init__(self, type): super(TypeProperty, self).__init__(fixed=type) @@ -125,9 +130,33 @@ REF_REGEX = re.compile("^[a-z][a-z-]+[a-z]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}" class ReferenceProperty(Property): - # TODO: support references that must be to a specific object type + def __init__(self, required=False, type=None): + """ + references sometimes must be to a specific object type + """ + self.type = type + super(ReferenceProperty, self).__init__(required, type=type) def validate(self, value): + if isinstance(value, _STIXBase): + value = value.id + if self.type: + if not value.startswith(self.type): + raise ValueError("must start with '{0}'.".format(self.type)) if not REF_REGEX.match(value): raise ValueError("must match --.") return value + + +SELECTOR_REGEX = re.compile("^[a-z0-9_-]{3,250}(\\.(\\[\\d+\\]|[a-z0-9_-]{1,250}))*$") + + +class SelectorProperty(Property): + def __init__(self, type=None): + # ignore type + super(SelectorProperty, self).__init__() + + def validate(self, value): + if not SELECTOR_REGEX.match(value): + raise ValueError("values must adhere to selector syntax") + return value diff --git a/stix2/sdo.py b/stix2/sdo.py index ffe073d..978840f 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -2,7 +2,7 @@ from .base import _STIXBase from .common import COMMON_PROPERTIES -from .properties import IDProperty, TypeProperty, Property +from .properties import IDProperty, TypeProperty, ListProperty, ReferenceProperty, Property from .utils import NOW @@ -137,7 +137,7 @@ class Report(_STIXBase): 'name': Property(required=True), 'description': Property(), 'published': Property(), - 'object_refs': Property(), + 'object_refs': ListProperty(ReferenceProperty), }) diff --git a/stix2/sro.py b/stix2/sro.py index 19751fb..4fcae6c 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -2,7 +2,7 @@ from .base import _STIXBase from .common import COMMON_PROPERTIES -from .properties import IDProperty, TypeProperty, Property +from .properties import IDProperty, TypeProperty, ReferenceProperty, ListProperty, Property class Relationship(_STIXBase): @@ -13,8 +13,9 @@ class Relationship(_STIXBase): 'id': IDProperty(_type), 'type': TypeProperty(_type), 'relationship_type': Property(required=True), - 'source_ref': Property(required=True), - 'target_ref': Property(required=True), + 'description': Property(), + 'source_ref': ReferenceProperty(required=True), + 'target_ref': ReferenceProperty(required=True), }) # Explicitly define the first three kwargs to make readable Relationship declarations. @@ -31,12 +32,31 @@ class Relationship(_STIXBase): if target_ref and not kwargs.get('target_ref'): kwargs['target_ref'] = target_ref - # If actual STIX objects (vs. just the IDs) are passed in, extract the - # ID values to use in the Relationship object. - if kwargs.get('source_ref') and isinstance(kwargs['source_ref'], _STIXBase): - kwargs['source_ref'] = kwargs['source_ref'].id - - if kwargs.get('target_ref') and isinstance(kwargs['target_ref'], _STIXBase): - kwargs['target_ref'] = kwargs['target_ref'].id - super(Relationship, self).__init__(**kwargs) + + +class Sighting(_STIXBase): + _type = 'sighting' + _properties = COMMON_PROPERTIES.copy() + _properties.update({ + 'id': IDProperty(_type), + 'type': TypeProperty(_type), + 'first_seen': Property(), + 'last_seen': Property(), + 'count': Property(), + 'sighting_of_ref': ReferenceProperty(required=True), + 'observed_data_refs': ListProperty(ReferenceProperty, element_type="observed-data"), + 'where_sighted_refs': ListProperty(ReferenceProperty, element_type="identity"), + 'summary': Property(), + }) + + # Explicitly define the first kwargs to make readable Sighting declarations. + def __init__(self, sighting_of_ref=None, **kwargs): + # TODO: + # - description + + # Allow sighting_of_ref as a positional arg. + if sighting_of_ref and not kwargs.get('sighting_of_ref'): + kwargs['sighting_of_ref'] = sighting_of_ref + + super(Sighting, self).__init__(**kwargs) diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 1fc09cb..6d88a84 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -8,6 +8,7 @@ INDICATOR_ID = "indicator--01234567-89ab-cdef-0123-456789abcdef" MALWARE_ID = "malware--fedcba98-7654-3210-fedc-ba9876543210" RELATIONSHIP_ID = "relationship--00000000-1111-2222-3333-444444444444" IDENTITY_ID = "identity--d4d765ce-cff7-40e8-b7a6-e205d005ac2c" +SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb" # Minimum required args for an Indicator instance INDICATOR_KWARGS = dict( @@ -27,3 +28,7 @@ RELATIONSHIP_KWARGS = dict( source_ref=INDICATOR_ID, target_ref=MALWARE_ID, ) + +SIGHTING_KWARGS = dict( + sighting_of_ref=INDICATOR_ID, +) diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index d6d92cd..542475f 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -95,7 +95,7 @@ def test_indicator_required_field_pattern(): def test_indicator_created_ref_invalid_format(): with pytest.raises(ValueError) as excinfo: stix2.Indicator(created_by_ref='myprefix--12345678', **INDICATOR_KWARGS) - assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must match --." + assert str(excinfo.value) == "Invalid value for Indicator 'created_by_ref': must start with 'identity'." def test_indicator_revoked_invalid(): diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py new file mode 100644 index 0000000..fd9b376 --- /dev/null +++ b/stix2/test/test_markings.py @@ -0,0 +1,115 @@ +import stix2 +from stix2.markings import TLP_WHITE +import pytest + +EXPECTED_TLP_MARKING_DEFINITION = """{ + "created": "2017-01-20T00:00:00.000Z", + "definition": { + "tlp": "white" + }, + "definition_type": "tlp", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "type": "marking-definition" +}""" + +EXPECTED_STATEMENT_MARKING_DEFINITION = """{ + "created": "2017-01-20T00:00:00.000Z", + "definition": { + "statement": "Copyright 2016, Example Corp" + }, + "definition_type": "statement", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "type": "marking-definition" +}""" + +EXPECTED_GRANULAR_MARKING = """{ + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "selectors": [ + "abc", + "abc.[23]", + "abc.def", + "abc.[2].efg" + ] +}""" + +EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ + "created": "2016-04-06T20:03:00.000Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "granular_markings": [ + { + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "selectors": [ + "description" + ] + } + ], + "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "modified": "2016-04-06T20:03:00.000Z", + "name": "Green Group Attacks Against Finance", + "type": "campaign" +}""" + + +def test_marking_def_example_with_tlp(): + assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION + + +def test_marking_def_example_with_statement(): + marking_definition = stix2.MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="statement", + definition=stix2.StatementMarking(statement="Copyright 2016, Example Corp") + ) + + assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + + +def test_marking_def_example_with_positional_statement(): + marking_definition = stix2.MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="statement", + definition=stix2.StatementMarking("Copyright 2016, Example Corp") + ) + + assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION + + +def test_granular_example(): + granular_marking = stix2.GranularMarking( + marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + selectors=["abc", "abc.[23]", "abc.def", "abc.[2].efg"] + ) + + assert str(granular_marking) == EXPECTED_GRANULAR_MARKING + + +def test_granular_example_with_bad_selector(): + with pytest.raises(ValueError) as excinfo: + stix2.GranularMarking( + marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + selectors=["abc[0]"] # missing "." + ) + + assert str(excinfo.value) == "Invalid value for GranularMarking 'selectors': values must adhere to selector syntax" + + +def test_campaign_with_granular_markings_example(): + campaign = stix2.Campaign( + id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T20:03:00.000Z", + modified="2016-04-06T20:03:00.000Z", + name="Green Group Attacks Against Finance", + description="Campaign by Green Group against a series of targets in the financial services sector.", + granular_markings=[ + stix2.GranularMarking( + marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + selectors=["description"]) + ]) + print(str(campaign)) + assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS + +# TODO: Add other examples diff --git a/stix2/test/test_report.py b/stix2/test/test_report.py index acd4e30..8f7249e 100644 --- a/stix2/test/test_report.py +++ b/stix2/test/test_report.py @@ -1,4 +1,6 @@ import stix2 +import pytest +from .constants import INDICATOR_KWARGS EXPECTED = """{ "created": "2015-12-21T19:59:11.000Z", @@ -39,4 +41,45 @@ def test_report_example(): assert str(report) == EXPECTED + +def test_report_example_objects_in_object_refs(): + report = stix2.Report( + id="report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3", + created_by_ref="identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", + created="2015-12-21T19:59:11.000Z", + modified="2015-12-21T19:59:11.000Z", + name="The Black Vine Cyberespionage Group", + description="A simple report with an indicator and campaign", + published="2016-01-201T17:00:00Z", + labels=["campaign"], + object_refs=[ + stix2.Indicator(id="indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", **INDICATOR_KWARGS), + "campaign--83422c77-904c-4dc1-aff5-5c38f3a2c55c", + "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a" + ], + ) + + assert str(report) == EXPECTED + + +def test_report_example_objects_in_object_refs_with_bad_id(): + with pytest.raises(ValueError) as excinfo: + stix2.Report( + id="report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3", + created_by_ref="identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", + created="2015-12-21T19:59:11.000Z", + modified="2015-12-21T19:59:11.000Z", + name="The Black Vine Cyberespionage Group", + description="A simple report with an indicator and campaign", + published="2016-01-201T17:00:00Z", + labels=["campaign"], + object_refs=[ + stix2.Indicator(id="indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", **INDICATOR_KWARGS), + "campaign-83422c77-904c-4dc1-aff5-5c38f3a2c55c", # the "bad" id, missing a "-" + "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a" + ], + ) + + assert str(excinfo.value) == "Invalid value for Report 'object_refs': must match --." + # TODO: Add other examples diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py new file mode 100644 index 0000000..ad30c63 --- /dev/null +++ b/stix2/test/test_sighting.py @@ -0,0 +1,81 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS + + +EXPECTED_SIGHTING = """{ + "created": "2016-04-06T20:06:37Z", + "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb", + "modified": "2016-04-06T20:06:37Z", + "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "type": "sighting", + "where_sighted_refs": [ + "identity--8cc7afd6-5455-4d2b-a736-e614ee631d99" + ] +}""" + +BAD_SIGHTING = """{ + "created": "2016-04-06T20:06:37Z", + "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb", + "modified": "2016-04-06T20:06:37Z", + "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "type": "sighting", + "where_sighted_refs": [ + "malware--8cc7afd6-5455-4d2b-a736-e614ee631d99" + ] +}""" + + +def test_sighting_all_required_fields(): + now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc) + + s = stix2.Sighting( + type='sighting', + id=SIGHTING_ID, + created=now, + modified=now, + sighting_of_ref=INDICATOR_ID, + where_sighted_refs=["identity--8cc7afd6-5455-4d2b-a736-e614ee631d99"] + ) + assert str(s) == EXPECTED_SIGHTING + + +def test_sighting_bad_where_sighted_refs(): + now = dt.datetime(2016, 4, 6, 20, 6, 37, tzinfo=pytz.utc) + + with pytest.raises(ValueError) as excinfo: + stix2.Sighting( + type='sighting', + id=SIGHTING_ID, + created=now, + modified=now, + sighting_of_ref=INDICATOR_ID, + where_sighted_refs=["malware--8cc7afd6-5455-4d2b-a736-e614ee631d99"] + ) + + assert str(excinfo.value) == "Invalid value for Sighting 'where_sighted_refs': must start with 'identity'." + + +def test_sighting_type_must_be_sightings(): + with pytest.raises(ValueError) as excinfo: + stix2.Sighting(type='xxx', **SIGHTING_KWARGS) + + assert str(excinfo.value) == "Invalid value for Sighting 'type': must equal 'sighting'." + + +def test_invalid_kwarg_to_sighting(): + with pytest.raises(TypeError) as excinfo: + stix2.Sighting(my_custom_property="foo", **SIGHTING_KWARGS) + assert str(excinfo.value) == "unexpected keyword arguments: ['my_custom_property']" in str(excinfo) + + +def test_create_sighting_from_objects_rather_than_ids(malware): # noqa: F811 + rel = stix2.Sighting(sighting_of_ref=malware) + + assert rel.sighting_of_ref == 'malware--00000000-0000-0000-0000-000000000001' + assert rel.id == 'sighting--00000000-0000-0000-0000-000000000002'