diff --git a/.gitignore b/.gitignore index 3b9971a..3b953e1 100644 --- a/.gitignore +++ b/.gitignore @@ -57,9 +57,12 @@ docs/_build/ # PyBuilder target/ +# External data cache +cache.sqlite + # Vim *.swp -# + # PyCharm .idea/ diff --git a/.isort.cfg b/.isort.cfg index 63f5b73..622e7a5 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests +known_third_party=ordereddict,dateutil,pytest,pytz,requests,simplejson,six,stix2patterns,stix2validator,taxii2client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/.travis.yml b/.travis.yml index fdbb686..aba764d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ sudo: false language: python cache: pip python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/README.rst b/README.rst index b2a8c9c..03c7cc3 100644 --- a/README.rst +++ b/README.rst @@ -39,8 +39,8 @@ constructor: from stix2 import Indicator indicator = Indicator(name="File hash for malware variant", - labels=['malicious-activity'], - pattern='file:hashes.md5 = "d41d8cd98f00b204e9800998ecf8427e"') + labels=["malicious-activity"], + pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") Certain required attributes of all objects will be set automatically if not provided as keyword arguments: diff --git a/setup.py b/setup.py index efe754c..e359147 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ def get_version(): with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() + setup( name='stix2', version=get_version(), @@ -46,9 +47,13 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'pytz', - 'six', 'python-dateutil', + 'pytz', 'requests', + 'simplejson', + 'six', + 'stix2-patterns', + 'stix2-validator', + 'taxii2-client', ], ) diff --git a/stix2/__init__.py b/stix2/__init__.py index 503194c..c2aae2e 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,7 +3,10 @@ # flake8: noqa from . import exceptions -from .bundle import Bundle +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) +from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomObservable, Directory, @@ -18,9 +21,6 @@ from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, WindowsRegistryValueType, WindowsServiceExt, X509Certificate, X509V3ExtenstionsType, parse_observable) -from .other import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) from .patterns import (AndBooleanExpression, AndObservationExpression, BasicObjectPathComponent, EqualityComparisonExpression, FloatConstant, FollowedByObservationExpression, @@ -37,59 +37,10 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, OrObservationExpression, ParentheticalExpression, QualifiedObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, - StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) + StartStopQualifier, StringConstant, WithinQualifier) from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, Vulnerability) from .sro import Relationship, Sighting -from .utils import get_dict +from .utils import get_dict, new_version, revoke from .version import __version__ - -OBJ_MAP = { - 'attack-pattern': AttackPattern, - 'bundle': Bundle, - 'campaign': Campaign, - 'course-of-action': CourseOfAction, - 'identity': Identity, - 'indicator': Indicator, - 'intrusion-set': IntrusionSet, - 'malware': Malware, - 'marking-definition': MarkingDefinition, - 'observed-data': ObservedData, - 'report': Report, - 'relationship': Relationship, - 'threat-actor': ThreatActor, - 'tool': Tool, - 'sighting': Sighting, - 'vulnerability': Vulnerability, -} - - -def parse(data, allow_custom=False): - """Deserialize a string or file-like object into a STIX object. - - Args: - data: The STIX 2 string to be parsed. - allow_custom (bool): Whether to allow custom properties or not. Default: False. - - Returns: - An instantiated Python STIX object. - """ - - obj = get_dict(data) - - if 'type' not in obj: - raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(obj)) - - try: - obj_class = OBJ_MAP[obj['type']] - except KeyError: - raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj['type']) - return obj_class(allow_custom=allow_custom, **obj) - - -def _register_type(new_type): - """Register a custom STIX Object type. - """ - - OBJ_MAP[new_type._type] = new_type diff --git a/stix2/base.py b/stix2/base.py index 7de193b..5608102 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -3,15 +3,18 @@ import collections import copy import datetime as dt -import json + +import simplejson as json from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, InvalidObjRefError, InvalidValueError, MissingPropertiesError, - MutuallyExclusivePropertiesError, RevokeError, - UnmodifiablePropertyError) -from .utils import NOW, format_datetime, get_timestamp, parse_into_datetime + MutuallyExclusivePropertiesError) +from .markings.utils import validate +from .utils import NOW, find_property_index, format_datetime, get_timestamp +from .utils import new_version as _new_version +from .utils import revoke as _revoke __all__ = ['STIXJSONEncoder', '_STIXBase'] @@ -36,6 +39,9 @@ def get_required_properties(properties): class _STIXBase(collections.Mapping): """Base class for STIX object types""" + def object_properties(self): + return list(self._properties.keys()) + def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: if hasattr(prop, 'default'): @@ -80,8 +86,7 @@ class _STIXBase(collections.Mapping): def _check_object_constraints(self): for m in self.get("granular_markings", []): - # TODO: check selectors - pass + validate(self, m.get("selectors")) def __init__(self, allow_custom=False, **kwargs): cls = self.__class__ @@ -141,12 +146,18 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - # TODO: put keys in specific order. Probably need custom JSON encoder. - return json.dumps(self, indent=4, sort_keys=True, cls=STIXJSONEncoder, - separators=(",", ": ")) # Don't include spaces after commas. + properties = self.object_properties() + + def sort_by(element): + return find_property_index(self, properties, element) + + # separators kwarg -> don't include spaces after commas. + return json.dumps(self, indent=4, cls=STIXJSONEncoder, + item_sort_key=sort_by, + separators=(",", ": ")) def __repr__(self): - props = [(k, self[k]) for k in sorted(self._properties) if self.get(k)] + props = [(k, self[k]) for k in self.object_properties() if self.get(k)] return "{0}({1})".format(self.__class__.__name__, ", ".join(["{0!s}={1!r}".format(k, v) for k, v in props])) @@ -162,30 +173,10 @@ class _STIXBase(collections.Mapping): # Versioning API def new_version(self, **kwargs): - unchangable_properties = [] - if self.get("revoked"): - raise RevokeError("new_version") - new_obj_inner = copy.deepcopy(self._inner) - properties_to_change = kwargs.keys() - for prop in ["created", "created_by_ref", "id", "type"]: - if prop in properties_to_change: - unchangable_properties.append(prop) - if unchangable_properties: - raise UnmodifiablePropertyError(unchangable_properties) - cls = type(self) - if 'modified' not in kwargs: - kwargs['modified'] = get_timestamp() - else: - new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') - if new_modified_property < self.modified: - raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") - new_obj_inner.update(kwargs) - return cls(**new_obj_inner) + return _new_version(self, **kwargs) def revoke(self): - if self.get("revoked"): - raise RevokeError("revoke") - return self.new_version(revoked=True) + return _revoke(self) class _Observable(_STIXBase): @@ -205,18 +196,14 @@ class _Observable(_STIXBase): try: allowed_types = prop.contained.valid_types except AttributeError: - try: - allowed_types = prop.valid_types - except AttributeError: - raise ValueError("'%s' is named like an object reference property but " - "is not an ObjectReferenceProperty or a ListProperty " - "containing ObjectReferenceProperty." % prop_name) + allowed_types = prop.valid_types + + try: + ref_type = self._STIXBase__valid_refs[ref] + except TypeError: + raise ValueError("'%s' must be created with _valid_refs as a dict, not a list." % self.__class__.__name__) if allowed_types: - try: - ref_type = self._STIXBase__valid_refs[ref] - except TypeError: - raise ValueError("'%s' must be created with _valid_refs as a dict, not a list." % self.__class__.__name__) if ref_type not in allowed_types: raise InvalidObjRefError(self.__class__, prop_name, "object reference '%s' is of an invalid type '%s'" % (ref, ref_type)) diff --git a/stix2/bundle.py b/stix2/bundle.py deleted file mode 100644 index b598ceb..0000000 --- a/stix2/bundle.py +++ /dev/null @@ -1,25 +0,0 @@ -"""STIX 2 Bundle object""" - -from .base import _STIXBase -from .properties import IDProperty, Property, TypeProperty - - -class Bundle(_STIXBase): - - _type = 'bundle' - _properties = { - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'spec_version': Property(fixed="2.0"), - 'objects': Property(), - } - - def __init__(self, *args, **kwargs): - # Add any positional arguments to the 'objects' kwarg. - if args: - if isinstance(args[0], list): - kwargs['objects'] = args[0] + list(args[1:]) + kwargs.get('objects', []) - else: - kwargs['objects'] = list(args) + kwargs.get('objects', []) - - super(Bundle, self).__init__(**kwargs) diff --git a/stix2/common.py b/stix2/common.py index 7c6e747..a2e6918 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,18 +1,183 @@ """STIX 2 Common Data Types and Properties""" -from .other import ExternalReference, GranularMarking -from .properties import (BooleanProperty, ListProperty, ReferenceProperty, - StringProperty, TimestampProperty) -from .utils import NOW +from collections import OrderedDict -COMMON_PROPERTIES = { - # 'type' and 'id' should be defined on each individual type - 'created': TimestampProperty(default=lambda: NOW, precision='millisecond'), - 'modified': TimestampProperty(default=lambda: NOW, precision='millisecond'), - 'external_references': ListProperty(ExternalReference), - 'revoked': BooleanProperty(), - 'labels': ListProperty(StringProperty), - 'created_by_ref': ReferenceProperty(type="identity"), - 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), - 'granular_markings': ListProperty(GranularMarking), +from .base import _STIXBase +from .properties import (HashesProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) +from .utils import NOW, get_dict + + +class ExternalReference(_STIXBase): + _properties = OrderedDict() + _properties.update([ + ('source_name', StringProperty(required=True)), + ('description', StringProperty()), + ('url', StringProperty()), + ('hashes', HashesProperty()), + ('external_id', StringProperty()), + ]) + + def _check_object_constraints(self): + super(ExternalReference, self)._check_object_constraints() + self._check_at_least_one_property(["description", "external_id", "url"]) + + +class KillChainPhase(_STIXBase): + _properties = OrderedDict() + _properties.update([ + ('kill_chain_name', StringProperty(required=True)), + ('phase_name', StringProperty(required=True)), + ]) + + +class GranularMarking(_STIXBase): + _properties = OrderedDict() + _properties.update([ + ('marking_ref', ReferenceProperty(required=True, type="marking-definition")), + ('selectors', ListProperty(SelectorProperty, required=True)), + ]) + + +class TLPMarking(_STIXBase): + # TODO: don't allow the creation of any other TLPMarkings than the ones below + _type = 'tlp' + _properties = OrderedDict() + _properties.update([ + ('tlp', Property(required=True)) + ]) + + +class StatementMarking(_STIXBase): + _type = 'statement' + _properties = OrderedDict() + _properties.update([ + ('statement', StringProperty(required=True)) + ]) + + def __init__(self, statement=None, **kwargs): + # Allow statement as positional args. + if statement and not kwargs.get('statement'): + kwargs['statement'] = statement + + super(StatementMarking, self).__init__(**kwargs) + + +class MarkingProperty(Property): + """Represent the marking objects in the `definition` property of + marking-definition objects. + """ + + def clean(self, value): + if type(value) in OBJ_MAP_MARKING.values(): + return value + else: + raise ValueError("must be a Statement, TLP Marking or a registered marking.") + + +class MarkingDefinition(_STIXBase): + _type = 'marking-definition' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ('definition_type', StringProperty(required=True)), + ('definition', MarkingProperty(required=True)), + ]) + + def __init__(self, **kwargs): + if set(('definition_type', 'definition')).issubset(kwargs.keys()): + # Create correct marking type object + try: + marking_type = OBJ_MAP_MARKING[kwargs['definition_type']] + except KeyError: + raise ValueError("definition_type must be a valid marking type") + + if not isinstance(kwargs['definition'], marking_type): + defn = get_dict(kwargs['definition']) + kwargs['definition'] = marking_type(**defn) + + super(MarkingDefinition, self).__init__(**kwargs) + + +OBJ_MAP_MARKING = { + 'tlp': TLPMarking, + 'statement': StatementMarking, } + + +def _register_marking(cls): + """Register a custom STIX Marking Definition type. + """ + OBJ_MAP_MARKING[cls._type] = cls + return cls + + +def CustomMarking(type='x-custom-marking', properties=None): + """ + Custom STIX Marking decorator. + + Examples: + + @CustomMarking('x-custom-marking', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) + class MyNewMarkingObjectType(): + pass + + """ + def custom_builder(cls): + + class _Custom(cls, _STIXBase): + _type = type + _properties = OrderedDict() + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update(properties) + + def __init__(self, **kwargs): + _STIXBase.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_marking(_Custom) + return _Custom + + return custom_builder + + +TLP_WHITE = MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="white") +) + +TLP_GREEN = MarkingDefinition( + id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="green") +) + +TLP_AMBER = MarkingDefinition( + id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="amber") +) + +TLP_RED = MarkingDefinition( + id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="red") +) diff --git a/stix2/core.py b/stix2/core.py new file mode 100644 index 0000000..be2a53d --- /dev/null +++ b/stix2/core.py @@ -0,0 +1,101 @@ +"""STIX 2.0 Objects that are neither SDOs nor SROs""" + +from collections import OrderedDict + +from . import exceptions +from .base import _STIXBase +from .common import MarkingDefinition +from .properties import IDProperty, ListProperty, Property, TypeProperty +from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, + IntrusionSet, Malware, ObservedData, Report, ThreatActor, + Tool, Vulnerability) +from .sro import Relationship, Sighting +from .utils import get_dict + + +class STIXObjectProperty(Property): + + def clean(self, value): + try: + dictified = get_dict(value) + except ValueError: + raise ValueError("This property may only contain a dictionary or object") + if dictified == {}: + raise ValueError("This property may only contain a non-empty dictionary or object") + if 'type' in dictified and dictified['type'] == 'bundle': + raise ValueError('This property may not contain a Bundle object') + + parsed_obj = parse(dictified) + return parsed_obj + + +class Bundle(_STIXBase): + + _type = 'bundle' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('spec_version', Property(fixed="2.0")), + ('objects', ListProperty(STIXObjectProperty)), + ]) + + def __init__(self, *args, **kwargs): + # Add any positional arguments to the 'objects' kwarg. + if args: + if isinstance(args[0], list): + kwargs['objects'] = args[0] + list(args[1:]) + kwargs.get('objects', []) + else: + kwargs['objects'] = list(args) + kwargs.get('objects', []) + + super(Bundle, self).__init__(**kwargs) + + +OBJ_MAP = { + 'attack-pattern': AttackPattern, + 'bundle': Bundle, + 'campaign': Campaign, + 'course-of-action': CourseOfAction, + 'identity': Identity, + 'indicator': Indicator, + 'intrusion-set': IntrusionSet, + 'malware': Malware, + 'marking-definition': MarkingDefinition, + 'observed-data': ObservedData, + 'report': Report, + 'relationship': Relationship, + 'threat-actor': ThreatActor, + 'tool': Tool, + 'sighting': Sighting, + 'vulnerability': Vulnerability, +} + + +def parse(data, allow_custom=False): + """Deserialize a string or file-like object into a STIX object. + + Args: + data: The STIX 2 string to be parsed. + allow_custom (bool): Whether to allow custom properties or not. Default: False. + + Returns: + An instantiated Python STIX object. + """ + + obj = get_dict(data) + + if 'type' not in obj: + raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(obj)) + + try: + obj_class = OBJ_MAP[obj['type']] + except KeyError: + raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj['type']) + return obj_class(allow_custom=allow_custom, **obj) + + +def _register_type(new_type): + """Register a custom STIX Object type. + """ + + OBJ_MAP[new_type._type] = new_type diff --git a/stix2/exceptions.py b/stix2/exceptions.py index ef47dd0..32db472 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -121,7 +121,7 @@ class DependentPropertiesError(STIXError, TypeError): def __str__(self): msg = "The property dependencies for {0}: ({1}) are not met." return msg.format(self.cls.__name__, - ", ".join(x for x in self.dependencies)) + ", ".join(name for x in self.dependencies for name in x)) class AtLeastOnePropertyError(STIXError, TypeError): @@ -157,3 +157,29 @@ class ParseError(STIXError, ValueError): def __init__(self, msg): super(ParseError, self).__init__(msg) + + +class InvalidSelectorError(STIXError, AssertionError): + """Granular Marking selector violation. The selector must resolve into an existing STIX object property.""" + + def __init__(self, cls, key): + super(InvalidSelectorError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Selector {0} in {1} is not valid!" + return msg.format(self.key, self.cls.__class__.__name__) + + +class MarkingNotFoundError(STIXError, AssertionError): + """Marking violation. The marking reference must be present in SDO or SRO.""" + + def __init__(self, cls, key): + super(MarkingNotFoundError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Marking {0} was not found in {1}!" + return msg.format(self.key, self.cls.__class__.__name__) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py new file mode 100644 index 0000000..4f72e4c --- /dev/null +++ b/stix2/markings/__init__.py @@ -0,0 +1,214 @@ +""" +Python STIX 2.0 Data Markings API. + +These high level functions will operate on both object level markings and +granular markings unless otherwise noted in each of the functions. +""" + +from stix2.markings import granular_markings, object_markings + + +def get_markings(obj, selectors=None, inherited=False, descendants=False): + """ + Get all markings associated to the field(s). + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + inherited: If True, include object level markings and granular markings + inherited relative to the properties. + descendants: If True, include granular markings applied to any children + relative to the properties. + + Returns: + list: Marking identifiers that matched the selectors expression. + + Note: + If ``selectors`` is None, operation will be performed only on object + level markings. + + """ + if selectors is None: + return object_markings.get_markings(obj) + + results = granular_markings.get_markings( + obj, + selectors, + inherited, + descendants + ) + + if inherited: + results.extend(object_markings.get_markings(obj)) + + return list(set(results)) + + +def set_markings(obj, marking, selectors=None): + """ + Removes all markings associated with selectors and appends a new granular + marking. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + return object_markings.set_markings(obj, marking) + else: + return granular_markings.set_markings(obj, marking, selectors) + + +def remove_markings(obj, marking, selectors=None): + """ + Removes granular_marking from the granular_markings collection. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + return object_markings.remove_markings(obj, marking) + else: + return granular_markings.remove_markings(obj, marking, selectors) + + +def add_markings(obj, marking, selectors=None): + """ + Appends a granular_marking to the granular_markings collection. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + + Returns: + A new version of the given SDO or SRO with specified markings added. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + return object_markings.add_markings(obj, marking) + else: + return granular_markings.add_markings(obj, marking, selectors) + + +def clear_markings(obj, selectors=None): + """ + Removes all granular_marking associated with the selectors. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). + + Raises: + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings cleared. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + return object_markings.clear_markings(obj) + else: + return granular_markings.clear_markings(obj, selectors) + + +def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): + """ + Checks if field(s) is marked by any marking or by specific marking(s). + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + inherited: If True, include object level markings and granular markings + inherited to determine if the properties is/are marked. + descendants: If True, include granular markings applied to any children + of the given selector to determine if the properties is/are marked. + + Returns: + bool: True if ``selectors`` is found on internal SDO or SRO collection. + False otherwise. + + Note: + When a list of marking identifiers is provided, if ANY of the provided + marking identifiers match, True is returned. + + If ``selectors`` is None, operation will be performed only on object + level markings. + + """ + if selectors is None: + return object_markings.is_marked(obj, marking) + + result = granular_markings.is_marked( + obj, + marking, + selectors, + inherited, + descendants + ) + + if inherited: + granular_marks = granular_markings.get_markings(obj, selectors) + object_marks = object_markings.get_markings(obj) + + if granular_marks: + result = granular_markings.is_marked( + obj, + granular_marks, + selectors, + inherited, + descendants + ) + + result = result or object_markings.is_marked(obj, object_marks) + + return result diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py new file mode 100644 index 0000000..7e9ccc7 --- /dev/null +++ b/stix2/markings/granular_markings.py @@ -0,0 +1,273 @@ + +from stix2 import exceptions +from stix2.markings import utils +from stix2.utils import new_version + + +def get_markings(obj, selectors, inherited=False, descendants=False): + """ + Get all markings associated to with the properties. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selector strings relative to the SDO or + SRO in which the properties appear. + inherited: If True, include markings inherited relative to the + properties. + descendants: If True, include granular markings applied to any children + relative to the properties. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + + Returns: + list: Marking identifiers that matched the selectors expression. + + """ + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) + + granular_markings = obj.get("granular_markings", []) + + if not granular_markings: + return [] + + results = set() + + for marking in granular_markings: + for user_selector in selectors: + for marking_selector in marking.get("selectors", []): + if any([(user_selector == marking_selector), # Catch explicit selectors. + (user_selector.startswith(marking_selector) and inherited), # Catch inherited selectors. + (marking_selector.startswith(user_selector) and descendants)]): # Catch descendants selectors + refs = marking.get("marking_ref", []) + results.update([refs]) + + return list(results) + + +def set_markings(obj, marking, selectors): + """ + Removes all markings associated with selectors and appends a new granular + marking. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selector strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. + + """ + obj = clear_markings(obj, selectors) + return add_markings(obj, marking, selectors) + + +def remove_markings(obj, marking, selectors): + """ + Removes granular_marking from the granular_markings collection. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. + + """ + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) + + granular_markings = obj.get("granular_markings") + + if not granular_markings: + return obj + + granular_markings = utils.expand_markings(granular_markings) + + if isinstance(marking, list): + to_remove = [] + for m in marking: + to_remove.append({"marking_ref": m, "selectors": selectors}) + else: + to_remove = [{"marking_ref": marking, "selectors": selectors}] + + remove = utils.build_granular_marking(to_remove).get("granular_markings") + + if not any(marking in granular_markings for marking in remove): + raise exceptions.MarkingNotFoundError(obj, remove) + + granular_markings = [ + m for m in granular_markings if m not in remove + ] + + granular_markings = utils.compress_markings(granular_markings) + + if granular_markings: + return new_version(obj, granular_markings=granular_markings) + else: + return new_version(obj, granular_markings=None) + + +def add_markings(obj, marking, selectors): + """ + Appends a granular_marking to the granular_markings collection. + + Args: + obj: An SDO or SRO object. + selectors: list of type string, selectors must be relative to the TLO + in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + + Returns: + A new version of the given SDO or SRO with specified markings added. + + """ + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) + + if isinstance(marking, list): + granular_marking = [] + for m in marking: + granular_marking.append({"marking_ref": m, "selectors": sorted(selectors)}) + else: + granular_marking = [{"marking_ref": marking, "selectors": sorted(selectors)}] + + if obj.get("granular_markings"): + granular_marking.extend(obj.get("granular_markings")) + + granular_marking = utils.expand_markings(granular_marking) + granular_marking = utils.compress_markings(granular_marking) + return new_version(obj, granular_markings=granular_marking) + + +def clear_markings(obj, selectors): + """ + Removes all granular_markings associated with the selectors. + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings cleared. + + """ + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) + + granular_markings = obj.get("granular_markings") + + if not granular_markings: + return obj + + granular_markings = utils.expand_markings(granular_markings) + + sdo = utils.build_granular_marking( + [{"selectors": selectors, "marking_ref": "N/A"}] + ) + + clear = sdo.get("granular_markings", []) + + if not any(clear_selector in sdo_selectors.get("selectors", []) + for sdo_selectors in granular_markings + for clear_marking in clear + for clear_selector in clear_marking.get("selectors", []) + ): + raise exceptions.MarkingNotFoundError(obj, clear) + + for granular_marking in granular_markings: + for s in selectors: + if s in granular_marking.get("selectors", []): + marking_refs = granular_marking.get("marking_ref") + + if marking_refs: + granular_marking["marking_ref"] = "" + + granular_markings = utils.compress_markings(granular_markings) + + if granular_markings: + return new_version(obj, granular_markings=granular_markings) + else: + return new_version(obj, granular_markings=None) + + +def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): + """ + Checks if field is marked by any marking or by specific marking(s). + + Args: + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. + inherited: If True, return markings inherited from the given selector. + descendants: If True, return granular markings applied to any children + of the given selector. + + Raises: + InvalidSelectorError: If `selectors` fail validation. + + Returns: + bool: True if ``selectors`` is found on internal SDO or SRO collection. + False otherwise. + + Note: + When a list of marking identifiers is provided, if ANY of the provided + marking identifiers match, True is returned. + + """ + if selectors is None: + raise TypeError("Required argument 'selectors' must be provided") + + selectors = utils.convert_to_list(selectors) + marking = utils.convert_to_list(marking) + utils.validate(obj, selectors) + + granular_markings = obj.get("granular_markings", []) + + marked = False + markings = set() + + for granular_marking in granular_markings: + for user_selector in selectors: + for marking_selector in granular_marking.get("selectors", []): + + if any([(user_selector == marking_selector), # Catch explicit selectors. + (user_selector.startswith(marking_selector) and inherited), # Catch inherited selectors. + (marking_selector.startswith(user_selector) and descendants)]): # Catch descendants selectors + marking_ref = granular_marking.get("marking_ref", "") + + if marking and any(x == marking_ref for x in marking): + markings.update([marking_ref]) + + marked = True + + if marking: + # All user-provided markings must be found. + return markings.issuperset(set(marking)) + + return marked diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py new file mode 100644 index 0000000..c39c036 --- /dev/null +++ b/stix2/markings/object_markings.py @@ -0,0 +1,130 @@ + +from stix2 import exceptions +from stix2.markings import utils +from stix2.utils import new_version + + +def get_markings(obj): + """ + Get all object level markings from the given SDO or SRO object. + + Args: + obj: A SDO or SRO object. + + Returns: + list: Marking identifiers contained in the SDO or SRO. Empty list if no + markings are present in `object_marking_refs`. + + """ + return obj.get("object_marking_refs", []) + + +def add_markings(obj, marking): + """ + Appends an object level marking to the object_marking_refs collection. + + Args: + obj: A SDO or SRO object. + marking: identifier or list of identifiers to apply SDO or SRO object. + + Returns: + A new version of the given SDO or SRO with specified markings added. + + """ + marking = utils.convert_to_list(marking) + + object_markings = set(obj.get("object_marking_refs", []) + marking) + + return new_version(obj, object_marking_refs=list(object_markings)) + + +def remove_markings(obj, marking): + """ + Removes object level marking from the object_marking_refs collection. + + Args: + obj: A SDO or SRO object. + marking: identifier or list of identifiers that apply to the + SDO or SRO object. + + Raises: + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. + + """ + marking = utils.convert_to_list(marking) + + object_markings = obj.get("object_marking_refs", []) + + if not object_markings: + return obj + + if any(x not in obj["object_marking_refs"] for x in marking): + raise exceptions.MarkingNotFoundError(obj, marking) + + new_markings = [x for x in object_markings if x not in marking] + if new_markings: + return new_version(obj, object_marking_refs=new_markings) + else: + return new_version(obj, object_marking_refs=None) + + +def set_markings(obj, marking): + """ + Removes all object level markings and appends new object level markings to + the collection. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: A SDO or SRO object. + marking: identifier or list of identifiers to apply in the + SDO or SRO object. + + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. + + """ + return add_markings(clear_markings(obj), marking) + + +def clear_markings(obj): + """ + Removes all object level markings from the object_marking_refs collection. + + Args: + obj: A SDO or SRO object. + + Returns: + A new version of the given SDO or SRO with object_marking_refs cleared. + + """ + return new_version(obj, object_marking_refs=None) + + +def is_marked(obj, marking=None): + """ + Checks if SDO or SRO is marked by any marking or by specific marking(s). + + Args: + obj: A SDO or SRO object. + marking: identifier or list of marking identifiers that apply to the + SDO or SRO object. + + Returns: + bool: True if SDO or SRO has object level markings. False otherwise. + + Note: + When an identifier or list of identifiers is provided, if ANY of the + provided marking refs match, True is returned. + + """ + marking = utils.convert_to_list(marking) + object_markings = obj.get("object_marking_refs", []) + + if marking: + return any(x in object_markings for x in marking) + else: + return bool(object_markings) diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py new file mode 100644 index 0000000..d0d38bb --- /dev/null +++ b/stix2/markings/utils.py @@ -0,0 +1,229 @@ + +import collections + +import six + +from stix2 import exceptions + + +def _evaluate_expression(obj, selector): + """ + Walks an SDO or SRO generating selectors to match against ``selector``. If + a match is found and the the value of this property is present in the + objects. Matching value of the property will be returned. + + Args: + obj: An SDO or SRO object. + selector: A string following the selector syntax. + + Returns: + list: Values contained in matching property. Otherwise empty list. + + """ + for items, value in iterpath(obj): + path = ".".join(items) + + if path == selector and value: + return [value] + + return [] + + +def _validate_selector(obj, selector): + """Internal method to evaluate each selector.""" + results = list(_evaluate_expression(obj, selector)) + + if len(results) >= 1: + return True + + +def validate(obj, selectors): + """Given an SDO or SRO, check that each selector is valid.""" + if selectors: + for s in selectors: + if not _validate_selector(obj, s): + raise exceptions.InvalidSelectorError(obj, s) + return + + raise exceptions.InvalidSelectorError(obj, selectors) + + +def convert_to_list(data): + """Convert input into a list for further processing.""" + if data is not None: + if isinstance(data, list): + return data + else: + return [data] + + +def compress_markings(granular_markings): + """ + Compress granular markings list. If there is more than one marking + identifier matches. It will collapse into a single granular marking. + + Examples: + Input: + [ + { + "selectors": [ + "description" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + }, + { + "selectors": [ + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Output: + [ + { + "selectors": [ + "description", + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Args: + granular_markings: The granular markings list property present in a + SDO or SRO. + + Returns: + list: A list with all markings collapsed. + + """ + if not granular_markings: + return + + map_ = collections.defaultdict(set) + + for granular_marking in granular_markings: + if granular_marking.get("marking_ref"): + map_[granular_marking.get("marking_ref")].update(granular_marking.get("selectors")) + + compressed = \ + [ + {"marking_ref": marking_ref, "selectors": sorted(selectors)} + for marking_ref, selectors in six.iteritems(map_) + ] + + return compressed + + +def expand_markings(granular_markings): + """ + Expands granular markings list. If there is more than one selector per + granular marking. It will be expanded using the same marking_ref. + + Examples: + Input: + [ + { + "selectors": [ + "description", + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Output: + [ + { + "selectors": [ + "description" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + }, + { + "selectors": [ + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Args: + granular_markings: The granular markings list property present in a + SDO or SRO. + + Returns: + list: A list with all markings expanded. + + """ + expanded = [] + + for marking in granular_markings: + selectors = marking.get("selectors") + marking_ref = marking.get("marking_ref") + + expanded.extend( + [ + {"marking_ref": marking_ref, "selectors": [selector]} + for selector in selectors + ] + ) + + return expanded + + +def build_granular_marking(granular_marking): + """Returns a dictionary with the required structure for a granular + marking""" + return {"granular_markings": expand_markings(granular_marking)} + + +def iterpath(obj, path=None): + """ + Generator which walks the input ``obj`` model. Each iteration yields a + tuple containing a list of ancestors and the property value. + + Args: + obj: An SDO or SRO object. + path: None, used recursively to store ancestors. + + Example: + >>> for item in iterpath(obj): + >>> print(item) + (['type'], 'campaign') + ... + (['cybox', 'objects', '[0]', 'hashes', 'sha1'], 'cac35ec206d868b7d7cb0b55f31d9425b075082b') + + Returns: + tuple: Containing two items: a list of ancestors and the + property value. + + """ + if path is None: + path = [] + + for varname, varobj in iter(sorted(six.iteritems(obj))): + path.append(varname) + yield (path, varobj) + + if isinstance(varobj, dict): + + for item in iterpath(varobj, path): + yield item + + elif isinstance(varobj, list): + + for item in varobj: + index = "[{0}]".format(varobj.index(item)) + path.append(index) + + yield (path, item) + + if isinstance(item, dict): + for descendant in iterpath(item, path): + yield descendant + + path.pop() + + path.pop() diff --git a/stix2/observables.py b/stix2/observables.py index 366e007..4caaaa5 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -5,6 +5,8 @@ embedded in Email Message objects, inherit from _STIXBase instead of Observable and do not have a '_type' attribute. """ +from collections import OrderedDict + from .base import _Extension, _Observable, _STIXBase from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, ParseError) @@ -24,16 +26,13 @@ class ObservableProperty(Property): except ValueError: raise ValueError("The observable property must contain a dictionary") if dictified == {}: - raise ValueError("The dictionary property must contain a non-empty dictionary") + raise ValueError("The observable property must contain a non-empty dictionary") valid_refs = dict((k, v['type']) for (k, v) in dictified.items()) # from .__init__ import parse_observable # avoid circular import for key, obj in dictified.items(): parsed_obj = parse_observable(obj, valid_refs) - if not issubclass(type(parsed_obj), _Observable): - raise ValueError("Objects in an observable property must be " - "Cyber Observable Objects") dictified[key] = parsed_obj return dictified @@ -53,7 +52,7 @@ class ExtensionsProperty(DictionaryProperty): except ValueError: raise ValueError("The extensions property must contain a dictionary") if dictified == {}: - raise ValueError("The dictionary property must contain a non-empty dictionary") + raise ValueError("The extensions property must contain a non-empty dictionary") if self.enclosing_type in EXT_MAP: specific_type_map = EXT_MAP[self.enclosing_type] @@ -69,19 +68,21 @@ class ExtensionsProperty(DictionaryProperty): else: raise ValueError("The key used in the extensions dictionary is not an extension type name") else: - raise ValueError("The enclosing type has no extensions defined") + raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type) return dictified class Artifact(_Observable): _type = 'artifact' - _properties = { - 'type': TypeProperty(_type), - 'mime_type': StringProperty(), - 'payload_bin': BinaryProperty(), - 'url': StringProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('mime_type', StringProperty()), + ('payload_bin', BinaryProperty()), + ('url', StringProperty()), + ('hashes', HashesProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() @@ -91,54 +92,64 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): _type = 'autonomous-system' - _properties = { - 'type': TypeProperty(_type), - 'number': IntegerProperty(), - 'name': StringProperty(), - 'rir': StringProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('number', IntegerProperty()), + ('name', StringProperty()), + ('rir', StringProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class Directory(_Observable): _type = 'directory' - _properties = { - 'type': TypeProperty(_type), - 'path': StringProperty(required=True), - 'path_enc': StringProperty(), + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('path', StringProperty(required=True)), + ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself - 'created': TimestampProperty(), - 'modified': TimestampProperty(), - 'accessed': TimestampProperty(), - 'contains_refs': ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory'])), - } + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class DomainName(_Observable): _type = 'domain-name' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'])), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class EmailAddress(_Observable): _type = 'email-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'display_name': StringProperty(), - 'belongs_to_ref': ObjectReferenceProperty(valid_types='user-account'), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('display_name', StringProperty()), + ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + + ]) class EmailMIMEComponent(_STIXBase): - _properties = { - 'body': StringProperty(), - 'body_raw_ref': ObjectReferenceProperty(valid_types=['artifact', 'file']), - 'content_type': StringProperty(), - 'content_disposition': StringProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('body', StringProperty()), + ('body_raw_ref', ObjectReferenceProperty(valid_types=['artifact', 'file'])), + ('content_type', StringProperty()), + ('content_disposition', StringProperty()), + ]) def _check_object_constraints(self): super(EmailMIMEComponent, self)._check_object_constraints() @@ -147,23 +158,25 @@ class EmailMIMEComponent(_STIXBase): class EmailMessage(_Observable): _type = 'email-message' - _properties = { - 'type': TypeProperty(_type), - 'is_multipart': BooleanProperty(required=True), - 'date': TimestampProperty(), - 'content_type': StringProperty(), - 'from_ref': ObjectReferenceProperty(valid_types='email-addr'), - 'sender_ref': ObjectReferenceProperty(valid_types='email-addr'), - 'to_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'cc_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'bcc_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'subject': StringProperty(), - 'received_lines': ListProperty(StringProperty), - 'additional_header_fields': DictionaryProperty(), - 'body': StringProperty(), - 'body_multipart': ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent)), - 'raw_email_ref': ObjectReferenceProperty(valid_types='artifact'), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('is_multipart', BooleanProperty(required=True)), + ('date', TimestampProperty()), + ('content_type', StringProperty()), + ('from_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('sender_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('to_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('cc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('subject', StringProperty()), + ('received_lines', ListProperty(StringProperty)), + ('additional_header_fields', DictionaryProperty()), + ('body', StringProperty()), + ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), + ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) def _check_object_constraints(self): super(EmailMessage, self)._check_object_constraints() @@ -174,82 +187,92 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): - _properties = { - 'contains_refs': ListProperty(ObjectReferenceProperty(valid_types='file'), required=True), - 'version': StringProperty(), - 'comment': StringProperty(), - } + _type = 'archive-ext' + _properties = OrderedDict() + _properties.update([ + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), + ('version', StringProperty()), + ('comment', StringProperty()), + ]) class AlternateDataStream(_STIXBase): - _properties = { - 'name': StringProperty(required=True), - 'hashes': HashesProperty(), - 'size': IntegerProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ]) class NTFSExt(_Extension): - _properties = { - 'sid': StringProperty(), - 'alternate_data_streams': ListProperty(EmbeddedObjectProperty(type=AlternateDataStream)), - } + _type = 'ntfs-ext' + _properties = OrderedDict() + _properties.update([ + ('sid', StringProperty()), + ('alternate_data_streams', ListProperty(EmbeddedObjectProperty(type=AlternateDataStream))), + ]) class PDFExt(_Extension): - _properties = { - 'version': StringProperty(), - 'is_optimized': BooleanProperty(), - 'document_info_dict': DictionaryProperty(), - 'pdfid0': StringProperty(), - 'pdfid1': StringProperty(), - } + _type = 'pdf-ext' + _properties = OrderedDict() + _properties.update([ + ('version', StringProperty()), + ('is_optimized', BooleanProperty()), + ('document_info_dict', DictionaryProperty()), + ('pdfid0', StringProperty()), + ('pdfid1', StringProperty()), + ]) class RasterImageExt(_Extension): - _properties = { - 'image_height': IntegerProperty(), - 'image_weight': IntegerProperty(), - 'bits_per_pixel': IntegerProperty(), - 'image_compression_algorithm': StringProperty(), - 'exif_tags': DictionaryProperty(), - } + _type = 'raster-image-ext' + _properties = OrderedDict() + _properties.update([ + ('image_height', IntegerProperty()), + ('image_weight', IntegerProperty()), + ('bits_per_pixel', IntegerProperty()), + ('image_compression_algorithm', StringProperty()), + ('exif_tags', DictionaryProperty()), + ]) class WindowsPEOptionalHeaderType(_STIXBase): - _properties = { - 'magic_hex': HexProperty(), - 'major_linker_version': IntegerProperty(), - 'minor_linker_version': IntegerProperty(), - 'size_of_code': IntegerProperty(), - 'size_of_initialized_data': IntegerProperty(), - 'size_of_uninitialized_data': IntegerProperty(), - 'address_of_entry_point': IntegerProperty(), - 'base_of_code': IntegerProperty(), - 'base_of_data': IntegerProperty(), - 'image_base': IntegerProperty(), - 'section_alignment': IntegerProperty(), - 'file_alignment': IntegerProperty(), - 'major_os_version': IntegerProperty(), - 'minor_os_version': IntegerProperty(), - 'major_image_version': IntegerProperty(), - 'minor_image_version': IntegerProperty(), - 'major_subsystem_version': IntegerProperty(), - 'minor_subsystem_version': IntegerProperty(), - 'win32_version_value_hex': HexProperty(), - 'size_of_image': IntegerProperty(), - 'size_of_headers': IntegerProperty(), - 'checksum_hex': HexProperty(), - 'subsystem_hex': HexProperty(), - 'dll_characteristics_hex': HexProperty(), - 'size_of_stack_reserve': IntegerProperty(), - 'size_of_stack_commit': IntegerProperty(), - 'size_of_heap_reserve': IntegerProperty(), - 'size_of_heap_commit': IntegerProperty(), - 'loader_flags_hex': HexProperty(), - 'number_of_rva_and_sizes': IntegerProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('magic_hex', HexProperty()), + ('major_linker_version', IntegerProperty()), + ('minor_linker_version', IntegerProperty()), + ('size_of_code', IntegerProperty()), + ('size_of_initialized_data', IntegerProperty()), + ('size_of_uninitialized_data', IntegerProperty()), + ('address_of_entry_point', IntegerProperty()), + ('base_of_code', IntegerProperty()), + ('base_of_data', IntegerProperty()), + ('image_base', IntegerProperty()), + ('section_alignment', IntegerProperty()), + ('file_alignment', IntegerProperty()), + ('major_os_version', IntegerProperty()), + ('minor_os_version', IntegerProperty()), + ('major_image_version', IntegerProperty()), + ('minor_image_version', IntegerProperty()), + ('major_subsystem_version', IntegerProperty()), + ('minor_subsystem_version', IntegerProperty()), + ('win32_version_value_hex', HexProperty()), + ('size_of_image', IntegerProperty()), + ('size_of_headers', IntegerProperty()), + ('checksum_hex', HexProperty()), + ('subsystem_hex', HexProperty()), + ('dll_characteristics_hex', HexProperty()), + ('size_of_stack_reserve', IntegerProperty()), + ('size_of_stack_commit', IntegerProperty()), + ('size_of_heap_reserve', IntegerProperty()), + ('size_of_heap_commit', IntegerProperty()), + ('loader_flags_hex', HexProperty()), + ('number_of_rva_and_sizes', IntegerProperty()), + ('hashes', HashesProperty()), + ]) def _check_object_constraints(self): super(WindowsPEOptionalHeaderType, self)._check_object_constraints() @@ -257,53 +280,57 @@ class WindowsPEOptionalHeaderType(_STIXBase): class WindowsPESection(_STIXBase): - _properties = { - 'name': StringProperty(required=True), - 'size': IntegerProperty(), - 'entropy': FloatProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('size', IntegerProperty()), + ('entropy', FloatProperty()), + ('hashes', HashesProperty()), + ]) class WindowsPEBinaryExt(_Extension): - _properties = { - 'pe_type': StringProperty(required=True), # open_vocab - 'imphash': StringProperty(), - 'machine_hex': HexProperty(), - 'number_of_sections': IntegerProperty(), - 'time_date_stamp': TimestampProperty(precision='second'), - 'pointer_to_symbol_table_hex': HexProperty(), - 'number_of_symbols': IntegerProperty(), - 'size_of_optional_header': IntegerProperty(), - 'characteristics_hex': HexProperty(), - 'file_header_hashes': HashesProperty(), - 'optional_header': EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType), - 'sections': ListProperty(EmbeddedObjectProperty(type=WindowsPESection)), - } + _type = 'windows-pebinary-ext' + _properties = OrderedDict() + _properties.update([ + ('pe_type', StringProperty(required=True)), # open_vocab + ('imphash', StringProperty()), + ('machine_hex', HexProperty()), + ('number_of_sections', IntegerProperty()), + ('time_date_stamp', TimestampProperty(precision='second')), + ('pointer_to_symbol_table_hex', HexProperty()), + ('number_of_symbols', IntegerProperty()), + ('size_of_optional_header', IntegerProperty()), + ('characteristics_hex', HexProperty()), + ('file_header_hashes', HashesProperty()), + ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), + ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), + ]) class File(_Observable): _type = 'file' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'hashes': HashesProperty(), - 'size': IntegerProperty(), - 'name': StringProperty(), - 'name_enc': StringProperty(), - 'magic_number_hex': HexProperty(), - 'mime_type': StringProperty(), + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ('name', StringProperty()), + ('name_enc', StringProperty()), + ('magic_number_hex', HexProperty()), + ('mime_type', StringProperty()), # these are not the created/modified timestamps of the object itself - 'created': TimestampProperty(), - 'modified': TimestampProperty(), - 'accessed': TimestampProperty(), - 'parent_directory_ref': ObjectReferenceProperty(valid_types='directory'), - 'is_encrypted': BooleanProperty(), - 'encryption_algorithm': StringProperty(), - 'decryption_key': StringProperty(), - 'contains_refs': ListProperty(ObjectReferenceProperty), - 'content_ref': ObjectReferenceProperty(valid_types='artifact'), - } + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), + ('is_encrypted', BooleanProperty()), + ('encryption_algorithm', StringProperty()), + ('decryption_key', StringProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty)), + ('content_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) def _check_object_constraints(self): super(File, self)._check_object_constraints() @@ -313,61 +340,75 @@ class File(_Observable): class IPv4Address(_Observable): _type = 'ipv4-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), - 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class IPv6Address(_Observable): _type = 'ipv6-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), - 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class MACAddress(_Observable): _type = 'mac-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class Mutex(_Observable): _type = 'mutex' - _properties = { - 'type': TypeProperty(_type), - 'name': StringProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class HTTPRequestExt(_Extension): - _properties = { - 'request_method': StringProperty(required=True), - 'request_value': StringProperty(required=True), - 'request_version': StringProperty(), - 'request_header': DictionaryProperty(), - 'message_body_length': IntegerProperty(), - 'message_body_data_ref': ObjectReferenceProperty(valid_types='artifact'), - } + _type = 'http-request-ext' + _properties = OrderedDict() + _properties.update([ + ('request_method', StringProperty(required=True)), + ('request_value', StringProperty(required=True)), + ('request_version', StringProperty()), + ('request_header', DictionaryProperty()), + ('message_body_length', IntegerProperty()), + ('message_body_data_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) class ICMPExt(_Extension): - _properties = { - 'icmp_type_hex': HexProperty(required=True), - 'icmp_code_hex': HexProperty(required=True), - } + _type = 'icmp-ext' + _properties = OrderedDict() + _properties.update([ + ('icmp_type_hex', HexProperty(required=True)), + ('icmp_code_hex', HexProperty(required=True)), + ]) class SocketExt(_Extension): - _properties = { - 'address_family': EnumProperty([ + _type = 'socket-ext' + _properties = OrderedDict() + _properties.update([ + ('address_family', EnumProperty([ "AF_UNSPEC", "AF_INET", "AF_IPX", @@ -376,58 +417,61 @@ class SocketExt(_Extension): "AF_INET6", "AF_IRDA", "AF_BTH", - ], required=True), - 'is_blocking': BooleanProperty(), - 'is_listening': BooleanProperty(), - 'protocol_family': EnumProperty([ + ], required=True)), + ('is_blocking', BooleanProperty()), + ('is_listening', BooleanProperty()), + ('protocol_family', EnumProperty([ "PF_INET", "PF_IPX", "PF_APPLETALK", "PF_INET6", "PF_AX25", "PF_NETROM" - ]), - 'options': DictionaryProperty(), - 'socket_type': EnumProperty([ + ])), + ('options', DictionaryProperty()), + ('socket_type', EnumProperty([ "SOCK_STREAM", "SOCK_DGRAM", "SOCK_RAW", "SOCK_RDM", "SOCK_SEQPACKET", - ]), - } + ])), + ]) class TCPExt(_Extension): - _properties = { - 'src_flags_hex': HexProperty(), - 'dst_flags_hex': HexProperty(), - } + _type = 'tcp-ext' + _properties = OrderedDict() + _properties.update([ + ('src_flags_hex', HexProperty()), + ('dst_flags_hex', HexProperty()), + ]) class NetworkTraffic(_Observable): _type = 'network-traffic' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'start': TimestampProperty(), - 'end': TimestampProperty(), - 'is_active': BooleanProperty(), - 'src_ref': ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name']), - 'dst_ref': ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name']), - 'src_port': IntegerProperty(), - 'dst_port': IntegerProperty(), - 'protocols': ListProperty(StringProperty, required=True), - 'src_byte_count': IntegerProperty(), - 'dst_byte_count': IntegerProperty(), - 'src_packets': IntegerProperty(), - 'dst_packets': IntegerProperty(), - 'ipfix': DictionaryProperty(), - 'src_payload_ref': ObjectReferenceProperty(valid_types='artifact'), - 'dst_payload_ref': ObjectReferenceProperty(valid_types='artifact'), - 'encapsulates_refs': ListProperty(ObjectReferenceProperty(valid_types='network-traffic')), - 'encapsulates_by_ref': ObjectReferenceProperty(valid_types='network-traffic'), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('start', TimestampProperty()), + ('end', TimestampProperty()), + ('is_active', BooleanProperty()), + ('src_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('dst_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('src_port', IntegerProperty()), + ('dst_port', IntegerProperty()), + ('protocols', ListProperty(StringProperty, required=True)), + ('src_byte_count', IntegerProperty()), + ('dst_byte_count', IntegerProperty()), + ('src_packets', IntegerProperty()), + ('dst_packets', IntegerProperty()), + ('ipfix', DictionaryProperty()), + ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), + ]) def _check_object_constraints(self): super(NetworkTraffic, self)._check_object_constraints() @@ -435,37 +479,41 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): - _properties = { - 'aslr_enabled': BooleanProperty(), - 'dep_enabled': BooleanProperty(), - 'priority': StringProperty(), - 'owner_sid': StringProperty(), - 'window_title': StringProperty(), - 'startup_info': DictionaryProperty(), - } + _type = 'windows-process-ext' + _properties = OrderedDict() + _properties.update([ + ('aslr_enabled', BooleanProperty()), + ('dep_enabled', BooleanProperty()), + ('priority', StringProperty()), + ('owner_sid', StringProperty()), + ('window_title', StringProperty()), + ('startup_info', DictionaryProperty()), + ]) class WindowsServiceExt(_Extension): - _properties = { - 'service_name': StringProperty(required=True), - 'descriptions': ListProperty(StringProperty), - 'display_name': StringProperty(), - 'group_name': StringProperty(), - 'start_type': EnumProperty([ + _type = 'windows-service-ext' + _properties = OrderedDict() + _properties.update([ + ('service_name', StringProperty(required=True)), + ('descriptions', ListProperty(StringProperty)), + ('display_name', StringProperty()), + ('group_name', StringProperty()), + ('start_type', EnumProperty([ "SERVICE_AUTO_START", "SERVICE_BOOT_START", "SERVICE_DEMAND_START", "SERVICE_DISABLED", "SERVICE_SYSTEM_ALERT", - ]), - 'service_dll_refs': ListProperty(ObjectReferenceProperty(valid_types='file')), - 'service_type': EnumProperty([ + ])), + ('service_dll_refs', ListProperty(ObjectReferenceProperty(valid_types='file'))), + ('service_type', EnumProperty([ "SERVICE_KERNEL_DRIVER", "SERVICE_FILE_SYSTEM_DRIVER", "SERVICE_WIN32_OWN_PROCESS", "SERVICE_WIN32_SHARE_PROCESS", - ]), - 'service_status': EnumProperty([ + ])), + ('service_status', EnumProperty([ "SERVICE_CONTINUE_PENDING", "SERVICE_PAUSE_PENDING", "SERVICE_PAUSED", @@ -473,30 +521,31 @@ class WindowsServiceExt(_Extension): "SERVICE_START_PENDING", "SERVICE_STOP_PENDING", "SERVICE_STOPPED", - ]), - } + ])), + ]) class Process(_Observable): _type = 'process' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'is_hidden': BooleanProperty(), - 'pid': IntegerProperty(), - 'name': StringProperty(), + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('is_hidden', BooleanProperty()), + ('pid', IntegerProperty()), + ('name', StringProperty()), # this is not the created timestamps of the object itself - 'created': TimestampProperty(), - 'cwd': StringProperty(), - 'arguments': ListProperty(StringProperty), - 'command_line': StringProperty(), - 'environment_variables': DictionaryProperty(), - 'opened_connection_refs': ListProperty(ObjectReferenceProperty(valid_types='network-traffic')), - 'creator_user_ref': ObjectReferenceProperty(valid_types='user-account'), - 'binary_ref': ObjectReferenceProperty(valid_types='file'), - 'parent_ref': ObjectReferenceProperty(valid_types='process'), - 'child_refs': ListProperty(ObjectReferenceProperty('process')), - } + ('created', TimestampProperty()), + ('cwd', StringProperty()), + ('arguments', ListProperty(StringProperty)), + ('command_line', StringProperty()), + ('environment_variables', DictionaryProperty()), + ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('binary_ref', ObjectReferenceProperty(valid_types='file')), + ('parent_ref', ObjectReferenceProperty(valid_types='process')), + ('child_refs', ListProperty(ObjectReferenceProperty('process'))), + ]) def _check_object_constraints(self): # no need to check windows-service-ext, since it has a required property @@ -515,60 +564,68 @@ class Process(_Observable): class Software(_Observable): _type = 'software' - _properties = { - 'type': TypeProperty(_type), - 'name': StringProperty(required=True), - 'cpe': StringProperty(), - 'languages': ListProperty(StringProperty), - 'vendor': StringProperty(), - 'version': StringProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty(required=True)), + ('cpe', StringProperty()), + ('languages', ListProperty(StringProperty)), + ('vendor', StringProperty()), + ('version', StringProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class URL(_Observable): _type = 'url' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) class UNIXAccountExt(_Extension): - _properties = { - 'gid': IntegerProperty(), - 'groups': ListProperty(StringProperty), - 'home_dir': StringProperty(), - 'shell': StringProperty(), - } + _type = 'unix-account-ext' + _properties = OrderedDict() + _properties.update([ + ('gid', IntegerProperty()), + ('groups', ListProperty(StringProperty)), + ('home_dir', StringProperty()), + ('shell', StringProperty()), + ]) class UserAccount(_Observable): _type = 'user-account' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'user_id': StringProperty(required=True), - 'account_login': StringProperty(), - 'account_type': StringProperty(), # open vocab - 'display_name': StringProperty(), - 'is_service_account': BooleanProperty(), - 'is_privileged': BooleanProperty(), - 'can_escalate_privs': BooleanProperty(), - 'is_disabled': BooleanProperty(), - 'account_created': TimestampProperty(), - 'account_expires': TimestampProperty(), - 'password_last_changed': TimestampProperty(), - 'account_first_login': TimestampProperty(), - 'account_last_login': TimestampProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('user_id', StringProperty(required=True)), + ('account_login', StringProperty()), + ('account_type', StringProperty()), # open vocab + ('display_name', StringProperty()), + ('is_service_account', BooleanProperty()), + ('is_privileged', BooleanProperty()), + ('can_escalate_privs', BooleanProperty()), + ('is_disabled', BooleanProperty()), + ('account_created', TimestampProperty()), + ('account_expires', TimestampProperty()), + ('password_last_changed', TimestampProperty()), + ('account_first_login', TimestampProperty()), + ('account_last_login', TimestampProperty()), + ]) class WindowsRegistryValueType(_STIXBase): _type = 'windows-registry-value-type' - _properties = { - 'name': StringProperty(required=True), - 'data': StringProperty(), - 'data_type': EnumProperty([ + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('data', StringProperty()), + ('data_type', EnumProperty([ 'REG_NONE', 'REG_SZ', 'REG_EXPAND_SZ', @@ -582,21 +639,23 @@ class WindowsRegistryValueType(_STIXBase): 'REG_RESOURCE_REQUIREMENTS_LIST', 'REG_QWORD', 'REG_INVALID_TYPE', - ]), - } + ])), + ]) class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' - _properties = { - 'type': TypeProperty(_type), - 'key': StringProperty(required=True), - 'values': ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType)), + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('key', StringProperty(required=True)), + ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself - 'modified': TimestampProperty(), - 'creator_user_ref': ObjectReferenceProperty(valid_types='user-account'), - 'number_of_subkeys': IntegerProperty(), - } + ('modified', TimestampProperty()), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('number_of_subkeys', IntegerProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) @property def values(self): @@ -606,44 +665,47 @@ class WindowsRegistryKey(_Observable): class X509V3ExtenstionsType(_STIXBase): _type = 'x509-v3-extensions-type' - _properties = { - 'basic_constraints': StringProperty(), - 'name_constraints': StringProperty(), - 'policy_constraints': StringProperty(), - 'key_usage': StringProperty(), - 'extended_key_usage': StringProperty(), - 'subject_key_identifier': StringProperty(), - 'authority_key_identifier': StringProperty(), - 'subject_alternative_name': StringProperty(), - 'issuer_alternative_name': StringProperty(), - 'subject_directory_attributes': StringProperty(), - 'crl_distribution_points': StringProperty(), - 'inhibit_any_policy': StringProperty(), - 'private_key_usage_period_not_before': TimestampProperty(), - 'private_key_usage_period_not_after': TimestampProperty(), - 'certificate_policies': StringProperty(), - 'policy_mappings': StringProperty(), - } + _properties = OrderedDict() + _properties.update([ + ('basic_constraints', StringProperty()), + ('name_constraints', StringProperty()), + ('policy_constraints', StringProperty()), + ('key_usage', StringProperty()), + ('extended_key_usage', StringProperty()), + ('subject_key_identifier', StringProperty()), + ('authority_key_identifier', StringProperty()), + ('subject_alternative_name', StringProperty()), + ('issuer_alternative_name', StringProperty()), + ('subject_directory_attributes', StringProperty()), + ('crl_distribution_points', StringProperty()), + ('inhibit_any_policy', StringProperty()), + ('private_key_usage_period_not_before', TimestampProperty()), + ('private_key_usage_period_not_after', TimestampProperty()), + ('certificate_policies', StringProperty()), + ('policy_mappings', StringProperty()), + ]) class X509Certificate(_Observable): _type = 'x509-certificate' - _properties = { - 'type': TypeProperty(_type), - 'is_self_signed': BooleanProperty(), - 'hashes': HashesProperty(), - 'version': StringProperty(), - 'serial_number': StringProperty(), - 'signature_algorithm': StringProperty(), - 'issuer': StringProperty(), - 'validity_not_before': TimestampProperty(), - 'validity_not_after': TimestampProperty(), - 'subject': StringProperty(), - 'subject_public_key_algorithm': StringProperty(), - 'subject_public_key_modulus': StringProperty(), - 'subject_public_key_exponent': IntegerProperty(), - 'x509_v3_extensions': EmbeddedObjectProperty(type=X509V3ExtenstionsType), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('is_self_signed', BooleanProperty()), + ('hashes', HashesProperty()), + ('version', StringProperty()), + ('serial_number', StringProperty()), + ('signature_algorithm', StringProperty()), + ('issuer', StringProperty()), + ('validity_not_before', TimestampProperty()), + ('validity_not_after', TimestampProperty()), + ('subject', StringProperty()), + ('subject_public_key_algorithm', StringProperty()), + ('subject_public_key_modulus', StringProperty()), + ('subject_public_key_exponent', IntegerProperty()), + ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) OBJ_MAP_OBSERVABLE = { @@ -667,65 +729,61 @@ OBJ_MAP_OBSERVABLE = { 'x509-certificate': X509Certificate, } -EXT_MAP_FILE = { - 'archive-ext': ArchiveExt, - 'ntfs-ext': NTFSExt, - 'pdf-ext': PDFExt, - 'raster-image-ext': RasterImageExt, - 'windows-pebinary-ext': WindowsPEBinaryExt -} - -EXT_MAP_NETWORK_TRAFFIC = { - 'http-request-ext': HTTPRequestExt, - 'icmp-ext': ICMPExt, - 'socket-ext': SocketExt, - 'tcp-ext': TCPExt, -} - -EXT_MAP_PROCESS = { - 'windows-process-ext': WindowsProcessExt, - 'windows-service-ext': WindowsServiceExt, -} - -EXT_MAP_USER_ACCOUNT = { - 'unix-account-ext': UNIXAccountExt, -} EXT_MAP = { - 'file': EXT_MAP_FILE, - 'network-traffic': EXT_MAP_NETWORK_TRAFFIC, - 'process': EXT_MAP_PROCESS, - 'user-account': EXT_MAP_USER_ACCOUNT, - + 'file': { + 'archive-ext': ArchiveExt, + 'ntfs-ext': NTFSExt, + 'pdf-ext': PDFExt, + 'raster-image-ext': RasterImageExt, + 'windows-pebinary-ext': WindowsPEBinaryExt + }, + 'network-traffic': { + 'http-request-ext': HTTPRequestExt, + 'icmp-ext': ICMPExt, + 'socket-ext': SocketExt, + 'tcp-ext': TCPExt, + }, + 'process': { + 'windows-process-ext': WindowsProcessExt, + 'windows-service-ext': WindowsServiceExt, + }, + 'user-account': { + 'unix-account-ext': UNIXAccountExt, + }, } -def parse_observable(data, _valid_refs=[], allow_custom=False): - """Deserialize a string or file-like object into a STIX Cyber Observable object. +def parse_observable(data, _valid_refs=None, allow_custom=False): + """Deserialize a string or file-like object into a STIX Cyber Observable + object. Args: data: The STIX 2 string to be parsed. - _valid_refs: A list of object references valid for the scope of the object being parsed. - allow_custom: Whether to allow custom properties or not. Default: False. + _valid_refs: A list of object references valid for the scope of the + object being parsed. Use empty list if no valid refs are present. + allow_custom: Whether to allow custom properties or not. + Default: False. Returns: An instantiated Python STIX Cyber Observable object. """ obj = get_dict(data) - obj['_valid_refs'] = _valid_refs + obj['_valid_refs'] = _valid_refs or [] if 'type' not in obj: - raise ParseError("Can't parse object with no 'type' property: %s" % str(obj)) + raise ParseError("Can't parse observable with no 'type' property: %s" % str(obj)) try: obj_class = OBJ_MAP_OBSERVABLE[obj['type']] except KeyError: - raise ParseError("Can't parse unknown object type '%s'! For custom observables, use the CustomObservable decorator." % obj['type']) + raise ParseError("Can't parse unknown observable type '%s'! For custom observables, " + "use the CustomObservable decorator." % obj['type']) if 'extensions' in obj and obj['type'] in EXT_MAP: for name, ext in obj['extensions'].items(): if name not in EXT_MAP[obj['type']]: - raise ParseError("Can't parse Unknown extension type '%s' for object type '%s'!" % (name, obj['type'])) + raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type'])) ext_class = EXT_MAP[obj['type']][name] obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) @@ -739,17 +797,41 @@ def _register_observable(new_observable): OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable -def CustomObservable(type='x-custom-observable', properties={}): +def CustomObservable(type='x-custom-observable', properties=None): """Custom STIX Cyber Observable type decorator + + Example 1: + + @CustomObservable('x-custom-observable', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) + class MyNewObservableType(): + pass """ def custom_builder(cls): class _Custom(cls, _Observable): _type = type - _properties = { - 'type': TypeProperty(_type), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ]) + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + # Check properties ending in "_ref/s" are ObjectReferenceProperties + for prop_name, prop in properties: + if prop_name.endswith('_ref') and not isinstance(prop, ObjectReferenceProperty): + raise ValueError("'%s' is named like an object reference property but " + "is not an ObjectReferenceProperty." % prop_name) + elif (prop_name.endswith('_refs') and (not isinstance(prop, ListProperty) + or not isinstance(prop.contained, ObjectReferenceProperty))): + raise ValueError("'%s' is named like an object reference list property but " + "is not a ListProperty containing ObjectReferenceProperty." % prop_name) + _properties.update(properties) def __init__(self, **kwargs): @@ -760,3 +842,50 @@ def CustomObservable(type='x-custom-observable', properties={}): return _Custom return custom_builder + + +def _register_extension(observable, new_extension): + """Register a custom extension to a STIX Cyber Observable type. + """ + + try: + observable_type = observable._type + except AttributeError: + raise ValueError("Unknown observable type. Custom observables must be " + "created with the @CustomObservable decorator.") + + try: + EXT_MAP[observable_type][new_extension._type] = new_extension + except KeyError: + if observable_type not in OBJ_MAP_OBSERVABLE: + raise ValueError("Unknown observable type '%s'. Custom observables " + "must be created with the @CustomObservable decorator." + % observable_type) + else: + EXT_MAP[observable_type] = {new_extension._type: new_extension} + + +def CustomExtension(observable=None, type='x-custom-observable', properties={}): + """Decorator for custom extensions to STIX Cyber Observables + """ + + if not observable or not issubclass(observable, _Observable): + raise ValueError("'observable' must be a valid Observable class!") + + def custom_builder(cls): + + class _Custom(cls, _Extension): + _type = type + _properties = { + 'extensions': ExtensionsProperty(enclosing_type=_type), + } + _properties.update(properties) + + def __init__(self, **kwargs): + _Extension.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_extension(observable, _Custom) + return _Custom + + return custom_builder diff --git a/stix2/other.py b/stix2/other.py deleted file mode 100644 index cd75745..0000000 --- a/stix2/other.py +++ /dev/null @@ -1,128 +0,0 @@ -"""STIX 2.0 Objects that are neither SDOs nor SROs""" - -from .base import _STIXBase -from .properties import (IDProperty, ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) -from .utils import NOW, get_dict - - -class ExternalReference(_STIXBase): - _properties = { - 'source_name': StringProperty(required=True), - 'description': StringProperty(), - 'url': StringProperty(), - 'external_id': StringProperty(), - } - - def _check_object_constraints(self): - super(ExternalReference, self)._check_object_constraints() - self._check_at_least_one_property(["description", "external_id", "url"]) - - -class KillChainPhase(_STIXBase): - _properties = { - 'kill_chain_name': StringProperty(required=True), - 'phase_name': StringProperty(required=True), - } - - -class GranularMarking(_STIXBase): - _properties = { - 'marking_ref': ReferenceProperty(required=True, type="marking-definition"), - 'selectors': ListProperty(SelectorProperty, required=True), - } - - -class TLPMarking(_STIXBase): - # TODO: don't allow the creation of any other TLPMarkings than the ones below - _properties = { - 'tlp': Property(required=True) - } - - -class StatementMarking(_STIXBase): - _properties = { - 'statement': StringProperty(required=True) - } - - def __init__(self, statement=None, **kwargs): - # Allow statement as positional args. - if statement and not kwargs.get('statement'): - kwargs['statement'] = statement - - super(StatementMarking, self).__init__(**kwargs) - - -class MarkingProperty(Property): - """Represent the marking objects in the `definition` property of - marking-definition objects. - """ - - def clean(self, value): - if type(value) in [TLPMarking, StatementMarking]: - return value - else: - raise ValueError("must be a Statement or TLP Marking.") - - -class MarkingDefinition(_STIXBase): - _type = 'marking-definition' - _properties = { - 'created': TimestampProperty(default=lambda: NOW), - 'external_references': ListProperty(ExternalReference), - 'created_by_ref': ReferenceProperty(type="identity"), - 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), - 'granular_markings': ListProperty(GranularMarking), - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'definition_type': StringProperty(required=True), - 'definition': MarkingProperty(required=True), - } - marking_map = { - 'tlp': TLPMarking, - 'statement': StatementMarking, - } - - def __init__(self, **kwargs): - if set(('definition_type', 'definition')).issubset(kwargs.keys()): - # Create correct marking type object - try: - marking_type = self.marking_map[kwargs['definition_type']] - except KeyError: - raise ValueError("definition_type must be a valid marking type") - - if not isinstance(kwargs['definition'], marking_type): - defn = get_dict(kwargs['definition']) - kwargs['definition'] = marking_type(**defn) - - super(MarkingDefinition, self).__init__(**kwargs) - - -TLP_WHITE = MarkingDefinition( - id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="white") -) - -TLP_GREEN = MarkingDefinition( - id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="green") -) - -TLP_AMBER = MarkingDefinition( - id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="amber") -) - -TLP_RED = MarkingDefinition( - id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="red") -) diff --git a/stix2/properties.py b/stix2/properties.py index db06763..4889a45 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -6,6 +6,7 @@ import re import uuid from six import string_types, text_type +from stix2patterns.validator import run_validator from .base import _STIXBase from .exceptions import DictionaryKeyError @@ -118,6 +119,9 @@ class ListProperty(Property): if type(self.contained) is EmbeddedObjectProperty: obj_type = self.contained.type + elif type(self.contained).__name__ is 'STIXObjectProperty': + # ^ this way of checking doesn't require a circular import + obj_type = type(valid) else: obj_type = self.contained @@ -308,6 +312,7 @@ class ReferenceProperty(Property): def clean(self, value): if isinstance(value, _STIXBase): value = value.id + value = str(value) if self.type: if not value.startswith(self.type): raise ValueError("must start with '{0}'.".format(self.type)) @@ -367,3 +372,17 @@ class EnumProperty(StringProperty): if value not in self.allowed: raise ValueError("value '%s' is not valid for this enumeration." % value) return self.string_type(value) + + +class PatternProperty(StringProperty): + + def __init__(self, **kwargs): + super(PatternProperty, self).__init__(**kwargs) + + def clean(self, value): + str_value = super(PatternProperty, self).clean(value) + errors = run_validator(str_value) + if errors: + raise ValueError(str(errors[0])) + + return self.string_type(value) diff --git a/stix2/sdo.py b/stix2/sdo.py index 8115b9d..77c781a 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -1,221 +1,313 @@ """STIX 2.0 Domain Objects""" +from collections import OrderedDict + import stix2 from .base import _STIXBase -from .common import COMMON_PROPERTIES +from .common import ExternalReference, GranularMarking, KillChainPhase from .observables import ObservableProperty -from .other import KillChainPhase -from .properties import (IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) +from .properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, PatternProperty, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty) from .utils import NOW class AttackPattern(_STIXBase): _type = 'attack-pattern' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Campaign(_STIXBase): _type = 'campaign' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'first_seen': TimestampProperty(), - 'last_seen': TimestampProperty(), - 'objective': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('objective', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class CourseOfAction(_STIXBase): _type = 'course-of-action' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Identity(_STIXBase): _type = 'identity' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'identity_class': StringProperty(required=True), - 'sectors': ListProperty(StringProperty), - 'contact_information': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('identity_class', StringProperty(required=True)), + ('sectors', ListProperty(StringProperty)), + ('contact_information', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Indicator(_STIXBase): _type = 'indicator' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(), - 'description': StringProperty(), - 'pattern': StringProperty(required=True), - 'valid_from': TimestampProperty(default=lambda: NOW), - 'valid_until': TimestampProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('labels', ListProperty(StringProperty, required=True)), + ('name', StringProperty()), + ('description', StringProperty()), + ('pattern', PatternProperty(required=True)), + ('valid_from', TimestampProperty(default=lambda: NOW)), + ('valid_until', TimestampProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class IntrusionSet(_STIXBase): _type = 'intrusion-set' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'first_seen': TimestampProperty(), - 'last_seen ': TimestampProperty(), - 'goals': ListProperty(StringProperty), - 'resource_level': StringProperty(), - 'primary_motivation': StringProperty(), - 'secondary_motivations': ListProperty(StringProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen ', TimestampProperty()), + ('goals', ListProperty(StringProperty)), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Malware(_STIXBase): _type = 'malware' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class ObservedData(_STIXBase): _type = 'observed-data' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'first_observed': TimestampProperty(required=True), - 'last_observed': TimestampProperty(required=True), - 'number_observed': IntegerProperty(required=True), - 'objects': ObservableProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_observed', TimestampProperty(required=True)), + ('last_observed', TimestampProperty(required=True)), + ('number_observed', IntegerProperty(required=True)), + ('objects', ObservableProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Report(_STIXBase): _type = 'report' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'published': TimestampProperty(), - 'object_refs': ListProperty(ReferenceProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('published', TimestampProperty()), + ('object_refs', ListProperty(ReferenceProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class ThreatActor(_STIXBase): _type = 'threat-actor' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'roles': ListProperty(StringProperty), - 'goals': ListProperty(StringProperty), - 'sophistication': StringProperty(), - 'resource_level': StringProperty(), - 'primary_motivation': StringProperty(), - 'secondary_motivations': ListProperty(StringProperty), - 'personal_motivations': ListProperty(StringProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('roles', ListProperty(StringProperty)), + ('goals', ListProperty(StringProperty)), + ('sophistication', StringProperty()), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('personal_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Tool(_STIXBase): _type = 'tool' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - 'tool_version': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('tool_version', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Vulnerability(_STIXBase): _type = 'vulnerability' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) -def CustomObject(type='x-custom-type', properties={}): +def CustomObject(type='x-custom-type', properties=None): """Custom STIX Object type decorator Example 1: - @CustomObject('x-type-name', { - 'property1': StringProperty(required=True), - 'property2': IntegerProperty(), - }) + @CustomObject('x-type-name', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) class MyNewObjectType(): pass Supply an __init__() function to add any special validations to the custom - type. Don't call super().__init() though - doing so will cause an error. + type. Don't call super().__init__() though - doing so will cause an error. Example 2: - @CustomObject('x-type-name', { - 'property1': StringProperty(required=True), - 'property2': IntegerProperty(), - }) + @CustomObject('x-type-name', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) class MyNewObjectType(): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: @@ -226,12 +318,31 @@ def CustomObject(type='x-custom-type', properties={}): class _Custom(cls, _STIXBase): _type = type - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - }) - _properties.update(properties) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ]) + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update([x for x in properties if not x[0].startswith("x_")]) + + # This is to follow the general properties structure. + _properties.update([ + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + # Put all custom properties at the bottom, sorted alphabetically. + _properties.update(sorted([x for x in properties if x[0].startswith("x_")], key=lambda x: x[0])) def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index f30e815..b50fd1d 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -1,411 +1,152 @@ """ -Python STIX 2.0 Composite Data Source and Data Source (classes) +Python STIX 2.0 Sources +Classes: + DataStore + DataSink + DataSource + STIXCommonPropertyFilters ----TODO/Questions--- +TODO:Test everything - -Test everything - - -add_filter(), remove_filter(), deduplicate() - if these functions remain - the exact same for both CompositeDataSource and DataSource, they just - inherit/have module access to +Notes: + add_filter(), remove_filter(), deduplicate() - if these functions remain + the exact same for DataSource, DataSink, CompositeDataSource etc... -> just + make those functions an interface to inherit? """ -import abc -import copy import uuid from six import iteritems +from stix2.sources.filters import (FILTER_OPS, FILTER_VALUE_TYPES, + STIX_COMMON_FIELDS, STIX_COMMON_FILTERS_MAP) + def make_id(): return str(uuid.uuid4()) -# STIX 2.0 fields used to denote object version -STIX_VERSION_FIELDS = ['id', 'modified'] +class DataStore(object): + """ + An implementer will create a concrete subclass from + this abstract class for the specific data store. -# Currently, only STIX 2.0 common SDO fields (that are not compex objects) -# are supported for filtering on -STIX_COMMON_FIELDS = [ - 'type', - 'id', - 'created_by_ref', - 'created', - 'modified', - 'revoked', - 'labels', - # 'external_references', # list of external references object type - not supported for filtering - 'object_references', - 'object_marking_refs', - 'granular_marking_refs', - # 'granular_markings' # list of granular-marking type - not supported for filtering -] - - -# Required fields in filter(dict) -FILTER_FIELDS = ['field', 'op', 'value'] - -# Supported filter operations -FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] - -# Supported filter value types -FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] - - -class CompositeDataSource(object): - """Composite Data Source - - Acts as a controller for all the defined/configured STIX Data Sources - e.g. a user can defined n Data Sources - creating Data Source (objects) - for each. There is only one instance of this for any python STIX 2.0 - application + Attributes: + id (str): A unique UUIDv4 to identify this DataStore. + source (DataStore): An object that implements DataStore class. + sink (DataSink): An object that implements DataSink class. """ + def __init__(self, source=None, sink=None): + self.id = make_id() + self.source = source + self.sink = sink - def __init__(self, name="CompositeDataSource"): + def get(self, stix_id): """ - Creates a new STIX Data Source. + Notes: + Translate API get() call to the appropriate DataSource call. Args: - 'data_sources' (dict): a dict of DataSource objects; to be - controlled and used by the Data Source Controller object - - filters : - name : - """ - self.id_ = make_id() - self.name = name - self.data_sources = {} - self.filters = {} - self.filter_allowed = {} - - def get(self, id_): - """Retrieve STIX object by 'id' - - federated retrieve method-iterates through all STIX data sources - defined in the "data_sources" parameter. Each data source has a - specific API retrieve-like function and associated parameters. This - function does a federated retrieval and consolidation of the data - returned from all the STIX data sources. - - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle - - Args: - id_ (str): the id of the STIX object to retrieve + stix_id (str): the id of the STIX 2.0 object to retrieve. Should + return a single object, the most recent version of the object + specified by the "id". Returns: - stix_obj (dict): the STIX object to be returned + stix_obj (dictionary): the STIX object to be returned """ + return self.source.get(stix_id) - all_data = [] - - # for every configured Data Source, call its retrieve handler - for ds_id, ds in iteritems(self.data_sources): - data = ds.get(id_=id_, _composite_filters=self.filters.values()) - all_data += data - - # remove duplicate versions - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - - return stix_obj - - def all_versions(self, id_): - """Retrieve STIX objects by 'id' - - Federated all_versions retrieve method - iterates through all STIX data - sources defined in "data_sources" - - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + def all_versions(self, stix_id): + """ + Implement: + Translate all_versions() call to the appropriate DataSource call Args: - id_ (str): id of the STIX objects to retrieve + stix_id (str): the id of the STIX 2.0 object to retrieve. Should + return a single object, the most recent version of the object + specified by the "id". Returns: - all_data (list): list of STIX objects that have the specified id + stix_objs (list): a list of STIX objects (where each object is a + STIX object) + """ - all_data = [] + return self.source.all_versions(stix_id) - # retrieve STIX objects from all configured data sources - for ds_id, ds in iteritems(self.data_sources): - data = ds.all_versions(id_=id_, _composite_filters=self.filters.values()) - all_data += data - - # remove exact duplicates (where duplicates are STIX 2.0 objects - # with the same 'id' and 'modified' values) - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - return all_data - - def query(self, query=None): - """composite data source query - - Federate the query to all Data Sources attached - to the Composite Data Source + def query(self, query): + """ + Notes: + Implement the specific data source API calls, processing, + functionality required for retrieving query from the data source. Args: - query (list): list of filters to search on + query (list): a list of filters (which collectively are the query) + to conduct search on. Returns: - all_data (list): list of STIX objects to be returned + stix_objs (list): a list of STIX objects (where each object is a + STIX object) """ - if not query: - query = [] + return self.source.query(query=query) - all_data = [] - - # federate query to all attached data sources, - # pass composite filters to them - for ds_id, ds in iteritems(self.data_sources): - data = ds.query(query=query, _composite_filters=self.filters.values()) - all_data += data - - # remove exact duplicates (where duplicates are STIX 2.0 - # objects with the same 'id' and 'modified' values) - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - return all_data - - def add_data_source(self, data_sources): - """add/attach Data Source to the Composite Data Source instance - - Args: - data_sources (list): a list of Data Source objects to attach - to the Composite Data Source - - Returns: + def add(self, stix_objs): + """ + Notes: + Translate add() to the appropriate DataSink call(). """ + return self.sink.add(stix_objs) - for ds in data_sources: - if issubclass(ds, DataSource): - if self.data_sources[ds['id']] in self.data_sources.keys(): - # data source already attached to Composite Data Source - continue - # add data source to Composite Data Source - # (its id will be its key identifier) - self.data_sources[ds['id']] = ds - else: - # the Data Source object is not a proper subclass - # of DataSource Abstract Class - # TODO: maybe log error? - continue +class DataSink(object): + """ + Abstract class for defining a data sink. Intended for subclassing into + different sink components. - return + Attributes: + id (str): A unique UUIDv4 to identify this DataSink. - def remove_data_source(self, data_source_ids): - """remove/detach Data Source from the Composite Data Source instance + """ + def __init__(self): + self.id = make_id() - Args: - data_source_ids (list): a list of Data Source - id's(which are strings) - - Returns: + def add(self, stix_objs): + """ + Notes: + Implement the specific data sink API calls, processing, + functionality required for adding data to the sink """ - - for id_ in data_source_ids: - try: - if self.data_sources[id_]: - del self.data_sources[id_] - except KeyError: - # Data Source 'id' was not found in CompositeDataSource's - # list of data sources - pass - return - - def get_data_sources(self): - """return all attached Data Sources - - TODO: Make this a property? - - Args: - - Returns: - - """ - return copy.deepcopy(self.data_sources.values()) - - def add_filter(self, filters): - """add/attach a filter to the Composite Data Source instance - - Args: - filters (list): list of filters (dict) to add to the Data Source - - Returns: - status (list): list of status/error messages - - """ - - status = [] - errors = [] - ids = [] - allowed = True - - for filter_ in filters: - # check required filter components ("field", "op", "value") exist - for field in FILTER_FIELDS: - if field not in filter_.keys(): - allowed = False - errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") - break - - if allowed: - # no need for further checks if filter is missing parameters - - # check filter field is a supported STIX 2.0 common field - if filter_['field'] not in STIX_COMMON_FIELDS: - allowed = False - errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - - # check filter operator is supported - if filter_['op'] not in FILTER_OPS: - allowed = False - errors.append("Filter operation(from 'op' field) not supported") - - # check filter value type is supported - if type(filter_['value']) not in FILTER_VALUE_TYPES: - allowed = False - errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - - # Filter is added regardless of whether it fits requirements - # to be a common filter. This is done because some filters - # may be added and used by third party Data Sources, where the - # filtering may be conducted within those plugins, just not here - - id_ = make_id() - filter_['id'] = id_ - self.filters['id_'] = filter_ - ids.append(id_) - - if allowed: - self.filter_allowed[id_] = True - status.append({ - "status": "added as a common filter", - "filter": filter_, - "data_source_name": self.name, - "data_source_id": self.id_ - }) - else: - self.filter_allowed[id_] = False - status.append({ - "status": "added but is not a common filter", - "filter": filter_, - "errors": errors, - "data_source_name": self.name, - "data_source_id": self.id_ - }) - del errors[:] - - allowed = True - - return ids, status - - def remove_filter(self, filter_ids): - """Remove/detach a filter from the Data Source instance - - Args: - filter_ids (list): list of filter id's (which are strings) - dettach from the Composite Data Source - - Returns: - - """ - - for filter_id in filter_ids: - try: - if filter_id in self.filters: - del self.filters[filter_id] - del self.filter_allowed[filter_id] - except KeyError: - # filter id not found in list of filters - # attached to the Composite Data Source - pass - - return - - def get_filters(self): - """return filters attached to Composite Data Source - - Args: - - Returns: - (list): the list of filters currently attached to the Data Source - - """ - return copy.deepcopy(list(self.filters.values())) - - def deduplicate(self, stix_obj_list): - """deduplicate a list fo STIX objects to a unique set - - Reduces a set of STIX objects to unique set by looking - at 'id' and 'modified' fields - as a unique object version - is determined by the combination of those fields - - Args: - stix_obj_list (list): list of STIX objects (dicts) - - Returns: - (list): unique set of the passed list of STIX objects - """ - - unique = [] - dont_have = False - for i in stix_obj_list: - dont_have = False - for j in unique: - for field in STIX_VERSION_FIELDS: - if not i[field] == j[field]: - dont_have = True - break - if dont_have: - unique.append(i) - return unique + raise NotImplementedError() class DataSource(object): """ - Abstract Data Source class for STIX 2.0 + Abstract class for defining a data source. Intended for subclassing into + different source components. - An implementer will create a concrete subclass from - this abstract class for the specific data source. + Attributes: + id (str): A unique UUIDv4 to identify this DataSource. + filters (set): A collection of filters present in this DataSource. - The purpose of the concrete subclasses is to then - supply them to a Composite Data Source which calls - the subclass methods when conducting STIX 2.0 - data retrievals. """ + def __init__(self): + self.id = make_id() + self.filters = set() - __metaclass__ = abc.ABCMeta - - def __init__(self, name="DataSource"): - self.name = name - self.id_ = make_id() - self.filters = {} - self.filter_allowed = {} - - @abc.abstractmethod - def get(self, id_, _composite_filters=None): + def get(self, stix_id, _composite_filters=None): """ Fill: - -implement the specific data source API calls, processing, + Implement the specific data source API calls, processing, functionality required for retrieving data from the data source Args: - - id_ (str): the id of the STIX 2.0 object to retrieve. Should + stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". @@ -418,18 +159,16 @@ class DataSource(object): """ raise NotImplementedError() - @abc.abstractmethod - def all_versions(self, id_, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None): """ - Fill: - -Similar to get() except returns list of all object versions of - the specified "id". - - -implement the specific data source API calls, processing, - functionality required for retrieving data from the data source + Notes: + Similar to get() except returns list of all object versions of + the specified "id". In addition, implement the specific data + source API calls, processing, functionality required for retrieving + data from the data source. Args: - id_ (str): The id of the STIX 2.0 object to retrieve. Should + stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". @@ -439,12 +178,10 @@ class DataSource(object): Returns: stix_objs (list): a list of STIX objects (where each object is a STIX object) + """ - stix_objs = [] + raise NotImplementedError() - return stix_objs - - @abc.abstractmethod def query(self, query, _composite_filters=None): """ Fill: @@ -460,138 +197,36 @@ class DataSource(object): Returns: - """ - stix_objs = [] + raise NotImplementedError() - return stix_objs - - @abc.abstractmethod - def close(self): - """ - Fill: - Close, release, shutdown any objects, contexts, variables - Args: - - Returns: - (list): list of status/error messages - """ - - status = [] - - return status - - def add_filter(self, filters): - """add/attach a filter to the Data Source instance + def add_filters(self, filters): + """Add multiple filters to the DataSource. Args: - filters (list): list of filters (dict) to add to the Data Source - - Returns: - status (list): list of status/error messages - + filters (list): list of filters (dict) to add to the Data Source. """ + for filter in filters: + self.add_filter(filter) - status = [] - errors = [] - ids = [] - allowed = True + def add_filter(self, filter): + """Add a filter.""" + # check filter field is a supported STIX 2.0 common field + if filter.field not in STIX_COMMON_FIELDS: + raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - for filter_ in filters: - # check required filter components ("field", "op", "value") exist - for field in FILTER_FIELDS: - if field not in filter_.keys(): - allowed = False - errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") - break + # check filter operator is supported + if filter.op not in FILTER_OPS: + raise ValueError("Filter operation (from 'op' field) not supported") - if allowed: - # no reason for further checks if missing filter parameters + # check filter value type is supported + if type(filter.value) not in FILTER_VALUE_TYPES: + raise ValueError("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - # check filter field is a supported STIX 2.0 common field - if filter_['field'] not in STIX_COMMON_FIELDS: - allowed = False - errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - - # check filter operator is supported - if filter_['op'] not in FILTER_OPS: - allowed = False - errors.append("Filter operation(from 'op' field) not supported") - - # check filter value type is supported - if type(filter_['value']) not in FILTER_VALUE_TYPES: - allowed = False - errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - - # Filter is added regardless of whether it fits requirements - # to be a common filter. This is done because some filters - # may be added and used by third party Data Sources, where the - # filtering may be conducted within those plugins, just not here - - id_ = make_id() - filter_['id'] = id_ - self.filters[id_] = filter_ - ids.append(id_) - - if allowed: - self.filter_allowed[id_] = True - status.append({ - "status": "added as a common filter", - "filter": filter_, - "data_source_name": self.name, - "data_source_id": self.id_ - }) - else: - self.filter_allowed[id_] = False - status.append({ - "status": "added but is not a common filter", - "filter": filter_, - "errors": copy.deepcopy(errors), - "data_source_name": self.name, - "data_source_id": self.id_ - }) - del errors[:] - - allowed = True - - return ids, status - - def remove_filter(self, filter_ids): - """remove/detach a filter from the Data Source instance - - Args: - filter_ids (list): list of filter ids to dettach/remove - from Data Source - - Returns: - - - """ - for filter_id in filter_ids: - try: - if filter_id in self.filters: - del self.filters[filter_id] - del self.filter_allowed[filter_id] - except KeyError: - # filter 'id' not found list of filters attached to Data Source - pass - - return - - def get_filters(self): - """return copy of all filters currently attached to Data Source - - TODO: make this a property? - - Returns: - (list): a copy of all the filters(dict) which are attached - to Data Source - - """ - return copy.deepcopy(list(self.filters.values())) + self.filters.add(filter) def apply_common_filters(self, stix_objs, query): - """evaluates filters against a set of STIX 2.0 objects + """Evaluates filters against a set of STIX 2.0 objects Supports only STIX 2.0 common property fields @@ -601,87 +236,51 @@ class DataSource(object): Returns: (list): list of STIX objects that successfully evaluate against - the query + the query. """ - filtered_stix_objs = [] # evaluate objects against filter for stix_obj in stix_objs: clean = True for filter_ in query: - # skip filter as filter was identified (when added) as # not a common filter - if 'id' in filter_ and self.filter_allowed[filter_['id']] is False: - continue + if filter_.field not in STIX_COMMON_FIELDS: + raise ValueError("Error, field: {0} is not supported for filtering on.".format(filter_.field)) - # check filter "field" is in STIX object - if cant be applied - # due to STIX object, STIX object is discarded (i.e. did not - # make it through the filter) - if filter_['field'] not in stix_obj.keys(): + # For properties like granular_markings and external_references + # need to break the first property from the string. + if "." in filter_.field: + field = filter_.field.split(".")[0] + else: + field = filter_.field + + # check filter "field" is in STIX object - if cant be + # applied due to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + if field not in stix_obj.keys(): + clean = False break - try: + match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) + if not match: + clean = False + break + elif match == -1: + raise ValueError("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) - if filter_['op'] == '=': - if not stix_obj[filter_['field']] == filter_['value']: - clean = False - break - elif filter_['op'] == "!=": - if not stix_obj[filter_['field']] != filter_['value']: - clean = False - break - elif filter_['op'] == "in": - if not stix_obj[filter_['field']] in filter_['value']: - clean = False - break - else: - # filter operation not supported - continue - - # TODO: I think the rest of the operations only - # apply to timestamps, in which case I don't think - # simple operator usage (like below) works - - # elif filter_['op'] == ">": - # if not stix_obj[filter_['field']] > filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == "<": - # if not stix_obj[filter_['field']] < filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == ">=": - # if not stix_obj[filter_['field']] >= filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == "<=": - # if not stix_obj[filter_['field']] <= filter_['value']: - # clean = False - # break - - except TypeError: - # type mismatch of comparison operands - ignore filter, - # no error raised for now - pass - - # if object unmarked after all filter, add it + # if object unmarked after all filters, add it if clean: filtered_stix_objs.append(stix_obj) - clean = True - return filtered_stix_objs def deduplicate(self, stix_obj_list): - """deduplicate a list of STIX objects into a unique set + """Deduplicate a list of STIX objects to a unique set - reduces a set of STIX objects to unique set by looking + Reduces a set of STIX objects to unique set by looking at 'id' and 'modified' fields - as a unique object version is determined by the combination of those fields @@ -689,18 +288,199 @@ class DataSource(object): stix_obj_list (list): list of STIX objects (dicts) Returns: - (list): a unique set of the passed STIX object list - + A list with a unique set of the passed list of STIX objects. """ - unique = [] - have = False - for i in stix_obj_list: - for j in unique: - if i['id'] == j['id'] and i['modified'] == j['modified']: - have = True - break - if not have: - unique.append(i) - have = False - return unique + unique_objs = {} + + for obj in stix_obj_list: + unique_objs[(obj['id'], obj['modified'])] = obj + + return list(unique_objs.values()) + + +class CompositeDataSource(DataSource): + """Composite Data Source + + Acts as a controller for all the defined/configured STIX Data Sources + e.g. a user can define n Data Sources - creating Data Source (objects) + for each. There is only one instance of this for any python STIX 2.0 + application. + + Attributes: + name (str): The name that identifies this CompositeDataSource. + data_sources (dict): A dictionary of DataSource objects; to be + controlled and used by the Data Source Controller object. + + """ + def __init__(self): + """ + Creates a new STIX Data Source. + + Args: + name (str): A string containing the name to attach in the + CompositeDataSource instance. + + """ + super(CompositeDataSource, self).__init__() + self.data_sources = {} + + def get(self, stix_id, _composite_filters=None): + """Retrieve STIX object by 'id' + + Federated retrieve method-iterates through all STIX data sources + defined in the "data_sources" parameter. Each data source has a + specific API retrieve-like function and associated parameters. This + function does a federated retrieval and consolidation of the data + returned from all the STIX data sources. + + Notes: + A composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle. + + Args: + stix_id (str): the id of the STIX object to retrieve. + + _composite_filters (list): a list of filters passed from the + Composite Data Source + + Returns: + stix_obj (dict): the STIX object to be returned. + + """ + all_data = [] + + # for every configured Data Source, call its retrieve handler + for ds_id, ds in iteritems(self.data_sources): + data = ds.get(stix_id=stix_id, _composite_filters=list(self.filters)) + all_data.append(data) + + # remove duplicate versions + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0] + + return stix_obj + + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve STIX objects by 'id' + + Federated all_versions retrieve method - iterates through all STIX data + sources defined in "data_sources" + + Notes: + A composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle + + Args: + stix_id (str): id of the STIX objects to retrieve + + _composite_filters (list): a list of filters passed from the + Composite Data Source + + Returns: + all_data (list): list of STIX objects that have the specified id + + """ + all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) + + # retrieve STIX objects from all configured data sources + for ds_id, ds in iteritems(self.data_sources): + data = ds.all_versions(stix_id=stix_id, _composite_filters=list(all_filters)) + all_data.extend(data) + + # remove exact duplicates (where duplicates are STIX 2.0 objects + # with the same 'id' and 'modified' values) + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + return all_data + + def query(self, query=None, _composite_filters=None): + """Composite data source query + + Federate the query to all Data Sources attached to the + Composite Data Source. + + Args: + query (list): list of filters to search on. + + _composite_filters (list): a list of filters passed from the + Composite Data Source + + Returns: + all_data (list): list of STIX objects to be returned + + """ + if not query: + query = [] + + all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) + + # federate query to all attached data sources, + # pass composite filters to id + for ds_id, ds in iteritems(self.data_sources): + data = ds.query(query=query, _composite_filters=list(all_filters)) + all_data.extend(data) + + # remove exact duplicates (where duplicates are STIX 2.0 + # objects with the same 'id' and 'modified' values) + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + return all_data + + def add_data_source(self, data_sources): + """Add/attach Data Source to the Composite Data Source instance + + Args: + data_sources (list): a list of Data Source objects to attach + to the Composite Data Source + + """ + for ds in data_sources: + if issubclass(ds.__class__, DataSource): + if ds.id in self.data_sources: + # data source already attached to Composite Data Source + continue + + # add data source to Composite Data Source + # (its id will be its key identifier) + self.data_sources[ds.id] = ds + else: + # the Data Source object is not a proper subclass + # of DataSource Abstract Class + # TODO: maybe log error? + continue + + return + + def remove_data_source(self, data_source_ids): + """Remove/detach Data Source from the Composite Data Source instance + + Args: + data_source_ids (list): a list of Data Source identifiers. + + """ + for id in data_source_ids: + if id in self.data_sources: + del self.data_sources[id] + else: + raise ValueError("DataSource 'id' not found in CompositeDataSource collection.") + return + + def get_all_data_sources(self): + """Return all attached Data Sources + + """ + return self.data_sources.values() diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py new file mode 100644 index 0000000..ade36c8 --- /dev/null +++ b/stix2/sources/filesystem.py @@ -0,0 +1,188 @@ +""" +Python STIX 2.0 FileSystem Source/Sink + +Classes: + FileSystemStore + FileSystemSink + FileSystemSource + +TODO: Test everything +""" + +import json +import os + +from stix2 import Bundle +from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources.filters import Filter + + +class FileSystemStore(DataStore): + """ + """ + def __init__(self, stix_dir="stix_data"): + super(FileSystemStore, self).__init__() + self.source = FileSystemSource(stix_dir=stix_dir) + self.sink = FileSystemSink(stix_dir=stix_dir) + + +class FileSystemSink(DataSink): + """ + """ + def __init__(self, stix_dir="stix_data"): + super(FileSystemSink, self).__init__() + self.stix_dir = os.path.abspath(stix_dir) + + # check directory path exists + if not os.path.exists(self.stix_dir): + print("Error: directory path for STIX data does not exist") + + @property + def stix_dir(self): + return self.stix_dir + + @stix_dir.setter + def stix_dir(self, dir): + self.stix_dir = dir + + def add(self, stix_objs=None): + """ + Q: bundlify or no? + """ + if not stix_objs: + stix_objs = [] + for stix_obj in stix_objs: + path = os.path.join(self.stix_dir, stix_obj["type"], stix_obj["id"]) + json.dump(Bundle([stix_obj]), open(path, 'w+'), indent=4) + + +class FileSystemSource(DataSource): + """ + """ + def __init__(self, stix_dir="stix_data"): + super(FileSystemSource, self).__init__() + self.stix_dir = os.path.abspath(stix_dir) + + # check directory path exists + if not os.path.exists(self.stix_dir): + print("Error: directory path for STIX data does not exist") + + @property + def stix_dir(self): + return self.stix_dir + + @stix_dir.setter + def stix_dir(self, dir): + self.stix_dir = dir + + def get(self, stix_id, _composite_filters=None): + """ + """ + query = [Filter("id", "=", stix_id)] + + all_data = self.query(query=query, _composite_filters=_composite_filters) + + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + + return stix_obj + + def all_versions(self, stix_id, _composite_filters=None): + """ + Notes: + Since FileSystem sources/sinks don't handle multiple versions + of a STIX object, this operation is unnecessary. Pass call to get(). + + """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + + def query(self, query=None, _composite_filters=None): + """ + """ + all_data = [] + + if query is None: + query = [] + + # combine all query filters + if self.filters: + query.extend(self.filters.values()) + if _composite_filters: + query.extend(_composite_filters) + + # extract any filters that are for "type" or "id" , as we can then do + # filtering before reading in the STIX objects. A STIX 'type' filter + # can reduce the query to a single sub-directory. A STIX 'id' filter + # allows for the fast checking of the file names versus loading it. + file_filters = self._parse_file_filters(query) + + # establish which subdirectories can be avoided in query + # by decluding as many as possible. A filter with "type" as the field + # means that certain STIX object types can be ruled out, and thus + # the corresponding subdirectories as well + include_paths = [] + declude_paths = [] + if "type" in [filter.field for filter in file_filters]: + for filter in file_filters: + if filter.field == "type": + if filter.op == "=": + include_paths.append(os.path.join(self.stix_dir, filter.value)) + elif filter.op == "!=": + declude_paths.append(os.path.join(self.stix_dir, filter.value)) + else: + # have to walk entire STIX directory + include_paths.append(self.stix_dir) + + # if a user specifies a "type" filter like "type = ", + # the filter is reducing the search space to single stix object types + # (and thus single directories). This makes such a filter more powerful + # than "type != " bc the latter is substracting + # only one type of stix object type (and thus only one directory), + # As such the former type of filters are given preference over the latter; + # i.e. if both exist in a query, that latter type will be ignored + + if not include_paths: + # user has specified types that are not wanted (i.e. "!=") + # so query will look in all STIX directories that are not + # the specified type. Compile correct dir paths + for dir in os.listdir(self.stix_dir): + if os.path.abspath(dir) not in declude_paths: + include_paths.append(os.path.abspath(dir)) + + # grab stix object ID as well - if present in filters, as + # may forgo the loading of STIX content into memory + if "id" in [filter.field for filter in file_filters]: + for filter in file_filters: + if filter.field == "id" and filter.op == "=": + id = filter.value + break + else: + id = None + else: + id = None + + # now iterate through all STIX objs + for path in include_paths: + for root, dirs, files in os.walk(path): + for file in files: + if id: + if id == file.split(".")[0]: + # since ID is specified in one of filters, can evaluate against filename first without loading + stix_obj = json.load(file)["objects"] + # check against other filters, add if match + all_data.extend(self.apply_common_filters([stix_obj], query)) + else: + # have to load into memory regardless to evaluate other filters + stix_obj = json.load(file)["objects"] + all_data.extend(self.apply_common_filters([stix_obj], query)) + + all_data = self.deduplicate(all_data) + return all_data + + def _parse_file_filters(self, query): + """ + """ + file_filters = [] + for filter in query: + if filter.field == "id" or filter.field == "type": + file_filters.append(filter) + return file_filters diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py new file mode 100644 index 0000000..7758369 --- /dev/null +++ b/stix2/sources/filters.py @@ -0,0 +1,204 @@ +""" +Filters for Python STIX 2.0 DataSources, DataSinks, DataStores + +Classes: + Filter + +TODO: The script at the bottom of the module works (to capture +all the callable filter methods), however it causes this module +to be imported by itself twice. Not sure how big of deal that is, +or if cleaner solution possible. +""" + +import collections +import types + +# Currently, only STIX 2.0 common SDO fields (that are not complex objects) +# are supported for filtering on +STIX_COMMON_FIELDS = [ + "created", + "created_by_ref", + "external_references.source_name", + "external_references.description", + "external_references.url", + "external_references.hashes", + "external_references.external_id", + "granular_markings.marking_ref", + "granular_markings.selectors", + "id", + "labels", + "modified", + "object_marking_refs", + "revoked", + "type", + "granular_markings" +] + +# Supported filter operations +FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] + +# Supported filter value types +FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] + +# filter lookup map - STIX 2 common fields -> filter method +STIX_COMMON_FILTERS_MAP = {} + + +class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): + __slots__ = () + + def __new__(cls, field, op, value): + # If value is a list, convert it to a tuple so it is hashable. + if isinstance(value, list): + value = tuple(value) + self = super(Filter, cls).__new__(cls, field, op, value) + return self + + +# primitive type filters + +def _all_filter(filter_, stix_obj_field): + """all filter operations (for filters whose value type can be applied to any operation type)""" + if filter_.op == "=": + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value + elif filter_.op == "in": + return stix_obj_field in filter_.value + elif filter_.op == ">": + return stix_obj_field > filter_.value + elif filter_.op == "<": + return stix_obj_field < filter_.value + elif filter_.op == ">=": + return stix_obj_field >= filter_.value + elif filter_.op == "<=": + return stix_obj_field <= filter_.value + else: + return -1 + + +def _id_filter(filter_, stix_obj_id): + """base filter types""" + if filter_.op == "=": + return stix_obj_id == filter_.value + elif filter_.op == "!=": + return stix_obj_id != filter_.value + else: + return -1 + + +def _boolean_filter(filter_, stix_obj_field): + if filter_.op == "=": + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value + else: + return -1 + + +def _string_filter(filter_, stix_obj_field): + return _all_filter(filter_, stix_obj_field) + + +def _timestamp_filter(filter_, stix_obj_timestamp): + return _all_filter(filter_, stix_obj_timestamp) + +# STIX 2.0 Common Property filters +# The naming of these functions is important as +# they are used to index a mapping dictionary from +# STIX common field names to these filter functions. +# +# REQUIRED naming scheme: +# "check__filter" + + +def check_created_filter(filter_, stix_obj): + return _timestamp_filter(filter_, stix_obj["created"]) + + +def check_created_by_ref_filter(filter_, stix_obj): + return _id_filter(filter_, stix_obj["created_by_ref"]) + + +def check_external_references_filter(filter_, stix_obj): + """ + STIX object's can have a list of external references + + external_references properties: + external_references.source_name (string) + external_references.description (string) + external_references.url (string) + external_references.hashes (hash, but for filtering purposes, a string) + external_references.external_id (string) + + """ + for er in stix_obj["external_references"]: + # grab er property name from filter field + filter_field = filter_.field.split(".")[1] + r = _string_filter(filter_, er[filter_field]) + if r: + return r + return False + + +def check_granular_markings_filter(filter_, stix_obj): + """ + STIX object's can have a list of granular marking references + + granular_markings properties: + granular_markings.marking_ref (id) + granular_markings.selectors (string) + + """ + for gm in stix_obj["granular_markings"]: + # grab gm property name from filter field + filter_field = filter_.field.split(".")[1] + + if filter_field == "marking_ref": + return _id_filter(filter_, gm[filter_field]) + + elif filter_field == "selectors": + for selector in gm[filter_field]: + r = _string_filter(filter_, selector) + if r: + return r + return False + + +def check_id_filter(filter_, stix_obj): + return _id_filter(filter_, stix_obj["id"]) + + +def check_labels_filter(filter_, stix_obj): + for label in stix_obj["labels"]: + r = _string_filter(filter_, label) + if r: + return r + return False + + +def check_modified_filter(filter_, stix_obj): + return _timestamp_filter(filter_, stix_obj["modified"]) + + +def check_object_marking_refs_filter(filter_, stix_obj): + for marking_id in stix_obj["object_marking_refs"]: + r = _id_filter(filter_, marking_id) + if r: + return r + return False + + +def check_revoked_filter(filter_, stix_obj): + return _boolean_filter(filter_, stix_obj["revoked"]) + + +def check_type_filter(filter_, stix_obj): + return _string_filter(filter_, stix_obj["type"]) + + +# Create mapping of field names to filter functions +for name, obj in dict(globals()).items(): + if "check_" in name and isinstance(obj, types.FunctionType): + field_name = "_".join(name.split("_")[1:-1]) + STIX_COMMON_FILTERS_MAP[field_name] = obj diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py new file mode 100644 index 0000000..9eca969 --- /dev/null +++ b/stix2/sources/memory.py @@ -0,0 +1,202 @@ +""" +Python STIX 2.0 Memory Source/Sink + +Classes: + MemoryStore + MemorySink + MemorySource + +TODO: Test everything. + +TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) + can save a lot of time for successive queries + +Notes: + Not worrying about STIX versioning. The in memory STIX data at anytime + will only hold one version of a STIX object. As such, when save() is called, + the single versions of all the STIX objects are what is written to file. + +""" + +import collections +import json +import os + +from stix2validator import validate_instance + +from stix2 import Bundle +from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources.filters import Filter + + +def _add(store, stix_data): + """Adds stix objects to MemoryStore/Source/Sink.""" + if isinstance(stix_data, collections.Mapping): + # stix objects are in a bundle + # verify STIX json data + r = validate_instance(stix_data) + # make dictionary of the objects for easy lookup + if r.is_valid: + for stix_obj in stix_data["objects"]: + store.data[stix_obj["id"]] = stix_obj + else: + raise ValueError("Error: data passed was found to not be valid by the STIX 2 Validator: \n%s", r.as_dict()) + elif isinstance(stix_data, list): + # stix objects are in a list + for stix_obj in stix_data: + r = validate_instance(stix_obj) + if r.is_valid: + store.data[stix_obj["id"]] = stix_obj + else: + raise ValueError("Error: STIX object %s is not valid under STIX 2 validator.\n%s", stix_obj["id"], r) + else: + raise ValueError("stix_data must be in bundle format or raw list") + + +class MemoryStore(DataStore): + """ + """ + def __init__(self, stix_data): + """ + Notes: + It doesn't make sense to create a MemoryStore by passing + in existing MemorySource and MemorySink because there could + be data concurrency issues. Just as easy to create new MemoryStore. + + """ + super(MemoryStore, self).__init__() + self.data = {} + + if stix_data: + _add(self, stix_data) + + self.source = MemorySource(stix_data=self.data, _store=True) + self.sink = MemorySink(stix_data=self.data, _store=True) + + def save_to_file(self, file_path): + return self.sink.save_to_file(file_path=file_path) + + def load_from_file(self, file_path): + return self.source.load_from_file(file_path=file_path) + + +class MemorySink(DataSink): + """ + """ + def __init__(self, stix_data, _store=False): + """ + Args: + stix_data (dictionary OR list): valid STIX 2.0 content in + bundle or a list. + _store (bool): if the MemorySink is a part of a DataStore, + in which case "stix_data" is a direct reference to + shared memory with DataSource. + + """ + super(MemorySink, self).__init__() + self.data = {} + + if _store: + self.data = stix_data + elif stix_data: + self.add(stix_data) + + def add(self, stix_data): + """ + """ + _add(self, stix_data) + + def save_to_file(self, file_path): + """ + """ + json.dump(Bundle(self.data.values()), file_path, indent=4) + + +class MemorySource(DataSource): + + def __init__(self, stix_data, _store=False): + """ + Args: + stix_data (dictionary OR list): valid STIX 2.0 content in + bundle or list. + _store (bool): if the MemorySource is a part of a DataStore, + in which case "stix_data" is a direct reference to shared + memory with DataSink. + + """ + super(MemorySource, self).__init__() + self.data = {} + + if _store: + self.data = stix_data + elif stix_data: + _add(self, stix_data) + + def get(self, stix_id, _composite_filters=None): + """ + """ + if _composite_filters is None: + # if get call is only based on 'id', no need to search, just retrieve from dict + try: + stix_obj = self.data[stix_id] + except KeyError: + stix_obj = None + return stix_obj + + # if there are filters from the composite level, process full query + query = [Filter("id", "=", stix_id)] + + all_data = self.query(query=query, _composite_filters=_composite_filters) + + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + + return stix_obj + + def all_versions(self, stix_id, _composite_filters=None): + """ + Notes: + Since Memory sources/sinks don't handle multiple versions of a + STIX object, this operation is unnecessary. Translate call to get(). + + Args: + stix_id (str): The id of the STIX 2.0 object to retrieve. Should + return a list of objects, all the versions of the object + specified by the "id". + + Returns: + (list): STIX object that matched ``stix_id``. + + """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + + def query(self, query=None, _composite_filters=None): + """ + """ + if query is None: + query = [] + + # combine all query filters + if self.filters: + query.extend(list(self.filters)) + if _composite_filters: + query.extend(_composite_filters) + + # Apply STIX common property filters. + all_data = self.apply_common_filters(self.data.values(), query) + + return all_data + + def load_from_file(self, file_path): + """ + """ + file_path = os.path.abspath(file_path) + stix_data = json.load(open(file_path, "r")) + + r = validate_instance(stix_data) + + if r.is_valid: + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj + + raise ValueError("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator.\n%s", file_path, r) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index c83d220..aa9bd11 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,132 +1,97 @@ -import requests -from requests.auth import HTTPBasicAuth +""" +Python STIX 2.0 TAXII Source/Sink -from stix2.sources import DataSource +Classes: + TAXIICollectionStore + TAXIICollectionSink + TAXIICollectionSource -# TODO: -Should we make properties for the TAXIIDataSource address and other -# possible variables that are found in "self.taxii_info" +TODO: Test everything +""" + +import json + +from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources.filters import Filter TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] -test = True - -class TAXIIDataSource(DataSource): - """STIX 2.0 Data Source - TAXII 2.0 module""" - - def __init__(self, api_root=None, auth=None, name="TAXII"): - super(TAXIIDataSource, self).__init__(name=name) - - if not api_root: - api_root = "http://localhost:5000" - if not auth: - auth = {"user": "admin", "pass": "taxii"} - - self.taxii_info = { - "api_root": { - "url": api_root - }, - "auth": auth - } - - if test: - return - - try: - # check api-root is reachable/exists and grab api collections - coll_url = self.taxii_info['api_root']['url'] + "/collections/" - headers = {} - - resp = requests.get(coll_url, - headers=headers, - auth=HTTPBasicAuth(self.taxii_info['auth']['user'], - self.taxii_info['auth']['pass'])) - # TESTING - # print("\n-------__init__() ----\n") - # print(resp.text) - # print("\n") - # print(resp.status_code) - # END TESTING - - # raise http error if request returned error code - resp.raise_for_status() - - resp_json = resp.json() - - try: - self.taxii_info['api_root']['collections'] = resp_json['collections'] - except KeyError as e: - if e == "collections": - raise - # raise type(e), type(e)(e.message + - # "To connect to the TAXII collections, the API root - # resource must contain a collection endpoint URL. - # This was not found in the API root resource received - # from the API root" ), sys.exc_info()[2] - - except requests.ConnectionError as e: - raise - # raise type(e), type(e)(e.message + - # "Attempting to connect to %s" % coll_url) - - def get(self, id_, _composite_filters=None): - """Get STIX 2.0 object from TAXII source by specified 'id' - - Notes: - Just pass _composite_filters to the query() as they are applied - there. de-duplication of results is also done within query() +class TAXIICollectionStore(DataStore): + """ + """ + def __init__(self, collection): + """ + Create a new TAXII Collection Data store Args: - id_ (str): id of STIX object to retrieve - - _composite_filters (list): filters passed from a Composite Data - Source (if this data source is attached to one) - - Returns: + collection (taxii2.Collection): Collection instance """ + super(TAXIICollectionStore, self).__init__() + self.source = TAXIICollectionSource(collection) + self.sink = TAXIICollectionSink(collection) - # make query in TAXII query format since 'id' is TAXii field - query = [ - { - "field": "match[id]", - "op": "=", - "value": id_ - } - ] - all_data = self.query(query=query, _composite_filters=_composite_filters) +class TAXIICollectionSink(DataSink): + """ + """ + def __init__(self, collection): + super(TAXIICollectionSink, self).__init__() + self.collection = collection - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + def add(self, stix_obj): + """ + """ + self.collection.add_objects(self.create_bundle([json.loads(str(stix_obj))])) + + @staticmethod + def create_bundle(objects): + return dict(id="bundle--%s" % make_id(), + objects=objects, + spec_version="2.0", + type="bundle") + + +class TAXIICollectionSource(DataSource): + """ + """ + def __init__(self, collection): + super(TAXIICollectionSource, self).__init__() + self.collection = collection + + def get(self, stix_id, _composite_filters=None): + """ + """ + # combine all query filters + query = [] + if self.filters: + query.extend(self.filters.values()) + if _composite_filters: + query.extend(_composite_filters) + + # separate taxii query terms (can be done remotely) + taxii_filters = self._parse_taxii_filters(query) + + stix_objs = self.collection.get_object(stix_id, taxii_filters)["objects"] + + stix_obj = self.apply_common_filters(stix_objs, query) + + if len(stix_obj) > 0: + stix_obj = stix_obj[0] + else: + stix_obj = None return stix_obj - def all_versions(self, id_, _composite_filters=None): - """Get all versions of STIX 2.0 object from TAXII source by - specified 'id' - - Notes: - Just passes _composite_filters to the query() as they are applied - there. de-duplication of results is also done within query() - - Args: - id_ (str): id of STIX objects to retrieve - _composite_filters (list): filters passed from a Composite Data - Source (if this data source is attached to one) - - Returns: - The query results with filters applied. + def all_versions(self, stix_id, _composite_filters=None): + """ """ - # make query in TAXII query format since 'id' is TAXII field query = [ - { - "field": "match[id]", - "op": "=", - "value": id_ - } + Filter("match[id]", "=", stix_id), + Filter("match[version]", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -134,84 +99,22 @@ class TAXIIDataSource(DataSource): return all_data def query(self, query=None, _composite_filters=None): - """Query the TAXII data source for STIX objects matching the query - - The final full query could contain filters from: - -the current API call - -Composite Data source filters (that are passed in via - '_composite_filters') - -TAXII data source filters that are attached - - TAXII filters ['added_after', 'match[<>]'] are extracted and sent - to TAXII if they are present - - TODO: Authentication for TAXII - - Args: - - query(list): list of filters (dicts) to search on - - _composite_filters (list): filters passed from a - Composite Data Source (if this data source is attached to one) - - Returns: - - """ - - all_data = [] - + """ if query is None: query = [] # combine all query filters if self.filters: - query += self.filters.values() + query.extend(self.filters.values()) if _composite_filters: - query += _composite_filters + query.extend(_composite_filters) # separate taxii query terms (can be done remotely) taxii_filters = self._parse_taxii_filters(query) - # for each collection endpoint - send query request - for collection in self.taxii_info['api_root']['collections']: - - coll_obj_url = "/".join([self.taxii_info['api_root']['url'], - "collections", str(collection['id']), - "objects"]) - headers = {} - try: - resp = requests.get(coll_obj_url, - params=taxii_filters, - headers=headers, - auth=HTTPBasicAuth(self.taxii_info['auth']['user'], - self.taxii_info['auth']['pass'])) - # TESTING - # print("\n-------query() ----\n") - # print("Request that was sent: \n") - # print(resp.url) - # print("Response: \n") - # print(json.dumps(resp.json(),indent=4)) - # print("\n") - # print(resp.status_code) - # print("------------------") - # END TESTING - - # raise http error if request returned error code - resp.raise_for_status() - resp_json = resp.json() - - # grab all STIX 2.0 objects in json response - for stix_obj in resp_json['objects']: - all_data.append(stix_obj) - - except requests.exceptions.RequestException as e: - raise e - # raise type(e), type(e)(e.message + - # "Attempting to connect to %s" % coll_url) - - # TODO: Is there a way to collect exceptions while carrying - # on then raise all of them at the end? + # query TAXII collection + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = self.deduplicate(all_data) @@ -222,16 +125,13 @@ class TAXIIDataSource(DataSource): return all_data def _parse_taxii_filters(self, query): - """Parse out TAXII filters that the TAXII server can filter on + """Parse out TAXII filters that the TAXII server can filter on. - TAXII filters should be analgous to how they are supplied - in the url to the TAXII endpoint. For instance - "?match[type]=indicator,sighting" should be in a query dict as follows - { - "field": "match[type]" - "op": "=", - "value": "indicator,sighting" - } + Notes: + For instance - "?match[type]=indicator,sighting" should be in a + query dict as follows: + + Filter("type", "=", "indicator,sighting") Args: query (list): list of filters to extract which ones are TAXII @@ -240,23 +140,15 @@ class TAXIIDataSource(DataSource): Returns: params (dict): dict of the TAXII filters but in format required for 'requests.get()'. - """ + """ params = {} - for q in query: - if q['field'] in TAXII_FILTERS: - if q['field'] == 'added_after': - params[q['field']] = q['value'] + for filter_ in query: + if filter_.field in TAXII_FILTERS: + if filter_.field == "added_after": + params[filter_.field] = filter_.value else: - taxii_field = 'match[' + q['field'] + ']' - params[taxii_field] = q['value'] + taxii_field = "match[%s]" % filter_.field + params[taxii_field] = filter_.value return params - - def close(self): - """Close down the Data Source - if any clean up is required. - - """ - pass - - # TODO: - getters/setters (properties) for TAXII config info diff --git a/stix2/sro.py b/stix2/sro.py index c13fff3..af483bc 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -1,31 +1,39 @@ """STIX 2.0 Relationship Objects.""" +from collections import OrderedDict + from .base import _STIXBase -from .common import COMMON_PROPERTIES -from .properties import (IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) +from .common import ExternalReference, GranularMarking +from .properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) +from .utils import NOW class Relationship(_STIXBase): _type = 'relationship' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - 'relationship_type': StringProperty(required=True), - 'description': StringProperty(), - 'source_ref': ReferenceProperty(required=True), - 'target_ref': ReferenceProperty(required=True), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('relationship_type', StringProperty(required=True)), + ('description', StringProperty()), + ('source_ref', ReferenceProperty(required=True)), + ('target_ref', ReferenceProperty(required=True)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) # Explicitly define the first three kwargs to make readable Relationship declarations. - def __init__(self, source_ref=None, relationship_type=None, target_ref=None, - **kwargs): - # TODO: - # - description - + def __init__(self, source_ref=None, relationship_type=None, + target_ref=None, **kwargs): # Allow (source_ref, relationship_type, target_ref) as positional args. if source_ref and not kwargs.get('source_ref'): kwargs['source_ref'] = source_ref @@ -39,24 +47,29 @@ class Relationship(_STIXBase): class Sighting(_STIXBase): _type = 'sighting' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - 'first_seen': TimestampProperty(), - 'last_seen': TimestampProperty(), - 'count': IntegerProperty(), - 'sighting_of_ref': ReferenceProperty(required=True), - 'observed_data_refs': ListProperty(ReferenceProperty(type="observed-data")), - 'where_sighted_refs': ListProperty(ReferenceProperty(type="identity")), - 'summary': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('count', IntegerProperty()), + ('sighting_of_ref', ReferenceProperty(required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))), + ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))), + ('summary', BooleanProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) # Explicitly define the first kwargs to make readable Sighting declarations. def __init__(self, sighting_of_ref=None, **kwargs): - # TODO: - # - description - # Allow sighting_of_ref as a positional arg. if sighting_of_ref and not kwargs.get('sighting_of_ref'): kwargs['sighting_of_ref'] = sighting_of_ref diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 958120b..839b547 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -20,6 +20,26 @@ TOOL_ID = "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb" VULNERABILITY_ID = "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" +MARKING_IDS = [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "marking-definition--443eb5c3-a76c-4a0a-8caa-e93998e7bc09", + "marking-definition--57fcd772-9c1d-41b0-8d1f-3d47713415d9", + "marking-definition--462bf1a6-03d2-419c-b74e-eee2238b2de4", + "marking-definition--68520ae2-fefe-43a9-84ee-2c2a934d2c7d", + "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f", +] + +# All required args for a Campaign instance, plus some optional args +CAMPAIGN_MORE_KWARGS = dict( + type='campaign', + id=CAMPAIGN_ID, + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T20:03:00.000Z", + modified="2016-04-06T20:03:00.000Z", + name="Green Group Attacks Against Finance", + description="Campaign by Green Group against a series of targets in the financial services sector.", +) + # Minimum required args for an Identity instance IDENTITY_KWARGS = dict( name="John Smith", @@ -38,6 +58,17 @@ MALWARE_KWARGS = dict( name="Cryptolocker", ) +# All required args for a Malware instance, plus some optional args +MALWARE_MORE_KWARGS = dict( + type='malware', + id=MALWARE_ID, + created="2016-04-06T20:03:00.000Z", + modified="2016-04-06T20:03:00.000Z", + labels=['ransomware'], + name="Cryptolocker", + description="A ransomware related to ..." +) + # Minimum required args for a Relationship instance RELATIONSHIP_KWARGS = dict( relationship_type="indicates", diff --git a/stix2/test/test_attack_pattern.py b/stix2/test/test_attack_pattern.py index 5bd5af2..07d0898 100644 --- a/stix2/test/test_attack_pattern.py +++ b/stix2/test/test_attack_pattern.py @@ -9,18 +9,18 @@ from .constants import ATTACK_PATTERN_ID EXPECTED = """{ + "type": "attack-pattern", + "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "name": "Spear Phishing", "description": "...", "external_references": [ { - "external_id": "CAPEC-163", - "source_name": "capec" + "source_name": "capec", + "external_id": "CAPEC-163" } - ], - "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", - "modified": "2016-05-12T08:17:27.000Z", - "name": "Spear Phishing", - "type": "attack-pattern" + ] }""" diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 54d7080..d70f63a 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -4,41 +4,41 @@ import stix2 EXPECTED_BUNDLE = """{ + "type": "bundle", "id": "bundle--00000000-0000-0000-0000-000000000004", + "spec_version": "2.0", "objects": [ { - "created": "2017-01-01T12:34:56.000Z", + "type": "indicator", "id": "indicator--00000000-0000-0000-0000-000000000001", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", "labels": [ "malicious-activity" ], - "modified": "2017-01-01T12:34:56.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", - "type": "indicator", "valid_from": "2017-01-01T12:34:56Z" }, { - "created": "2017-01-01T12:34:56.000Z", + "type": "malware", "id": "malware--00000000-0000-0000-0000-000000000002", - "labels": [ - "ransomware" - ], + "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "name": "Cryptolocker", - "type": "malware" + "labels": [ + "ransomware" + ] }, { - "created": "2017-01-01T12:34:56.000Z", + "type": "relationship", "id": "relationship--00000000-0000-0000-0000-000000000003", + "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "relationship_type": "indicates", "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "type": "relationship" + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" } - ], - "spec_version": "2.0", - "type": "bundle" + ] }""" @@ -116,3 +116,45 @@ def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationsh bundle = stix2.Bundle([indicator], malware, objects=[relationship]) assert str(bundle) == EXPECTED_BUNDLE + + +def test_create_bundle_invalid(indicator, malware, relationship): + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[1]) + assert excinfo.value.reason == "This property may only contain a dictionary or object" + + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[{}]) + assert excinfo.value.reason == "This property may only contain a non-empty dictionary or object" + + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[{'type': 'bundle'}]) + assert excinfo.value.reason == 'This property may not contain a Bundle object' + + +def test_parse_bundle(): + bundle = stix2.parse(EXPECTED_BUNDLE) + + assert bundle.type == "bundle" + assert bundle.id.startswith("bundle--") + assert bundle.spec_version == "2.0" + assert type(bundle.objects[0]) is stix2.Indicator + assert bundle.objects[0].type == 'indicator' + assert bundle.objects[1].type == 'malware' + assert bundle.objects[2].type == 'relationship' + + +def test_parse_unknown_type(): + unknown = { + "type": "other", + "id": "other--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created": "2016-04-06T20:03:00Z", + "modified": "2016-04-06T20:03:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "name": "Green Group Attacks Against Finance", + } + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse(unknown) + assert str(excinfo.value) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator." diff --git a/stix2/test/test_campaign.py b/stix2/test/test_campaign.py index 30b9444..202534d 100644 --- a/stix2/test/test_campaign.py +++ b/stix2/test/test_campaign.py @@ -9,13 +9,13 @@ from .constants import CAMPAIGN_ID EXPECTED = """{ - "created": "2016-04-06T20:03:00.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "type": "campaign", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", "name": "Green Group Attacks Against Finance", - "type": "campaign" + "description": "Campaign by Green Group against a series of targets in the financial services sector." }""" diff --git a/stix2/test/test_course_of_action.py b/stix2/test/test_course_of_action.py index e7a1b22..3dc379d 100644 --- a/stix2/test/test_course_of_action.py +++ b/stix2/test/test_course_of_action.py @@ -9,13 +9,13 @@ from .constants import COURSE_OF_ACTION_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", + "type": "course-of-action", "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", - "type": "course-of-action" + "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ..." }""" diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 60e982c..a71650b 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -6,7 +6,7 @@ from .constants import FAKE_TIME def test_identity_custom_property(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", created="2015-12-21T19:59:11Z", @@ -15,6 +15,7 @@ def test_identity_custom_property(): identity_class="individual", custom_properties="foobar", ) + assert str(excinfo.value) == "'custom_properties' must be a dictionary" identity = stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", @@ -31,7 +32,7 @@ def test_identity_custom_property(): def test_identity_custom_property_invalid(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", created="2015-12-21T19:59:11Z", @@ -40,6 +41,9 @@ def test_identity_custom_property_invalid(): identity_class="individual", x_foo="bar", ) + assert excinfo.value.cls == stix2.Identity + assert excinfo.value.properties == ['x_foo'] + assert "Unexpected properties for" in str(excinfo.value) def test_identity_custom_property_allowed(): @@ -67,18 +71,21 @@ def test_identity_custom_property_allowed(): }""", ]) def test_parse_identity_custom_property(data): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: identity = stix2.parse(data) + assert excinfo.value.cls == stix2.Identity + assert excinfo.value.properties == ['foo'] + assert "Unexpected properties for" in str(excinfo.value) identity = stix2.parse(data, allow_custom=True) assert identity.foo == "bar" -@stix2.sdo.CustomObject('x-new-type', { - 'property1': stix2.properties.StringProperty(required=True), - 'property2': stix2.properties.IntegerProperty(), -}) -class NewType(): +@stix2.sdo.CustomObject('x-new-type', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), +]) +class NewType(object): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") @@ -88,11 +95,13 @@ def test_custom_object_type(): nt = NewType(property1='something') assert nt.property1 == 'something' - with pytest.raises(stix2.exceptions.MissingPropertiesError): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewType(property2=42) + assert "No values for required properties" in str(excinfo.value) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewType(property1='something', property2=4) + assert "'property2' is too small." in str(excinfo.value) def test_parse_custom_object_type(): @@ -106,10 +115,24 @@ def test_parse_custom_object_type(): assert nt.property1 == 'something' -@stix2.observables.CustomObservable('x-new-observable', { - 'property1': stix2.properties.StringProperty(required=True), - 'property2': stix2.properties.IntegerProperty(), -}) +def test_parse_unregistered_custom_object_type(): + nt_string = """{ + "type": "x-foobar-observable", + "created": "2015-12-21T19:59:11Z", + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse(nt_string) + assert "Can't parse unknown object type" in str(excinfo.value) + assert "use the CustomObject decorator." in str(excinfo.value) + + +@stix2.observables.CustomObservable('x-new-observable', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ('x_property3', stix2.properties.BooleanProperty()), +]) class NewObservable(): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: @@ -120,11 +143,75 @@ def test_custom_observable_object(): no = NewObservable(property1='something') assert no.property1 == 'something' - with pytest.raises(stix2.exceptions.MissingPropertiesError): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewObservable(property2=42) + assert excinfo.value.properties == ['property1'] + assert "No values for required properties" in str(excinfo.value) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewObservable(property1='something', property2=4) + assert "'property2' is too small." in str(excinfo.value) + + +def test_custom_observable_object_invalid_ref_property(): + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomObservable('x-new-obs', [ + ('property_ref', stix2.properties.StringProperty()), + ]) + class NewObs(): + pass + assert "is named like an object reference property but is not an ObjectReferenceProperty" in str(excinfo.value) + + +def test_custom_observable_object_invalid_refs_property(): + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomObservable('x-new-obs', [ + ('property_refs', stix2.properties.StringProperty()), + ]) + class NewObs(): + pass + assert "is named like an object reference list property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + + +def test_custom_observable_object_invalid_refs_list_property(): + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomObservable('x-new-obs', [ + ('property_refs', stix2.properties.ListProperty(stix2.properties.StringProperty)), + ]) + class NewObs(): + pass + assert "is named like an object reference list property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + + +def test_custom_observable_object_invalid_valid_refs(): + @stix2.observables.CustomObservable('x-new-obs', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property_ref', stix2.properties.ObjectReferenceProperty(valid_types='email-addr')), + ]) + class NewObs(): + pass + + with pytest.raises(Exception) as excinfo: + NewObs(_valid_refs=['1'], + property1='something', + property_ref='1') + assert "must be created with _valid_refs as a dict, not a list" in str(excinfo.value) + + +def test_custom_no_properties_raises_exception(): + with pytest.raises(ValueError): + + @stix2.sdo.CustomObject('x-new-object-type') + class NewObject1(object): + pass + + +def test_custom_wrong_properties_arg_raises_exception(): + with pytest.raises(ValueError): + + @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) + class NewObject2(object): + pass def test_parse_custom_observable_object(): @@ -133,16 +220,38 @@ def test_parse_custom_observable_object(): "property1": "something" }""" - nt = stix2.parse_observable(nt_string) + nt = stix2.parse_observable(nt_string, []) assert nt.property1 == 'something' +def test_parse_unregistered_custom_observable_object(): + nt_string = """{ + "type": "x-foobar-observable", + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse_observable(nt_string) + assert "Can't parse unknown observable type" in str(excinfo.value) + + +def test_parse_invalid_custom_observable_object(): + nt_string = """{ + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse_observable(nt_string) + assert "Can't parse observable with no 'type' property" in str(excinfo.value) + + def test_observable_custom_property(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewObservable( property1='something', custom_properties="foobar", ) + assert "'custom_properties' must be a dictionary" in str(excinfo.value) no = NewObservable( property1='something', @@ -154,11 +263,13 @@ def test_observable_custom_property(): def test_observable_custom_property_invalid(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: NewObservable( property1='something', x_foo="bar", ) + assert excinfo.value.properties == ['x_foo'] + assert "Unexpected properties for" in str(excinfo.value) def test_observable_custom_property_allowed(): @@ -180,3 +291,107 @@ def test_observed_data_with_custom_observable_object(): allow_custom=True, ) assert ob_data.objects['0'].property1 == 'something' + + +@stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + 'property2': stix2.properties.IntegerProperty(), +}) +class NewExtension(): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + +def test_custom_extension(): + ext = NewExtension(property1='something') + assert ext.property1 == 'something' + + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: + NewExtension(property2=42) + assert excinfo.value.properties == ['property1'] + assert str(excinfo.value) == "No values for required properties for _Custom: (property1)." + + with pytest.raises(ValueError) as excinfo: + NewExtension(property1='something', property2=4) + assert str(excinfo.value) == "'property2' is too small." + + +def test_custom_extension_wrong_observable_type(): + ext = NewExtension(property1='something') + with pytest.raises(ValueError) as excinfo: + stix2.File(name="abc.txt", + extensions={ + "ntfs-ext": ext, + }) + + assert 'Cannot determine extension type' in excinfo.value.reason + + +def test_custom_extension_invalid_observable(): + # These extensions are being applied to improperly-created Observables. + # The Observable classes should have been created with the CustomObservable decorator. + class Foo(object): + pass + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomExtension(Foo, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class FooExtension(): + pass # pragma: no cover + assert str(excinfo.value) == "'observable' must be a valid Observable class!" + + class Bar(stix2.observables._Observable): + pass + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomExtension(Bar, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class BarExtension(): + pass + assert "Unknown observable type" in str(excinfo.value) + assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) + + class Baz(stix2.observables._Observable): + _type = 'Baz' + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomExtension(Baz, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class BazExtension(): + pass + assert "Unknown observable type" in str(excinfo.value) + assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) + + +def test_parse_observable_with_custom_extension(): + input_str = """{ + "type": "domain-name", + "value": "example.com", + "extensions": { + "x-new-ext": { + "property1": "foo", + "property2": 12 + } + } + }""" + + parsed = stix2.parse_observable(input_str) + assert parsed.extensions['x-new-ext'].property2 == 12 + + +def test_parse_observable_with_unregistered_custom_extension(): + input_str = """{ + "type": "domain-name", + "value": "example.com", + "extensions": { + "x-foobar-ext": { + "property1": "foo", + "property2": 12 + } + } + }""" + + with pytest.raises(ValueError) as excinfo: + stix2.parse_observable(input_str) + assert "Can't parse Unknown extension type" in str(excinfo.value) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0b57729..1415c34 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,51 +1,187 @@ -from stix2.sources import taxii +import pytest +from taxii2client import Collection + +from stix2.sources import (CompositeDataSource, DataSink, DataSource, + DataStore, make_id, taxii) +from stix2.sources.filters import Filter +from stix2.sources.memory import MemorySource, MemoryStore + +COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' -def test_ds_taxii(): - ds = taxii.TAXIIDataSource() - assert ds.name == 'TAXII' +class MockTAXIIClient(object): + """Mock for taxii2_client.TAXIIClient""" + pass -def test_ds_taxii_name(): - ds = taxii.TAXIIDataSource(name='My Data Source Name') - assert ds.name == "My Data Source Name" +@pytest.fixture +def collection(): + return Collection(COLLECTION_URL, MockTAXIIClient()) -def test_ds_params(): - url = "http://taxii_url.com:5000" - creds = {"username": "Wade", "password": "Wilson"} - ds = taxii.TAXIIDataSource(api_root=url, auth=creds) - assert ds.taxii_info['api_root']['url'] == url - assert ds.taxii_info['auth'] == creds +@pytest.fixture +def ds(): + return DataSource() + + +IND1 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND2 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND3 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND4 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND5 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND6 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND7 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND8 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} + +STIX_OBJS2 = [IND6, IND7, IND8] +STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] + + +def test_ds_abstract_class_smoke(): + ds1 = DataSource() + ds2 = DataSink() + ds3 = DataStore(source=ds1, sink=ds2) + + with pytest.raises(NotImplementedError): + ds3.add(None) + + with pytest.raises(NotImplementedError): + ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): + ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): + ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + + +def test_memory_store_smoke(): + # Initialize MemoryStore with dict + ms = MemoryStore(STIX_OBJS1) + + # Add item to sink + ms.add(dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS2, + spec_version="2.0", + type="bundle")) + + resp = ms.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(resp) == 1 + + resp = ms.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + + query = [Filter('type', '=', 'malware')] + + resp = ms.query(query) + assert len(resp) == 0 + + +def test_ds_taxii(collection): + ds = taxii.TAXIICollectionSource(collection) + assert ds.collection is not None + + +def test_ds_taxii_name(collection): + ds = taxii.TAXIICollectionSource(collection) + assert ds.collection is not None def test_parse_taxii_filters(): query = [ - { - "field": "added_after", - "op": "=", - "value": "2016-02-01T00:00:01.000Z" - }, - { - "field": "id", - "op": "=", - "value": "taxii stix object ID" - }, - { - "field": "type", - "op": "=", - "value": "taxii stix object ID" - }, - { - "field": "version", - "op": "=", - "value": "first" - }, - { - "field": "created_by_ref", - "op": "=", - "value": "Bane" - } + Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), + Filter("id", "=", "taxii stix object ID"), + Filter("type", "=", "taxii stix object ID"), + Filter("version", "=", "first"), + Filter("created_by_ref", "=", "Bane"), ] expected_params = { @@ -55,100 +191,349 @@ def test_parse_taxii_filters(): "match[version]": "first" } - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource(collection) taxii_filters = ds._parse_taxii_filters(query) assert taxii_filters == expected_params -def test_add_get_remove_filter(): - class dummy(object): - x = 4 - - obj_1 = dummy() +def test_add_get_remove_filter(ds): # First 3 filters are valid, remaining fields are erroneous in some way - filters = [ + valid_filters = [ + Filter('type', '=', 'malware'), + Filter('id', '!=', 'stix object id'), + Filter('labels', 'in', ["heartbleed", "malicious-activity"]), + ] + invalid_filters = [ + Filter('description', '=', 'not supported field - just place holder'), + Filter('modified', '*', 'not supported operator - just place holder'), + Filter('created', '=', object()), + ] + + assert len(ds.filters) == 0 + + ds.add_filter(valid_filters[0]) + assert len(ds.filters) == 1 + + # Addin the same filter again will have no effect since `filters` uses a set + ds.add_filter(valid_filters[0]) + assert len(ds.filters) == 1 + + ds.add_filter(valid_filters[1]) + assert len(ds.filters) == 2 + ds.add_filter(valid_filters[2]) + assert len(ds.filters) == 3 + + # TODO: make better error messages + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[0]) + assert str(excinfo.value) == "Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported" + + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[1]) + assert str(excinfo.value) == "Filter operation (from 'op' field) not supported" + + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[2]) + assert str(excinfo.value) == "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" + + assert set(valid_filters) == ds.filters + + # remove + ds.filters.remove(valid_filters[0]) + + assert len(ds.filters) == 2 + + ds.add_filters(valid_filters) + + +def test_apply_common_filters(ds): + stix_objs = [ { - "field": "type", - "op": '=', - "value": "malware" + "created": "2017-01-27T13:49:53.997Z", + "description": "\n\nTITLE:\n\tPoison Ivy", + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "labels": [ + "remote-access-trojan" + ], + "modified": "2017-01-27T13:49:53.997Z", + "name": "Poison Ivy", + "type": "malware" }, { - "field": "id", - "op": "!=", - "value": "stix object id" + "created": "2014-05-08T09:00:00.000Z", + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "labels": [ + "file-hash-watchlist" + ], + "modified": "2014-05-08T09:00:00.000Z", + "name": "File hash for Poison Ivy variant", + "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "type": "indicator", + "valid_from": "2014-05-08T09:00:00.000000Z" }, { - "field": "labels", - "op": "in", - "value": ["heartbleed", "malicious-activity"] + "created": "2014-05-08T09:00:00.000Z", + "granular_markings": [ + { + "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + "selectors": [ + "relationship_type" + ] + } + ], + "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", + "modified": "2014-05-08T09:00:00.000Z", + "object_marking_refs": [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + ], + "relationship_type": "indicates", + "revoked": True, + "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "type": "relationship" }, { - "field": "revoked", - "value": "filter missing \'op\' field" - }, - { - "field": "granular_markings", - "op": "=", - "value": "not supported field - just place holder" - }, - { - "field": "modified", - "op": "*", - "value": "not supported operator - just place holder" - }, - { - "field": "created", - "op": "=", - "value": obj_1 + "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", + "created": "2016-02-14T00:00:00.000Z", + "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", + "modified": "2016-02-14T00:00:00.000Z", + "type": "vulnerability", + "name": "CVE-2014-0160", + "description": "The (1) TLS...", + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2014-0160" + } + ], + "labels": ["heartbleed", "has-logo"] } ] - expected_errors = [ - "Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.", - "Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported", - "Filter operation(from 'op' field) not supported", - "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" + filters = [ + Filter("type", "!=", "relationship"), + Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), + Filter("labels", "in", "remote-access-trojan"), + Filter("created", ">", "2015-01-01T01:00:00.000Z"), + Filter("revoked", "=", True), + Filter("revoked", "!=", True), + Filter("revoked", "?", False), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "relationship_type"), + Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), + Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), + Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "description"), + Filter("external_references.source_name", "=", "CVE"), ] - ds = taxii.TAXIIDataSource() - # add - ids, statuses = ds.add_filter(filters) + # "Return any object whose type is not relationship" + resp = ds.apply_common_filters(stix_objs, [filters[0]]) + ids = [r['id'] for r in resp] + assert stix_objs[0]['id'] in ids + assert stix_objs[1]['id'] in ids + assert stix_objs[3]['id'] in ids + assert len(ids) == 3 - # 7 filters should have been successfully added - assert len(ids) == 7 + # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" + resp = ds.apply_common_filters(stix_objs, [filters[1]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 - # all filters added to data source - for idx, status in enumerate(statuses): - assert status['filter'] == filters[idx] + # "Return any object that contains remote-access-trojan in labels" + resp = ds.apply_common_filters(stix_objs, [filters[2]]) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 1 - # proper status warnings were triggered - assert statuses[3]['errors'][0] == expected_errors[0] - assert statuses[4]['errors'][0] == expected_errors[1] - assert statuses[5]['errors'][0] == expected_errors[2] - assert statuses[6]['errors'][0] == expected_errors[3] + # "Return any object created after 2015-01-01T01:00:00.000Z" + resp = ds.apply_common_filters(stix_objs, [filters[3]]) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 2 + + # "Return any revoked object" + resp = ds.apply_common_filters(stix_objs, [filters[4]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object whose not revoked" + # Note that if 'revoked' property is not present in object. + # Currently we can't use such an expression to filter for... :( + resp = ds.apply_common_filters(stix_objs, [filters[5]]) + assert len(resp) == 0 + + # Assert unknown operator for _boolean() raises exception. + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(stix_objs, [filters[6]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}" + .format(filters[6].op, filters[6].field)) + + # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs" + resp = ds.apply_common_filters(stix_objs, [filters[7]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object that contains relationship_type in their selectors AND + # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref" + resp = ds.apply_common_filters(stix_objs, [filters[8], filters[9]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id" + resp = ds.apply_common_filters(stix_objs, [filters[10]]) + assert resp[0]['id'] == stix_objs[3]['id'] + assert len(resp) == 1 + + # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9" + resp = ds.apply_common_filters(stix_objs, [filters[11]]) + assert len(resp) == 1 + + # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None) + resp = ds.apply_common_filters(stix_objs, [filters[12]]) + assert len(resp) == 0 + + # "Return any object that contains description in its selectors" (None) + resp = ds.apply_common_filters(stix_objs, [filters[13]]) + assert len(resp) == 0 + + # "Return any object that object that matches CVE in source_name" (None, case sensitive) + resp = ds.apply_common_filters(stix_objs, [filters[14]]) + assert len(resp) == 0 -# def test_data_source_file(): -# ds = file.FileDataSource() -# -# assert ds.name == "DataSource" -# -# -# def test_data_source_name(): -# ds = file.FileDataSource(name="My File Data Source") -# -# assert ds.name == "My File Data Source" -# -# -# def test_data_source_get(): -# ds = file.FileDataSource(name="My File Data Source") -# -# with pytest.raises(NotImplementedError): -# ds.get("foo") -# -# #filter testing -# def test_add_filter(): -# ds = file.FileDataSource() +def test_filters0(ds): + # "Return any object modified before 2017-01-28T13:49:53.935Z" + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + +def test_filters1(ds): + # "Return any object modified after 2017-01-28T13:49:53.935Z" + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + +def test_filters2(ds): + # "Return any object modified after or on 2017-01-28T13:49:53.935Z" + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + +def test_filters3(ds): + # "Return any object modified before or on 2017-01-28T13:49:53.935Z" + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + +def test_filters4(ds): + fltr4 = Filter("modified", "?", "2017-01-27T13:49:53.935Z") + # Assert unknown operator for _all() raises exception. + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [fltr4]) + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(fltr4.op, fltr4.field) + + +def test_filters5(ds): + # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + +def test_filters6(ds): + fltr6 = Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + # Assert unknown operator for _id() raises exception. + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [fltr6]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(fltr6.op, fltr6.field) + + +def test_filters7(ds): + fltr7 = Filter("notacommonproperty", "=", "bar") + # Assert unknown field raises exception. + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [fltr7]) + + assert str(excinfo.value) == ("Error, field: {0} is not supported for " + "filtering on.").format(fltr7.field) + + +def test_deduplicate(ds): + unique = ds.deduplicate(STIX_OBJS1) + + # Only 3 objects are unique + # 2 id's vary + # 2 modified times vary for a particular id + + assert len(unique) == 3 + + ids = [obj['id'] for obj in unique] + mods = [obj['modified'] for obj in unique] + + assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids + assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids + assert "2017-01-27T13:49:53.935Z" in mods + assert "2017-01-27T13:49:53.936Z" in mods + + +def test_add_remove_composite_datasource(): + cds = CompositeDataSource() + ds1 = DataSource() + ds2 = DataSource() + ds3 = DataSink() + + cds.add_data_source([ds1, ds2, ds1, ds3]) + + assert len(cds.get_all_data_sources()) == 2 + + cds.remove_data_source([ds1.id, ds2.id]) + + assert len(cds.get_all_data_sources()) == 0 + + with pytest.raises(ValueError): + cds.remove_data_source([ds3.id]) + + +def test_composite_datasource_operations(): + BUNDLE1 = dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS1, + spec_version="2.0", + type="bundle") + cds = CompositeDataSource() + ds1 = MemorySource(stix_data=BUNDLE1) + ds2 = MemorySource(stix_data=STIX_OBJS2) + + cds.add_data_source([ds1, ds2]) + + indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + # In STIX_OBJS2 changed the 'modified' property to a later time... + assert len(indicators) == 2 + + indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + query = [ + Filter("type", "=", "indicator") + ] + + results = cds.query(query) + + # STIX_OBJS2 has indicator with later time, one with different id, one with + # original time in STIX_OBJS1 + assert len(results) == 3 diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py index 10a6e53..c4c3755 100644 --- a/stix2/test/test_external_reference.py +++ b/stix2/test/test_external_reference.py @@ -8,9 +8,12 @@ import stix2 VERIS = """{ - "external_id": "0001AA7F-C601-424A-B2B8-BE6C9F5164E7", "source_name": "veris", - "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json" + "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", + "hashes": { + "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b" + }, + "external_id": "0001AA7F-C601-424A-B2B8-BE6C9F5164E7" }""" @@ -18,6 +21,9 @@ def test_external_reference_veris(): ref = stix2.ExternalReference( source_name="veris", external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7", + hashes={ + "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b" + }, url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", ) @@ -25,8 +31,8 @@ def test_external_reference_veris(): CAPEC = """{ - "external_id": "CAPEC-550", - "source_name": "capec" + "source_name": "capec", + "external_id": "CAPEC-550" }""" @@ -37,13 +43,13 @@ def test_external_reference_capec(): ) assert str(ref) == CAPEC - assert re.match("ExternalReference\(external_id=u?'CAPEC-550', source_name=u?'capec'\)", repr(ref)) + assert re.match("ExternalReference\(source_name=u?'capec', external_id=u?'CAPEC-550'\)", repr(ref)) CAPEC_URL = """{ - "external_id": "CAPEC-550", "source_name": "capec", - "url": "http://capec.mitre.org/data/definitions/550.html" + "url": "http://capec.mitre.org/data/definitions/550.html", + "external_id": "CAPEC-550" }""" @@ -58,8 +64,8 @@ def test_external_reference_capec_url(): THREAT_REPORT = """{ - "description": "Threat report", "source_name": "ACME Threat Intel", + "description": "Threat report", "url": "http://www.example.com/threat-report.pdf" }""" @@ -75,9 +81,9 @@ def test_external_reference_threat_report(): BUGZILLA = """{ - "external_id": "1370", "source_name": "ACME Bugzilla", - "url": "https://www.example.com/bugs/1370" + "url": "https://www.example.com/bugs/1370", + "external_id": "1370" }""" @@ -92,8 +98,8 @@ def test_external_reference_bugzilla(): OFFLINE = """{ - "description": "Threat report", - "source_name": "ACME Threat Intel" + "source_name": "ACME Threat Intel", + "description": "Threat report" }""" @@ -104,7 +110,7 @@ def test_external_reference_offline(): ) assert str(ref) == OFFLINE - assert re.match("ExternalReference\(description=u?'Threat report', source_name=u?'ACME Threat Intel'\)", repr(ref)) + assert re.match("ExternalReference\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\)", repr(ref)) # Yikes! This works assert eval("stix2." + repr(ref)) == ref diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py new file mode 100644 index 0000000..e910ad3 --- /dev/null +++ b/stix2/test/test_granular_markings.py @@ -0,0 +1,1026 @@ + +import pytest + +from stix2 import Malware, markings + +from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST +from .constants import MARKING_IDS + +"""Tests for the Data Markings API.""" + +MALWARE_KWARGS = MALWARE_KWARGS_CONST.copy() + + +def test_add_marking_mark_one_selector_multiple_refs(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +@pytest.mark.parametrize("data", [ + ( + Malware(**MALWARE_KWARGS), + Malware( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS), + ), + ( + MALWARE_KWARGS, + dict( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS), + ), +]) +def test_add_marking_mark_multiple_selector_one_refs(data): + before = data[0] + after = data[1] + + before = markings.add_markings(before, [MARKING_IDS[0]], ["description", "name"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_add_marking_mark_multiple_selector_multiple_refs(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "name"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_add_marking_mark_another_property_same_marking(): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, [MARKING_IDS[0]], ["name"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_add_marking_mark_same_property_same_marking(): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, [MARKING_IDS[0]], ["description"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +@pytest.mark.parametrize("data,marking", [ + ({"description": "test description"}, + [["title"], ["marking-definition--1", "marking-definition--2"], + "", ["marking-definition--1", "marking-definition--2"], + [], ["marking-definition--1", "marking-definition--2"], + [""], ["marking-definition--1", "marking-definition--2"], + ["description"], [""], + ["description"], [], + ["description"], ["marking-definition--1", 456] + ]) +]) +def test_add_marking_bad_selector(data, marking): + with pytest.raises(AssertionError): + markings.add_markings(data, marking[0], marking[1]) + + +GET_MARKINGS_TEST_DATA = { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] +} + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_smoke(data): + """Test get_markings does not fail.""" + assert len(markings.get_markings(data, "a")) >= 1 + assert markings.get_markings(data, "a") == ["1"] + + +@pytest.mark.parametrize("data", [ + GET_MARKINGS_TEST_DATA, + {"b": 1234}, +]) +def test_get_markings_not_marked(data): + """Test selector that is not marked returns empty list.""" + results = markings.get_markings(data, "b") + assert len(results) == 0 + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_multiple_selectors(data): + """Test multiple selectors return combination of markings.""" + total = markings.get_markings(data, ["x.y", "x.z"]) + xy_markings = markings.get_markings(data, ["x.y"]) + xz_markings = markings.get_markings(data, ["x.z"]) + + assert set(xy_markings).issubset(total) + assert set(xz_markings).issubset(total) + assert set(xy_markings).union(xz_markings).issuperset(total) + + +@pytest.mark.parametrize("data,selector", [ + (GET_MARKINGS_TEST_DATA, "foo"), + (GET_MARKINGS_TEST_DATA, ""), + (GET_MARKINGS_TEST_DATA, []), + (GET_MARKINGS_TEST_DATA, [""]), + (GET_MARKINGS_TEST_DATA, "x.z.[-2]"), + (GET_MARKINGS_TEST_DATA, "c.f"), + (GET_MARKINGS_TEST_DATA, "c.[2].i"), + (GET_MARKINGS_TEST_DATA, "c.[3]"), + (GET_MARKINGS_TEST_DATA, "d"), + (GET_MARKINGS_TEST_DATA, "x.[0]"), + (GET_MARKINGS_TEST_DATA, "z.y.w"), + (GET_MARKINGS_TEST_DATA, "x.z.[1]"), + (GET_MARKINGS_TEST_DATA, "x.z.foo3") +]) +def test_get_markings_bad_selector(data, selector): + """Test bad selectors raise exception""" + with pytest.raises(AssertionError): + markings.get_markings(data, selector) + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_positional_arguments_combinations(data): + """Test multiple combinations for inherited and descendant markings.""" + assert set(markings.get_markings(data, "a", False, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, True)) == set(["1"]) + assert set(markings.get_markings(data, "a", False, True)) == set(["1"]) + + assert set(markings.get_markings(data, "b", False, False)) == set([]) + assert set(markings.get_markings(data, "b", True, False)) == set([]) + assert set(markings.get_markings(data, "b", True, True)) == set([]) + assert set(markings.get_markings(data, "b", False, True)) == set([]) + + assert set(markings.get_markings(data, "c", False, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, True)) == set(["2", "3", "4", "5"]) + assert set(markings.get_markings(data, "c", False, True)) == set(["2", "3", "4", "5"]) + + assert set(markings.get_markings(data, "c.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "c.[0]", True, False)) == set(["2"]) + assert set(markings.get_markings(data, "c.[0]", True, True)) == set(["2"]) + assert set(markings.get_markings(data, "c.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "c.[1]", False, False)) == set(["3"]) + assert set(markings.get_markings(data, "c.[1]", True, False)) == set(["2", "3"]) + assert set(markings.get_markings(data, "c.[1]", True, True)) == set(["2", "3"]) + assert set(markings.get_markings(data, "c.[1]", False, True)) == set(["3"]) + + assert set(markings.get_markings(data, "c.[2]", False, False)) == set(["4"]) + assert set(markings.get_markings(data, "c.[2]", True, False)) == set(["2", "4"]) + assert set(markings.get_markings(data, "c.[2]", True, True)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2]", False, True)) == set(["4", "5"]) + + assert set(markings.get_markings(data, "c.[2].g", False, False)) == set(["5"]) + assert set(markings.get_markings(data, "c.[2].g", True, False)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2].g", True, True)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2].g", False, True)) == set(["5"]) + + assert set(markings.get_markings(data, "x", False, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, True)) == set(["6", "7", "8", "9", "10"]) + assert set(markings.get_markings(data, "x", False, True)) == set(["6", "7", "8", "9", "10"]) + + assert set(markings.get_markings(data, "x.y", False, False)) == set(["7"]) + assert set(markings.get_markings(data, "x.y", True, False)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y", True, True)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y", False, True)) == set(["7", "8"]) + + assert set(markings.get_markings(data, "x.y.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "x.y.[0]", True, False)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y.[0]", True, True)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.y.[1]", False, False)) == set(["8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, False)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, True)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y.[1]", False, True)) == set(["8"]) + + assert set(markings.get_markings(data, "x.z", False, False)) == set(["9"]) + assert set(markings.get_markings(data, "x.z", True, False)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z", True, True)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z", False, True)) == set(["9", "10"]) + + assert set(markings.get_markings(data, "x.z.foo1", False, False)) == set([]) + assert set(markings.get_markings(data, "x.z.foo1", True, False)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z.foo1", True, True)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z.foo1", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.z.foo2", False, False)) == set(["10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, False)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, True)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) + + +@pytest.mark.parametrize("before", [ + Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + ], + **MALWARE_KWARGS + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + ], + **MALWARE_KWARGS + ), +]) +def test_remove_marking_remove_one_selector_with_multiple_refs(before): + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) + assert "granular_markings" not in before + + +def test_remove_marking_remove_multiple_selector_one_ref(): + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, MARKING_IDS[0], ["description", "modified"]) + assert "granular_markings" not in before + + +def test_remove_marking_mark_one_selector_from_multiple_ones(): + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_remove_marking_mark_one_selector_markings_from_multiple_ones(): + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_remove_marking_mark_mutilple_selector_multiple_refs(): + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"]) + assert "granular_markings" not in before + + +def test_remove_marking_mark_another_property_same_marking(): + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_remove_marking_mark_same_property_same_marking(): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["description"]) + assert "granular_markings" not in before + + +def test_remove_no_markings(): + before = { + "description": "test description", + } + after = markings.remove_markings(before, ["marking-definition--1"], ["description"]) + assert before == after + + +def test_remove_marking_bad_selector(): + before = { + "description": "test description", + } + with pytest.raises(AssertionError): + markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"]) + + +IS_MARKED_TEST_DATA = [ + Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[2] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[3] + }, + ], + **MALWARE_KWARGS + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[2] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[3] + }, + ], + **MALWARE_KWARGS + ), +] + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_smoke(data): + """Smoke test is_marked call does not fail.""" + assert markings.is_marked(data, selectors=["description"]) + assert markings.is_marked(data, selectors=["modified"]) is False + + +@pytest.mark.parametrize("data,selector", [ + (IS_MARKED_TEST_DATA[0], "foo"), + (IS_MARKED_TEST_DATA[0], ""), + (IS_MARKED_TEST_DATA[0], []), + (IS_MARKED_TEST_DATA[0], [""]), + (IS_MARKED_TEST_DATA[0], "x.z.[-2]"), + (IS_MARKED_TEST_DATA[0], "c.f"), + (IS_MARKED_TEST_DATA[0], "c.[2].i"), + (IS_MARKED_TEST_DATA[1], "c.[3]"), + (IS_MARKED_TEST_DATA[1], "d"), + (IS_MARKED_TEST_DATA[1], "x.[0]"), + (IS_MARKED_TEST_DATA[1], "z.y.w"), + (IS_MARKED_TEST_DATA[1], "x.z.[1]"), + (IS_MARKED_TEST_DATA[1], "x.z.foo3") +]) +def test_is_marked_invalid_selector(data, selector): + """Test invalid selector raises an error.""" + with pytest.raises(AssertionError): + markings.is_marked(data, selectors=selector) + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_mix_selector(data): + """Test valid selector, one marked and one not marked returns True.""" + assert markings.is_marked(data, selectors=["description", "labels"]) + assert markings.is_marked(data, selectors=["description"]) + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_valid_selector_no_refs(data): + """Test that a valid selector return True when it has marking refs and False when not.""" + assert markings.is_marked(data, selectors=["description"]) + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[3]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[2]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[5]], ["description"]) is False + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_valid_selector_and_refs(data): + """Test that a valid selector returns True when marking_refs match.""" + assert markings.is_marked(data, [MARKING_IDS[1]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[1]], ["modified"]) is False + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_valid_selector_multiple_refs(data): + """Test that a valid selector returns True if aall marking_refs match. + Otherwise False.""" + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[3]], ["labels"]) + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[1]], ["labels"]) is False + assert markings.is_marked(data, MARKING_IDS[2], ["labels"]) + assert markings.is_marked(data, ["marking-definition--1234"], ["labels"]) is False + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_no_marking_refs(data): + """Test that a valid content selector with no marking_refs returns True + if there is a granular_marking that asserts that field, False + otherwise.""" + assert markings.is_marked(data, selectors=["type"]) is False + assert markings.is_marked(data, selectors=["labels"]) + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_no_selectors(data): + """Test that we're ensuring 'selectors' is provided.""" + with pytest.raises(TypeError) as excinfo: + markings.granular_markings.is_marked(data) + assert "'selectors' must be provided" in str(excinfo.value) + + +def test_is_marked_positional_arguments_combinations(): + """Test multiple combinations for inherited and descendant markings.""" + test_sdo = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + assert markings.is_marked(test_sdo, ["1"], "a", False, False) + assert markings.is_marked(test_sdo, ["1"], "a", True, False) + assert markings.is_marked(test_sdo, ["1"], "a", True, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, True) + + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "b", inherited=True, descendants=False) is False + assert markings.is_marked(test_sdo, "b", inherited=True, descendants=True) is False + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["2"], "c", False, False) + assert markings.is_marked(test_sdo, ["2"], "c", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", True, True) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", False, True) + + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["2"], "c.[0]", True, False) + assert markings.is_marked(test_sdo, ["2"], "c.[0]", True, True) + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, False) + assert markings.is_marked(test_sdo, ["2", "3"], "c.[1]", True, False) + assert markings.is_marked(test_sdo, ["2", "3"], "c.[1]", True, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, True) + + assert markings.is_marked(test_sdo, ["4"], "c.[2]", False, False) + assert markings.is_marked(test_sdo, ["2", "4"], "c.[2]", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2]", True, True) + assert markings.is_marked(test_sdo, ["4", "5"], "c.[2]", False, True) + + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2].g", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2].g", True, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, True) + + assert markings.is_marked(test_sdo, ["6"], "x", False, False) + assert markings.is_marked(test_sdo, ["6"], "x", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", True, True) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", False, True) + + assert markings.is_marked(test_sdo, ["7"], "x.y", False, False) + assert markings.is_marked(test_sdo, ["6", "7"], "x.y", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y", True, True) + assert markings.is_marked(test_sdo, ["7", "8"], "x.y", False, True) + + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["6", "7"], "x.y.[0]", True, False) + assert markings.is_marked(test_sdo, ["6", "7"], "x.y.[0]", True, True) + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y.[1]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y.[1]", True, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, True) + + assert markings.is_marked(test_sdo, ["9"], "x.z", False, False) + assert markings.is_marked(test_sdo, ["6", "9"], "x.z", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z", True, True) + assert markings.is_marked(test_sdo, ["9", "10"], "x.z", False, True) + + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["6", "9"], "x.z.foo1", True, False) + assert markings.is_marked(test_sdo, ["6", "9"], "x.z.foo1", True, True) + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z.foo2", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z.foo2", True, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, True) + + +def test_create_sdo_with_invalid_marking(): + with pytest.raises(AssertionError) as excinfo: + Malware( + granular_markings=[ + { + "selectors": ["foo"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + assert str(excinfo.value) == "Selector foo in Malware is not valid!" + + +def test_set_marking_mark_one_selector_multiple_refs(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_set_marking_mark_multiple_selector_one_refs(): + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[0]], ["description", "modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_set_marking_mark_multiple_selector_multiple_refs_from_none(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_set_marking_mark_another_property_same_marking(): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[2] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[1], MARKING_IDS[2]], ["description"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +@pytest.mark.parametrize("marking", [ + ([MARKING_IDS[4], MARKING_IDS[5]], ["foo"]), + ([MARKING_IDS[4], MARKING_IDS[5]], ""), + ([MARKING_IDS[4], MARKING_IDS[5]], []), + ([MARKING_IDS[4], MARKING_IDS[5]], [""]), +]) +def test_set_marking_bad_selector(marking): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + + with pytest.raises(AssertionError): + before = markings.set_markings(before, marking[0], marking[1]) + + assert before == after + + +def test_set_marking_mark_same_property_same_marking(): + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[0]], ["description"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +CLEAR_MARKINGS_TEST_DATA = [ + Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified", "description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["modified", "description", "type"], + "marking_ref": MARKING_IDS[2] + }, + ], + **MALWARE_KWARGS + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified", "description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["modified", "description", "type"], + "marking_ref": MARKING_IDS[2] + }, + ], + **MALWARE_KWARGS + ) +] + + +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) +def test_clear_marking_smoke(data): + """Test clear_marking call does not fail.""" + data = markings.clear_markings(data, "modified") + assert markings.is_marked(data, "modified") is False + + +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) +def test_clear_marking_multiple_selectors(data): + """Test clearing markings for multiple selectors effectively removes associated markings.""" + data = markings.clear_markings(data, ["type", "description"]) + assert markings.is_marked(data, ["type", "description"]) is False + + +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) +def test_clear_marking_one_selector(data): + """Test markings associated with one selector were removed.""" + data = markings.clear_markings(data, "description") + assert markings.is_marked(data, "description") is False + + +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) +def test_clear_marking_all_selectors(data): + data = markings.clear_markings(data, ["description", "type", "modified"]) + assert markings.is_marked(data, "description") is False + assert "granular_markings" not in data + + +@pytest.mark.parametrize("data,selector", [ + (CLEAR_MARKINGS_TEST_DATA[0], "foo"), + (CLEAR_MARKINGS_TEST_DATA[0], ""), + (CLEAR_MARKINGS_TEST_DATA[1], []), + (CLEAR_MARKINGS_TEST_DATA[1], [""]), +]) +def test_clear_marking_bad_selector(data, selector): + """Test bad selector raises exception.""" + with pytest.raises(AssertionError): + markings.clear_markings(data, selector) diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py index ed51958..a9415fe 100644 --- a/stix2/test/test_identity.py +++ b/stix2/test/test_identity.py @@ -9,12 +9,12 @@ from .constants import IDENTITY_ID EXPECTED = """{ - "created": "2015-12-21T19:59:11.000Z", + "type": "identity", "id": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c", - "identity_class": "individual", + "created": "2015-12-21T19:59:11.000Z", "modified": "2015-12-21T19:59:11.000Z", "name": "John Smith", - "type": "identity" + "identity_class": "individual" }""" diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index 5daa0f6..1f8e3bd 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -10,25 +10,25 @@ from .constants import FAKE_TIME, INDICATOR_ID, INDICATOR_KWARGS EXPECTED_INDICATOR = """{ - "created": "2017-01-01T00:00:01.000Z", + "type": "indicator", "id": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "created": "2017-01-01T00:00:01.000Z", + "modified": "2017-01-01T00:00:01.000Z", "labels": [ "malicious-activity" ], - "modified": "2017-01-01T00:00:01.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", - "type": "indicator", "valid_from": "1970-01-01T00:00:01Z" }""" EXPECTED_INDICATOR_REPR = "Indicator(" + " ".join(""" - created=STIXdatetime(2017, 1, 1, 0, 0, 1, tzinfo=), - id='indicator--01234567-89ab-cdef-0123-456789abcdef', - labels=['malicious-activity'], - modified=STIXdatetime(2017, 1, 1, 0, 0, 1, tzinfo=), - pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", type='indicator', - valid_from=datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=) + id='indicator--01234567-89ab-cdef-0123-456789abcdef', + created='2017-01-01T00:00:01.000Z', + modified='2017-01-01T00:00:01.000Z', + labels=['malicious-activity'], + pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + valid_from='1970-01-01T00:00:01Z' """.split()) + ")" @@ -174,3 +174,23 @@ def test_parse_indicator(data): assert idctr.valid_from == dt.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc) assert idctr.labels[0] == "malicious-activity" assert idctr.pattern == "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']" + + +def test_invalid_indicator_pattern(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern="file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e'", + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'input is missing square brackets' in excinfo.value.reason + + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern='[file:hashes.MD5 = "d41d8cd98f00b204e9800998ecf8427e"]', + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'mismatched input' in excinfo.value.reason diff --git a/stix2/test/test_intrusion_set.py b/stix2/test/test_intrusion_set.py index a6eee7f..481b3cb 100644 --- a/stix2/test/test_intrusion_set.py +++ b/stix2/test/test_intrusion_set.py @@ -9,21 +9,21 @@ from .constants import INTRUSION_SET_ID EXPECTED = """{ + "type": "intrusion-set", + "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", + "modified": "2016-04-06T20:03:48.000Z", + "name": "Bobcat Breakin", + "description": "Incidents usually feature a shared TTP of a bobcat being released...", "aliases": [ "Zookeeper" ], - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Incidents usually feature a shared TTP of a bobcat being released...", "goals": [ "acquisition-theft", "harassment", "damage" - ], - "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29", - "modified": "2016-04-06T20:03:48.000Z", - "name": "Bobcat Breakin", - "type": "intrusion-set" + ] }""" diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py index ff0b394..7f665ea 100644 --- a/stix2/test/test_malware.py +++ b/stix2/test/test_malware.py @@ -10,14 +10,14 @@ from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS EXPECTED_MALWARE = """{ - "created": "2016-05-12T08:17:27.000Z", + "type": "malware", "id": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "labels": [ - "ransomware" - ], + "created": "2016-05-12T08:17:27.000Z", "modified": "2016-05-12T08:17:27.000Z", "name": "Cryptolocker", - "type": "malware" + "labels": [ + "ransomware" + ] }""" diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 70d67dd..0c6069a 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -10,36 +10,36 @@ from .constants import MARKING_DEFINITION_ID EXPECTED_TLP_MARKING_DEFINITION = """{ + "type": "marking-definition", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "created": "2017-01-20T00:00:00Z", + "definition_type": "tlp", "definition": { "tlp": "white" - }, - "definition_type": "tlp", - "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - "type": "marking-definition" + } }""" EXPECTED_STATEMENT_MARKING_DEFINITION = """{ + "type": "marking-definition", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "created": "2017-01-20T00:00:00Z", + "definition_type": "statement", "definition": { "statement": "Copyright 2016, Example Corp" - }, - "definition_type": "statement", - "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - "type": "marking-definition" + } }""" EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING = """{ - "created": "2016-04-06T20:03:00.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "type": "campaign", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", "name": "Green Group Attacks Against Finance", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", "object_marking_refs": [ "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" - ], - "type": "campaign" + ] }""" EXPECTED_GRANULAR_MARKING = """{ @@ -53,8 +53,12 @@ EXPECTED_GRANULAR_MARKING = """{ }""" EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ - "created": "2016-04-06T20:03:00.000Z", + "type": "campaign", + "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "name": "Green Group Attacks Against Finance", "description": "Campaign by Green Group against a series of targets in the financial services sector.", "granular_markings": [ { @@ -63,11 +67,7 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ "description" ] } - ], - "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "modified": "2016-04-06T20:03:00.000Z", - "name": "Green Group Attacks Against Finance", - "type": "campaign" + ] }""" @@ -75,7 +75,7 @@ def test_marking_def_example_with_tlp(): assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION -def test_marking_def_example_with_statement(): +def test_marking_def_example_with_statement_positional_argument(): marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", @@ -86,12 +86,13 @@ def test_marking_def_example_with_statement(): assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION -def test_marking_def_example_with_positional_statement(): +def test_marking_def_example_with_kwargs_statement(): + kwargs = dict(statement="Copyright 2016, Example Corp") marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", definition_type="statement", - definition=stix2.StatementMarking("Copyright 2016, Example Corp") + definition=stix2.StatementMarking(**kwargs) ) assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION @@ -102,7 +103,7 @@ def test_marking_def_invalid_type(): stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", - definition_type="my-definiition-type", + definition_type="my-definition-type", definition=stix2.StatementMarking("Copyright 2016, Example Corp") ) @@ -180,4 +181,64 @@ def test_parse_marking_definition(data): assert gm.definition_type == "tlp" +@stix2.common.CustomMarking('x-new-marking-type', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), +]) +class NewMarking(object): + def __init__(self, property2=None, **kwargs): + return + + +def test_registered_custom_marking(): + nm = NewMarking(property1='something', property2=55) + + marking_def = stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type", + definition=nm + ) + + assert marking_def.type == "marking-definition" + assert marking_def.id == "marking-definition--00000000-0000-0000-0000-000000000012" + assert marking_def.created == dt.datetime(2017, 1, 22, 0, 0, 0, tzinfo=pytz.utc) + assert marking_def.definition.property1 == "something" + assert marking_def.definition.property2 == 55 + assert marking_def.definition_type == "x-new-marking-type" + + +def test_not_registered_marking_raises_exception(): + with pytest.raises(ValueError) as excinfo: + # Used custom object on purpose to demonstrate a not-registered marking + @stix2.sdo.CustomObject('x-new-marking-type2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ]) + class NewObject2(object): + def __init__(self, property2=None, **kwargs): + return + + no = NewObject2(property1='something', property2=55) + + stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type2", + definition=no + ) + + assert str(excinfo.value) == "definition_type must be a valid marking type" + + +def test_marking_wrong_type_construction(): + with pytest.raises(ValueError) as excinfo: + # Test passing wrong type for properties. + @stix2.CustomMarking('x-new-marking-type2', ("a", "b")) + class NewObject3(object): + pass + + assert str(excinfo.value) == "Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]" + + # TODO: Add other examples diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py new file mode 100644 index 0000000..36e8e4d --- /dev/null +++ b/stix2/test/test_object_markings.py @@ -0,0 +1,534 @@ + +import pytest + +from stix2 import Malware, exceptions, markings + +from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS +from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST + +"""Tests for the Data Markings API.""" + +MALWARE_KWARGS = MALWARE_KWARGS_CONST.copy() +MALWARE_KWARGS.update({ + 'id': MALWARE_ID, + 'created': FAKE_TIME, + 'modified': FAKE_TIME, +}) + + +@pytest.mark.parametrize("data", [ + ( + Malware(**MALWARE_KWARGS), + Malware(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + ), + ( + MALWARE_KWARGS, + dict(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + ), +]) +def test_add_markings_one_marking(data): + before = data[0] + after = data[1] + + before = markings.add_markings(before, MARKING_IDS[0], None) + + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] + + +def test_add_markings_multiple_marking(): + before = Malware( + **MALWARE_KWARGS + ) + + after = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]], + **MALWARE_KWARGS + ) + + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], None) + + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] + + +def test_add_markings_combination(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]], + granular_markings=[ + { + "selectors": ["labels"], + "marking_ref": MARKING_IDS[2] + }, + { + "selectors": ["name"], + "marking_ref": MARKING_IDS[3] + } + ], + **MALWARE_KWARGS + ) + + before = markings.add_markings(before, MARKING_IDS[0], None) + before = markings.add_markings(before, MARKING_IDS[1], None) + before = markings.add_markings(before, MARKING_IDS[2], "labels") + before = markings.add_markings(before, MARKING_IDS[3], "name") + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] + + +@pytest.mark.parametrize("data", [ + ([""]), + (""), + ([]), + ([MARKING_IDS[0], 456]) +]) +def test_add_markings_bad_markings(data): + before = Malware( + **MALWARE_KWARGS + ) + with pytest.raises(exceptions.InvalidValueError): + before = markings.add_markings(before, data, None) + + assert "object_marking_refs" not in before + + +GET_MARKINGS_TEST_DATA = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "object_marking_refs": ["11"], + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_object_marking(data): + assert set(markings.get_markings(data, None)) == set(["11"]) + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_object_and_granular_combinations(data): + """Test multiple combinations for inherited and descendant markings.""" + assert set(markings.get_markings(data, "a", False, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, False)) == set(["1", "11"]) + assert set(markings.get_markings(data, "a", True, True)) == set(["1", "11"]) + assert set(markings.get_markings(data, "a", False, True)) == set(["1"]) + + assert set(markings.get_markings(data, "b", False, False)) == set([]) + assert set(markings.get_markings(data, "b", True, False)) == set(["11"]) + assert set(markings.get_markings(data, "b", True, True)) == set(["11"]) + assert set(markings.get_markings(data, "b", False, True)) == set([]) + + assert set(markings.get_markings(data, "c", False, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, False)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c", True, True)) == set(["2", "3", "4", "5", "11"]) + assert set(markings.get_markings(data, "c", False, True)) == set(["2", "3", "4", "5"]) + + assert set(markings.get_markings(data, "c.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "c.[0]", True, False)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c.[0]", True, True)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "c.[1]", False, False)) == set(["3"]) + assert set(markings.get_markings(data, "c.[1]", True, False)) == set(["2", "3", "11"]) + assert set(markings.get_markings(data, "c.[1]", True, True)) == set(["2", "3", "11"]) + assert set(markings.get_markings(data, "c.[1]", False, True)) == set(["3"]) + + assert set(markings.get_markings(data, "c.[2]", False, False)) == set(["4"]) + assert set(markings.get_markings(data, "c.[2]", True, False)) == set(["2", "4", "11"]) + assert set(markings.get_markings(data, "c.[2]", True, True)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2]", False, True)) == set(["4", "5"]) + + assert set(markings.get_markings(data, "c.[2].g", False, False)) == set(["5"]) + assert set(markings.get_markings(data, "c.[2].g", True, False)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2].g", True, True)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2].g", False, True)) == set(["5"]) + + assert set(markings.get_markings(data, "x", False, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, False)) == set(["6", "11"]) + assert set(markings.get_markings(data, "x", True, True)) == set(["6", "7", "8", "9", "10", "11"]) + assert set(markings.get_markings(data, "x", False, True)) == set(["6", "7", "8", "9", "10"]) + + assert set(markings.get_markings(data, "x.y", False, False)) == set(["7"]) + assert set(markings.get_markings(data, "x.y", True, False)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y", True, True)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y", False, True)) == set(["7", "8"]) + + assert set(markings.get_markings(data, "x.y.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "x.y.[0]", True, False)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y.[0]", True, True)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.y.[1]", False, False)) == set(["8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, False)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y.[1]", True, True)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y.[1]", False, True)) == set(["8"]) + + assert set(markings.get_markings(data, "x.z", False, False)) == set(["9"]) + assert set(markings.get_markings(data, "x.z", True, False)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z", True, True)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z", False, True)) == set(["9", "10"]) + + assert set(markings.get_markings(data, "x.z.foo1", False, False)) == set([]) + assert set(markings.get_markings(data, "x.z.foo1", True, False)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z.foo1", True, True)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z.foo1", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.z.foo2", False, False)) == set(["10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, False)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z.foo2", True, True)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) + + +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_remove_markings_object_level(data): + before = data[0] + after = data[1] + + before = markings.remove_markings(before, MARKING_IDS[0], None) + + assert 'object_marking_refs' not in before + assert 'object_marking_refs' not in after + + modified = after['modified'] + after = markings.remove_markings(after, MARKING_IDS[0], None) + modified == after['modified'] + + +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(object_marking_refs=[MARKING_IDS[1]], + **MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + dict(object_marking_refs=[MARKING_IDS[1]], + **MALWARE_KWARGS), + ), +]) +def test_remove_markings_multiple(data): + before = data[0] + after = data[1] + + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[2]], None) + + assert before['object_marking_refs'] == after['object_marking_refs'] + + +def test_remove_markings_bad_markings(): + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + with pytest.raises(AssertionError) as excinfo: + markings.remove_markings(before, [MARKING_IDS[4]], None) + assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] + + +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_clear_markings(data): + before = data[0] + after = data[1] + + before = markings.clear_markings(before, None) + + assert 'object_marking_refs' not in before + assert 'object_marking_refs' not in after + + +def test_is_marked_object_and_granular_combinations(): + """Test multiple combinations for inherited and descendant markings.""" + test_sdo = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "object_marking_refs": "11", + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + assert markings.is_marked(test_sdo, ["1"], "a", False, False) + assert markings.is_marked(test_sdo, ["1", "11"], "a", True, False) + assert markings.is_marked(test_sdo, ["1", "11"], "a", True, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, True) + + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["11"], "b", True, False) + assert markings.is_marked(test_sdo, ["11"], "b", True, True) + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["2"], "c", False, False) + assert markings.is_marked(test_sdo, ["2", "11"], "c", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5", "11"], "c", True, True) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", False, True) + + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["2", "11"], "c.[0]", True, False) + assert markings.is_marked(test_sdo, ["2", "11"], "c.[0]", True, True) + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, False) + assert markings.is_marked(test_sdo, ["2", "3", "11"], "c.[1]", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "11"], "c.[1]", True, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, True) + + assert markings.is_marked(test_sdo, ["4"], "c.[2]", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "11"], "c.[2]", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2]", True, True) + assert markings.is_marked(test_sdo, ["4", "5"], "c.[2]", False, True) + + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2].g", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2].g", True, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, True) + + assert markings.is_marked(test_sdo, ["6"], "x", False, False) + assert markings.is_marked(test_sdo, ["6", "11"], "x", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10", "11"], "x", True, True) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", False, True) + + assert markings.is_marked(test_sdo, ["7"], "x.y", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y", True, True) + assert markings.is_marked(test_sdo, ["7", "8"], "x.y", False, True) + + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y.[0]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y.[0]", True, True) + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y.[1]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y.[1]", True, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, True) + + assert markings.is_marked(test_sdo, ["9"], "x.z", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z", True, True) + assert markings.is_marked(test_sdo, ["9", "10"], "x.z", False, True) + + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z.foo1", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z.foo1", True, True) + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False + + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z.foo2", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z.foo2", True, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, True) + + assert markings.is_marked(test_sdo, ["11"], None, True, True) + assert markings.is_marked(test_sdo, ["2"], None, True, True) is False + + +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_is_marked_no_markings(data): + marked = data[0] + nonmarked = data[1] + + assert markings.is_marked(marked) + assert markings.is_marked(nonmarked) is False + + +def test_set_marking(): + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[4], MARKING_IDS[5]], + **MALWARE_KWARGS + ) + + before = markings.set_markings(before, [MARKING_IDS[4], MARKING_IDS[5]], None) + + for m in before["object_marking_refs"]: + assert m in [MARKING_IDS[4], MARKING_IDS[5]] + + assert [MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]] not in before["object_marking_refs"] + + for x in before["object_marking_refs"]: + assert x in after["object_marking_refs"] + + +@pytest.mark.parametrize("data", [ + ([]), + ([""]), + (""), + ([MARKING_IDS[4], 687]) +]) +def test_set_marking_bad_input(data): + before = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS + ) + with pytest.raises(exceptions.InvalidValueError): + before = markings.set_markings(before, data, None) + + assert before == after diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index d5641e7..3029b68 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -8,22 +8,24 @@ import stix2 from .constants import OBSERVED_DATA_ID +OBJECTS_REGEX = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL) + EXPECTED = """{ - "created": "2016-04-06T19:58:16.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "first_observed": "2015-12-21T19:00:00Z", + "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - "last_observed": "2015-12-21T19:00:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "objects": { "0": { - "name": "foo.exe", - "type": "file" + "type": "file", + "name": "foo.exe" } - }, - "type": "observed-data" + } }""" @@ -48,27 +50,27 @@ def test_observed_data_example(): EXPECTED_WITH_REF = """{ - "created": "2016-04-06T19:58:16.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "first_observed": "2015-12-21T19:00:00Z", + "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - "last_observed": "2015-12-21T19:00:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "objects": { "0": { - "name": "foo.exe", - "type": "file" + "type": "file", + "name": "foo.exe" }, "1": { + "type": "directory", + "path": "/usr/home", "contains_refs": [ "0" - ], - "path": "/usr/home", - "type": "directory" + ] } - }, - "type": "observed-data" + } }""" @@ -125,6 +127,42 @@ def test_observed_data_example_with_bad_refs(): assert excinfo.value.reason == "Invalid object reference for 'Directory:contains_refs': '2' is not a valid object in local scope" +def test_observed_data_example_with_non_dictionary(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects="file: foo.exe", + ) + + assert excinfo.value.cls == stix2.ObservedData + assert excinfo.value.prop_name == "objects" + assert 'must contain a dictionary' in excinfo.value.reason + + +def test_observed_data_example_with_empty_dictionary(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects={}, + ) + + assert excinfo.value.cls == stix2.ObservedData + assert excinfo.value.prop_name == "objects" + assert 'must contain a non-empty dictionary' in excinfo.value.reason + + @pytest.mark.parametrize("data", [ EXPECTED, { @@ -173,7 +211,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +232,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -204,6 +242,7 @@ def test_artifact_example_dependency_error(): stix2.Artifact(url="http://example.com/sirvizio.exe") assert excinfo.value.dependencies == [("hashes", "url")] + assert str(excinfo.value) == "The property dependencies for Artifact: (hashes, url) are not met." @pytest.mark.parametrize("data", [ @@ -215,7 +254,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +397,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" @@ -416,6 +455,8 @@ def test_parse_email_message_with_at_least_one_error(data): assert excinfo.value.cls == stix2.EmailMIMEComponent assert excinfo.value.properties == ["body", "body_raw_ref"] + assert "At least one of the" in str(excinfo.value) + assert "must be populated" in str(excinfo.value) @pytest.mark.parametrize("data", [ @@ -555,6 +596,7 @@ def test_artifact_mutual_exclusion_error(): assert excinfo.value.cls == stix2.Artifact assert excinfo.value.properties == ["payload_bin", "url"] + assert 'are mutually exclusive' in str(excinfo.value) def test_directory_example(): @@ -800,6 +842,8 @@ def test_file_example_encryption_error(): assert excinfo.value.cls == stix2.File assert excinfo.value.dependencies == [("is_encrypted", "encryption_algorithm")] + assert "property dependencies" in str(excinfo.value) + assert "are not met" in str(excinfo.value) with pytest.raises(stix2.exceptions.DependentPropertiesError) as excinfo: stix2.File(name="qwerty.dll", @@ -925,6 +969,10 @@ def test_process_example_empty_error(): properties_of_process = list(stix2.Process._properties.keys()) properties_of_process.remove("type") assert excinfo.value.properties == sorted(properties_of_process) + msg = "At least one of the ({1}) properties for {0} must be populated." + msg = msg.format(stix2.Process.__name__, + ", ".join(sorted(properties_of_process))) + assert str(excinfo.value) == msg def test_process_example_empty_with_extensions(): diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 01daebf..7d03b9e 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -5,10 +5,10 @@ from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError from stix2.observables import EmailMIMEComponent, ExtensionsProperty from stix2.properties import (BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, - EnumProperty, HashesProperty, HexProperty, - IDProperty, IntegerProperty, ListProperty, - Property, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty) + EnumProperty, FloatProperty, HashesProperty, + HexProperty, IDProperty, IntegerProperty, + ListProperty, Property, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty) from .constants import FAKE_TIME @@ -119,6 +119,27 @@ def test_integer_property_invalid(value): int_prop.clean(value) +@pytest.mark.parametrize("value", [ + 2, + -1, + 3.14, + False, +]) +def test_float_property_valid(value): + int_prop = FloatProperty() + assert int_prop.clean(value) is not None + + +@pytest.mark.parametrize("value", [ + "something", + StringProperty(), +]) +def test_float_property_invalid(value): + int_prop = FloatProperty() + with pytest.raises(ValueError): + int_prop.clean(value) + + @pytest.mark.parametrize("value", [ True, False, @@ -206,15 +227,42 @@ def test_dictionary_property_valid(d): @pytest.mark.parametrize("d", [ - {'a': 'something'}, - {'a'*300: 'something'}, - {'Hey!': 'something'}, + [{'a': 'something'}, "Invalid dictionary key a: (shorter than 3 characters)."], + [{'a'*300: 'something'}, "Invalid dictionary key aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaa: (longer than 256 characters)."], + [{'Hey!': 'something'}, "Invalid dictionary key Hey!: (contains characters other thanlowercase a-z, " + "uppercase A-Z, numerals 0-9, hyphen (-), or underscore (_))."], +]) +def test_dictionary_property_invalid_key(d): + dict_prop = DictionaryProperty() + + with pytest.raises(DictionaryKeyError) as excinfo: + dict_prop.clean(d[0]) + + assert str(excinfo.value) == d[1] + + +@pytest.mark.parametrize("d", [ + ({}, "The dictionary property must contain a non-empty dictionary"), + # TODO: This error message could be made more helpful. The error is caused + # because `json.loads()` doesn't like the *single* quotes around the key + # name, even though they are valid in a Python dictionary. While technically + # accurate (a string is not a dictionary), if we want to be able to load + # string-encoded "dictionaries" that are, we need a better error message + # or an alternative to `json.loads()` ... and preferably *not* `eval()`. :-) + # Changing the following to `'{"description": "something"}'` does not cause + # any ValueError to be raised. + ("{'description': 'something'}", "The dictionary property must contain a dictionary"), ]) def test_dictionary_property_invalid(d): dict_prop = DictionaryProperty() - with pytest.raises(DictionaryKeyError): - dict_prop.clean(d) + with pytest.raises(ValueError) as excinfo: + dict_prop.clean(d[0]) + assert str(excinfo.value) == d[1] @pytest.mark.parametrize("value", [ @@ -250,10 +298,18 @@ def test_embedded_property(): emb_prop.clean("string") -def test_enum_property(): - enum_prop = EnumProperty(['a', 'b', 'c']) +@pytest.mark.parametrize("value", [ + ['a', 'b', 'c'], + ('a', 'b', 'c'), + 'b', +]) +def test_enum_property_valid(value): + enum_prop = EnumProperty(value) assert enum_prop.clean('b') + +def test_enum_property_invalid(): + enum_prop = EnumProperty(['a', 'b', 'c']) with pytest.raises(ValueError): enum_prop.clean('z') diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py index 362348b..6d65544 100644 --- a/stix2/test/test_relationship.py +++ b/stix2/test/test_relationship.py @@ -10,13 +10,13 @@ from .constants import (FAKE_TIME, INDICATOR_ID, MALWARE_ID, RELATIONSHIP_ID, EXPECTED_RELATIONSHIP = """{ - "created": "2016-04-06T20:06:37.000Z", + "type": "relationship", "id": "relationship--00000000-1111-2222-3333-444444444444", + "created": "2016-04-06T20:06:37.000Z", "modified": "2016-04-06T20:06:37.000Z", "relationship_type": "indicates", "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "type": "relationship" + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" }""" diff --git a/stix2/test/test_report.py b/stix2/test/test_report.py index 4a1f905..a5775e3 100644 --- a/stix2/test/test_report.py +++ b/stix2/test/test_report.py @@ -9,22 +9,22 @@ from .constants import INDICATOR_KWARGS, REPORT_ID EXPECTED = """{ - "created": "2015-12-21T19:59:11.000Z", - "created_by_ref": "identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", - "description": "A simple report with an indicator and campaign", + "type": "report", "id": "report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3", - "labels": [ - "campaign" - ], + "created_by_ref": "identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", + "created": "2015-12-21T19:59:11.000Z", "modified": "2015-12-21T19:59:11.000Z", "name": "The Black Vine Cyberespionage Group", + "description": "A simple report with an indicator and campaign", + "published": "2016-01-20T17:00:00Z", "object_refs": [ "indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", "campaign--83422c77-904c-4dc1-aff5-5c38f3a2c55c", "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a" ], - "published": "2016-01-20T17:00:00Z", - "type": "report" + "labels": [ + "campaign" + ] }""" diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py index 2036457..af91413 100644 --- a/stix2/test/test_sighting.py +++ b/stix2/test/test_sighting.py @@ -9,11 +9,11 @@ from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS EXPECTED_SIGHTING = """{ - "created": "2016-04-06T20:06:37.000Z", + "type": "sighting", "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb", + "created": "2016-04-06T20:06:37.000Z", "modified": "2016-04-06T20:06:37.000Z", "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "type": "sighting", "where_sighted_refs": [ "identity--8cc7afd6-5455-4d2b-a736-e614ee631d99" ] diff --git a/stix2/test/test_threat_actor.py b/stix2/test/test_threat_actor.py index 1bab744..c095c3b 100644 --- a/stix2/test/test_threat_actor.py +++ b/stix2/test/test_threat_actor.py @@ -9,16 +9,16 @@ from .constants import THREAT_ACTOR_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "The Evil Org threat actor group", + "type": "threat-actor", "id": "threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "labels": [ - "crime-syndicate" - ], + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "Evil Org", - "type": "threat-actor" + "description": "The Evil Org threat actor group", + "labels": [ + "crime-syndicate" + ] }""" diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py index 04da7b3..be52f6d 100644 --- a/stix2/test/test_tool.py +++ b/stix2/test/test_tool.py @@ -9,15 +9,15 @@ from .constants import TOOL_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "type": "tool", "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "labels": [ - "remote-access" - ], + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "VNC", - "type": "tool" + "labels": [ + "remote-access" + ] }""" diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 281ae71..8695a30 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -2,16 +2,11 @@ import pytest import stix2 +from .constants import CAMPAIGN_MORE_KWARGS + def test_making_new_version(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.new_version(name="fred") @@ -25,14 +20,7 @@ def test_making_new_version(): def test_making_new_version_with_unset(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.new_version(description=None) @@ -47,16 +35,11 @@ def test_making_new_version_with_unset(): def test_making_new_version_with_embedded_object(): campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", external_references=[{ "source_name": "capec", "external_id": "CAPEC-163" }], - description="Campaign by Green Group against a series of targets in the financial services sector." + **CAMPAIGN_MORE_KWARGS ) campaign_v2 = campaign_v1.new_version(external_references=[{ @@ -74,14 +57,7 @@ def test_making_new_version_with_embedded_object(): def test_revoke(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() @@ -96,14 +72,7 @@ def test_revoke(): def test_versioning_error_invalid_property(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.UnmodifiablePropertyError) as excinfo: campaign_v1.new_version(type="threat-actor") @@ -112,14 +81,7 @@ def test_versioning_error_invalid_property(): def test_versioning_error_bad_modified_value(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: campaign_v1.new_version(modified="2015-04-06T20:03:00.000Z") @@ -128,16 +90,14 @@ def test_versioning_error_bad_modified_value(): assert excinfo.value.prop_name == "modified" assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + msg = "Invalid value for {0} '{1}': {2}" + msg = msg.format(stix2.Campaign.__name__, "modified", + "The new modified datetime cannot be before the current modified datatime.") + assert str(excinfo.value) == msg + def test_versioning_error_usetting_required_property(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: campaign_v1.new_version(name=None) @@ -145,38 +105,104 @@ def test_versioning_error_usetting_required_property(): assert excinfo.value.cls == stix2.Campaign assert excinfo.value.properties == ["name"] + msg = "No values for required properties for {0}: ({1})." + msg = msg.format(stix2.Campaign.__name__, "name") + assert str(excinfo.value) == msg + def test_versioning_error_new_version_of_revoked(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) - + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() with pytest.raises(stix2.exceptions.RevokeError) as excinfo: campaign_v2.new_version(name="barney") + assert str(excinfo.value) == "Cannot create a new version of a revoked object." assert excinfo.value.called_by == "new_version" + assert str(excinfo.value) == "Cannot create a new version of a revoked object." def test_versioning_error_revoke_of_revoked(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) - + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() with pytest.raises(stix2.exceptions.RevokeError) as excinfo: campaign_v2.revoke() + assert str(excinfo.value) == "Cannot revoke an already revoked object." assert excinfo.value.called_by == "revoke" + assert str(excinfo.value) == "Cannot revoke an already revoked object." + + +def test_making_new_version_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.new_version(CAMPAIGN_MORE_KWARGS, name="fred") + + assert campaign_v1['id'] == campaign_v2['id'] + assert campaign_v1['created_by_ref'] == campaign_v2['created_by_ref'] + assert campaign_v1['created'] == campaign_v2['created'] + assert campaign_v1['name'] != campaign_v2['name'] + assert campaign_v2['name'] == "fred" + assert campaign_v1['description'] == campaign_v2['description'] + assert stix2.utils.parse_into_datetime(campaign_v1['modified'], precision='millisecond') < campaign_v2['modified'] + + +def test_versioning_error_dict_bad_modified_value(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.utils.new_version(CAMPAIGN_MORE_KWARGS, modified="2015-04-06T20:03:00.000Z") + + assert excinfo.value.cls == dict + assert excinfo.value.prop_name == "modified" + assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + + +def test_versioning_error_dict_no_modified_value(): + campaign_v1 = { + 'type': 'campaign', + 'id': "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + 'created': "2016-04-06T20:03:00.000Z", + 'name': "Green Group Attacks Against Finance", + } + campaign_v2 = stix2.utils.new_version(campaign_v1, modified="2017-04-06T20:03:00.000Z") + + assert str(campaign_v2['modified']) == "2017-04-06T20:03:00.000Z" + + +def test_making_new_version_invalid_cls(): + campaign_v1 = "This is a campaign." + with pytest.raises(ValueError) as excinfo: + stix2.utils.new_version(campaign_v1, name="fred") + + assert 'cannot create new version of object of this type' in str(excinfo.value) + + +def test_revoke_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.revoke(campaign_v1) + + assert campaign_v1['id'] == campaign_v2['id'] + assert campaign_v1['created_by_ref'] == campaign_v2['created_by_ref'] + assert campaign_v1['created'] == campaign_v2['created'] + assert campaign_v1['name'] == campaign_v2['name'] + assert campaign_v1['description'] == campaign_v2['description'] + assert stix2.utils.parse_into_datetime(campaign_v1['modified'], precision='millisecond') < campaign_v2['modified'] + + assert campaign_v2['revoked'] + + +def test_versioning_error_revoke_of_revoked_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.revoke(campaign_v1) + + with pytest.raises(stix2.exceptions.RevokeError) as excinfo: + stix2.utils.revoke(campaign_v2) + + assert excinfo.value.called_by == "revoke" + + +def test_revoke_invalid_cls(): + campaign_v1 = "This is a campaign." + with pytest.raises(ValueError) as excinfo: + stix2.utils.revoke(campaign_v1) + + assert 'cannot revoke object of this type' in str(excinfo.value) diff --git a/stix2/test/test_vulnerability.py b/stix2/test/test_vulnerability.py index 27ab85f..a6426d8 100644 --- a/stix2/test/test_vulnerability.py +++ b/stix2/test/test_vulnerability.py @@ -9,17 +9,17 @@ from .constants import VULNERABILITY_ID EXPECTED = """{ - "created": "2016-05-12T08:17:27.000Z", - "external_references": [ - { - "external_id": "CVE-2016-1234", - "source_name": "cve" - } - ], + "type": "vulnerability", "id": "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", + "created": "2016-05-12T08:17:27.000Z", "modified": "2016-05-12T08:17:27.000Z", "name": "CVE-2016-1234", - "type": "vulnerability" + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2016-1234" + } + ] }""" diff --git a/stix2/utils.py b/stix2/utils.py index 12b889c..ca195f6 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,11 +1,16 @@ """Utility functions and classes for the stix2 library.""" +from collections import Mapping +import copy import datetime as dt import json from dateutil import parser import pytz +from .exceptions import (InvalidValueError, RevokeError, + UnmodifiablePropertyError) + # Sentinel value for properties that should be set to the current time. # We can't use the standard 'default' approach, since if there are multiple # timestamps in a single object, the timestamps will vary by a few microseconds. @@ -24,6 +29,9 @@ class STIXdatetime(dt.datetime): self.precision = precision return self + def __repr__(self): + return "'%s'" % format_datetime(self) + def get_timestamp(): return STIXdatetime.now(tz=pytz.UTC) @@ -77,7 +85,7 @@ def parse_into_datetime(value, precision=None): # Ensure correct precision if not precision: - return ts + return STIXdatetime(ts, precision=precision) ms = ts.microsecond if precision == 'second': ts = ts.replace(microsecond=0) @@ -112,3 +120,86 @@ def get_dict(data): return dict(data) except (ValueError, TypeError): raise ValueError("Cannot convert '%s' to dictionary." % str(data)) + + +def find_property_index(obj, properties, tuple_to_find): + """Recursively find the property in the object model, return the index + according to the _properties OrderedDict. If its a list look for + individual objects. + """ + from .base import _STIXBase + try: + if tuple_to_find[1] in obj._inner.values(): + return properties.index(tuple_to_find[0]) + raise ValueError + except ValueError: + for pv in obj._inner.values(): + if isinstance(pv, list): + for item in pv: + if isinstance(item, _STIXBase): + val = find_property_index(item, + item.object_properties(), + tuple_to_find) + if val is not None: + return val + elif isinstance(pv, dict): + if pv.get(tuple_to_find[0]) is not None: + try: + return int(tuple_to_find[0]) + except ValueError: + return len(tuple_to_find[0]) + for item in pv.values(): + if isinstance(item, _STIXBase): + val = find_property_index(item, + item.object_properties(), + tuple_to_find) + if val is not None: + return val + + +def new_version(data, **kwargs): + """Create a new version of a STIX object, by modifying properties and + updating the `modified` property. + """ + + if not isinstance(data, Mapping): + raise ValueError('cannot create new version of object of this type! ' + 'Try a dictionary or instance of an SDO or SRO class.') + + unchangable_properties = [] + if data.get("revoked"): + raise RevokeError("new_version") + try: + new_obj_inner = copy.deepcopy(data._inner) + except AttributeError: + new_obj_inner = copy.deepcopy(data) + properties_to_change = kwargs.keys() + + # Make sure certain properties aren't trying to change + for prop in ["created", "created_by_ref", "id", "type"]: + if prop in properties_to_change: + unchangable_properties.append(prop) + if unchangable_properties: + raise UnmodifiablePropertyError(unchangable_properties) + + cls = type(data) + if 'modified' not in kwargs: + kwargs['modified'] = get_timestamp() + elif 'modified' in data: + old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond') + new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') + if new_modified_property < old_modified_property: + raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") + new_obj_inner.update(kwargs) + # Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass + return cls(**{k: v for k, v in new_obj_inner.items() if v is not None}) + + +def revoke(data): + if not isinstance(data, Mapping): + raise ValueError('cannot revoke object of this type! Try a dictionary ' + 'or instance of an SDO or SRO class.') + + if data.get("revoked"): + raise RevokeError("revoke") + return new_version(data, revoked=True) diff --git a/tox.ini b/tox.ini index b1265ec..bca18c3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,py35,py36,pycodestyle,isort-check +envlist = py27,py33,py34,py35,py36,pycodestyle,isort-check [testenv] deps = @@ -36,7 +36,6 @@ commands = [travis] python = - 2.6: py26 2.7: py27, pycodestyle 3.3: py33, pycodestyle 3.4: py34, pycodestyle