From d2f960f2fc962bf6a8f096c848b12e490d2c2037 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 18 Jun 2020 20:49:25 -0400 Subject: [PATCH 01/21] Initial revamp of customization enforcement and detection. --- stix2/base.py | 99 ++++++--- stix2/properties.py | 307 +++++++++++++++++---------- stix2/test/test_properties.py | 65 +++--- stix2/test/test_workbench.py | 2 +- stix2/test/v20/test_bundle.py | 2 +- stix2/test/v20/test_custom.py | 79 ++++++- stix2/test/v20/test_properties.py | 325 ++++++++++++++++++++++++++--- stix2/test/v21/test_bundle.py | 2 +- stix2/test/v21/test_custom.py | 118 ++++++++++- stix2/test/v21/test_indicator.py | 1 + stix2/test/v21/test_properties.py | 330 ++++++++++++++++++++++++++++-- stix2/v20/bundle.py | 3 - stix2/v20/common.py | 4 +- stix2/v20/sdo.py | 6 - stix2/v21/bundle.py | 3 - stix2/v21/common.py | 4 +- stix2/v21/sdo.py | 2 - stix2/versioning.py | 7 +- 18 files changed, 1119 insertions(+), 240 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 2c48ef6..c0e52e6 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -1,6 +1,7 @@ """Base classes for type definitions in the STIX2 library.""" import copy +import itertools import re import uuid @@ -12,7 +13,7 @@ from stix2.canonicalization.Canonicalize import canonicalize from .exceptions import ( AtLeastOnePropertyError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, InvalidObjRefError, InvalidValueError, - MissingPropertiesError, MutuallyExclusivePropertiesError, + MissingPropertiesError, MutuallyExclusivePropertiesError, STIXError, ) from .markings import _MarkingsMixin from .markings.utils import validate @@ -54,7 +55,7 @@ class _STIXBase(Mapping): return all_properties - def _check_property(self, prop_name, prop, kwargs): + def _check_property(self, prop_name, prop, kwargs, allow_custom): if prop_name not in kwargs: if hasattr(prop, 'default'): value = prop.default() @@ -62,9 +63,12 @@ class _STIXBase(Mapping): value = self.__now kwargs[prop_name] = value + has_custom = False if prop_name in kwargs: try: - kwargs[prop_name] = prop.clean(kwargs[prop_name]) + kwargs[prop_name], has_custom = prop.clean( + kwargs[prop_name], allow_custom, + ) except InvalidValueError: # No point in wrapping InvalidValueError in another # InvalidValueError... so let those propagate. @@ -74,6 +78,8 @@ class _STIXBase(Mapping): self.__class__, prop_name, reason=str(exc), ) from exc + return has_custom + # interproperty constraint methods def _check_mutually_exclusive_properties(self, list_of_properties, at_least_one=True): @@ -113,7 +119,6 @@ class _STIXBase(Mapping): def __init__(self, allow_custom=False, **kwargs): cls = self.__class__ - self._allow_custom = allow_custom # Use the same timestamp for any auto-generated datetimes self.__now = get_timestamp() @@ -123,16 +128,18 @@ class _STIXBase(Mapping): if custom_props and not isinstance(custom_props, dict): raise ValueError("'custom_properties' must be a dictionary") - extra_kwargs = list(set(kwargs) - set(self._properties)) - if extra_kwargs and not self._allow_custom: + extra_kwargs = kwargs.keys() - self._properties.keys() + if extra_kwargs and not allow_custom: raise ExtraPropertiesError(cls, extra_kwargs) - # because allow_custom is true, any extra kwargs are custom - if custom_props or extra_kwargs: - self._allow_custom = True - if isinstance(self, stix2.v21._STIXBase21): - all_custom_prop_names = extra_kwargs - all_custom_prop_names.extend(list(custom_props.keys())) + if custom_props: + # loophole for custom_properties... + allow_custom = True + + all_custom_prop_names = extra_kwargs | custom_props.keys() - \ + self._properties.keys() + if all_custom_prop_names: + if not isinstance(self, stix2.v20._STIXBase20): for prop_name in all_custom_prop_names: if not re.match(PREFIX_21_REGEX, prop_name): raise InvalidValueError( @@ -141,12 +148,11 @@ class _STIXBase(Mapping): ) # Remove any keyword arguments whose value is None or [] (i.e. empty list) - setting_kwargs = {} - props = kwargs.copy() - props.update(custom_props) - for prop_name, prop_value in props.items(): - if prop_value is not None and prop_value != []: - setting_kwargs[prop_name] = prop_value + setting_kwargs = { + k: v + for k, v in itertools.chain(kwargs.items(), custom_props.items()) + if v is not None and v != [] + } # Detect any missing required properties required_properties = set(get_required_properties(self._properties)) @@ -154,8 +160,13 @@ class _STIXBase(Mapping): if missing_kwargs: raise MissingPropertiesError(cls, missing_kwargs) + has_custom = bool(all_custom_prop_names) for prop_name, prop_metadata in self._properties.items(): - self._check_property(prop_name, prop_metadata, setting_kwargs) + temp_custom = self._check_property( + prop_name, prop_metadata, setting_kwargs, allow_custom, + ) + + has_custom = has_custom or temp_custom # Cache defaulted optional properties for serialization defaulted = [] @@ -174,6 +185,22 @@ class _STIXBase(Mapping): self._check_object_constraints() + if allow_custom: + self.__has_custom = has_custom + + else: + # The simple case: our property cleaners are supposed to do their + # job and prevent customizations, so we just set to False. But + # this sanity check is helpful for finding bugs in those clean() + # methods. + if has_custom: + raise STIXError( + "Internal error: a clean() method did not properly enforce " + "allow_custom=False!", + ) + + self.__has_custom = False + def __getitem__(self, key): return self._inner[key] @@ -220,13 +247,16 @@ class _STIXBase(Mapping): if isinstance(self, _Observable): # Assume: valid references in the original object are still valid in the new version new_inner['_valid_refs'] = {'*': '*'} - new_inner['allow_custom'] = self._allow_custom - return cls(**new_inner) + return cls(allow_custom=True, **new_inner) def properties_populated(self): return list(self._inner.keys()) -# Versioning API + @property + def has_custom(self): + return self.__has_custom + + # Versioning API def new_version(self, **kwargs): return _new_version(self, **kwargs) @@ -348,20 +378,21 @@ class _Observable(_STIXBase): if ref_type not in allowed_types: raise InvalidObjRefError(self.__class__, prop_name, "object reference '%s' is of an invalid type '%s'" % (ref, ref_type)) - def _check_property(self, prop_name, prop, kwargs): - super(_Observable, self)._check_property(prop_name, prop, kwargs) - if prop_name not in kwargs: - return + def _check_property(self, prop_name, prop, kwargs, allow_custom): + has_custom = super(_Observable, self)._check_property(prop_name, prop, kwargs, allow_custom) - from .properties import ObjectReferenceProperty - if prop_name.endswith('_ref'): - if isinstance(prop, ObjectReferenceProperty): - ref = kwargs[prop_name] - self._check_ref(ref, prop, prop_name) - elif prop_name.endswith('_refs'): - if isinstance(prop.contained, ObjectReferenceProperty): - for ref in kwargs[prop_name]: + if prop_name in kwargs: + from .properties import ObjectReferenceProperty + if prop_name.endswith('_ref'): + if isinstance(prop, ObjectReferenceProperty): + ref = kwargs[prop_name] self._check_ref(ref, prop, prop_name) + elif prop_name.endswith('_refs'): + if isinstance(prop.contained, ObjectReferenceProperty): + for ref in kwargs[prop_name]: + self._check_ref(ref, prop, prop_name) + + return has_custom def _generate_id(self): """ diff --git a/stix2/properties.py b/stix2/properties.py index dbbe667..85d4e08 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -134,11 +134,23 @@ class Property(object): Subclasses can also define the following functions: - - ``def clean(self, value) -> any:`` - - Return a value that is valid for this property. If ``value`` is not - valid for this property, this will attempt to transform it first. If - ``value`` is not valid and no such transformation is possible, it - should raise an exception. + - ``def clean(self, value, allow_custom) -> (any, has_custom):`` + - Return a value that is valid for this property, and enforce and + detect value customization. If ``value`` is not valid for this + property, you may attempt to transform it first. If ``value`` is not + valid and no such transformation is possible, it must raise an + exception. The method is also responsible for enforcing and + detecting customizations. If allow_custom is False, no customizations + must be allowed. If any are encountered, an exception must be raised + (e.g. CustomContentError). If none are encountered, False must be + returned for has_custom. If allow_custom is True, then the clean() + method is responsible for detecting any customizations in the value + (just because the user has elected to allow customizations doesn't + mean there actually are any). The method must return an appropriate + value for has_custom. Customization may not be applicable/possible + for a property. In that case, allow_custom can be ignored, and + has_custom must be returned as False. + - ``def default(self):`` - provide a default value for this property. - ``default()`` can return the special value ``NOW`` to use the current @@ -159,10 +171,10 @@ class Property(object): """ - def _default_clean(self, value): + def _default_clean(self, value, allow_custom=False): if value != self._fixed_value: raise ValueError("must equal '{}'.".format(self._fixed_value)) - return value + return value, False def __init__(self, required=False, fixed=None, default=None): self.required = required @@ -180,14 +192,8 @@ class Property(object): if default: self.default = default - def clean(self, value): - return value - - def __call__(self, value=None): - """Used by ListProperty to handle lists that have been defined with - either a class or an instance. - """ - return value + def clean(self, value, allow_custom=False): + return value, False class ListProperty(Property): @@ -219,7 +225,7 @@ class ListProperty(Property): super(ListProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom): try: iter(value) except TypeError: @@ -228,21 +234,22 @@ class ListProperty(Property): if isinstance(value, (_STIXBase, str)): value = [value] + result = [] + has_custom = False if isinstance(self.contained, Property): - result = [ - self.contained.clean(item) - for item in value - ] + for item in value: + valid, temp_custom = self.contained.clean(item, allow_custom) + result.append(valid) + has_custom = has_custom or temp_custom else: # self.contained must be a _STIXBase subclass - result = [] for item in value: if isinstance(item, self.contained): valid = item elif isinstance(item, Mapping): # attempt a mapping-like usage... - valid = self.contained(**item) + valid = self.contained(allow_custom=allow_custom, **item) else: raise ValueError( @@ -252,12 +259,16 @@ class ListProperty(Property): ) result.append(valid) + has_custom = has_custom or valid.has_custom + + if not allow_custom and has_custom: + raise CustomContentError("custom content encountered") # STIX spec forbids empty lists if len(result) < 1: raise ValueError("must not be empty.") - return result + return result, has_custom class StringProperty(Property): @@ -265,10 +276,10 @@ class StringProperty(Property): def __init__(self, **kwargs): super(StringProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom=False): if not isinstance(value, str): - return str(value) - return value + value = str(value) + return value, False class TypeProperty(Property): @@ -286,9 +297,9 @@ class IDProperty(Property): self.spec_version = spec_version super(IDProperty, self).__init__() - def clean(self, value): + def clean(self, value, allow_custom=False): _validate_id(value, self.spec_version, self.required_prefix) - return value + return value, False def default(self): return self.required_prefix + str(uuid.uuid4()) @@ -301,7 +312,7 @@ class IntegerProperty(Property): self.max = max super(IntegerProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom=False): try: value = int(value) except Exception: @@ -315,7 +326,7 @@ class IntegerProperty(Property): msg = "maximum value is {}. received {}".format(self.max, value) raise ValueError(msg) - return value + return value, False class FloatProperty(Property): @@ -325,7 +336,7 @@ class FloatProperty(Property): self.max = max super(FloatProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom=False): try: value = float(value) except Exception: @@ -339,29 +350,26 @@ class FloatProperty(Property): msg = "maximum value is {}. received {}".format(self.max, value) raise ValueError(msg) - return value + return value, False class BooleanProperty(Property): + _trues = ['true', 't', '1', 1, True] + _falses = ['false', 'f', '0', 0, False] - def clean(self, value): - if isinstance(value, bool): - return value + def clean(self, value, allow_custom=False): - trues = ['true', 't', '1'] - falses = ['false', 'f', '0'] - try: - if value.lower() in trues: - return True - if value.lower() in falses: - return False - except AttributeError: - if value == 1: - return True - if value == 0: - return False + if isinstance(value, str): + value = value.lower() - raise ValueError("must be a boolean value.") + if value in self._trues: + result = True + elif value in self._falses: + result = False + else: + raise ValueError("must be a boolean value.") + + return result, False class TimestampProperty(Property): @@ -372,10 +380,10 @@ class TimestampProperty(Property): super(TimestampProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom=False): return parse_into_datetime( value, self.precision, self.precision_constraint, - ) + ), False class DictionaryProperty(Property): @@ -384,7 +392,7 @@ class DictionaryProperty(Property): self.spec_version = spec_version super(DictionaryProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom=False): try: dictified = _get_dict(value) except ValueError: @@ -409,7 +417,7 @@ class DictionaryProperty(Property): if len(dictified) < 1: raise ValueError("must not be empty.") - return dictified + return dictified, False HASHES_REGEX = { @@ -433,8 +441,14 @@ HASHES_REGEX = { class HashesProperty(DictionaryProperty): - def clean(self, value): - clean_dict = super(HashesProperty, self).clean(value) + def clean(self, value, allow_custom): + # ignore the has_custom return value here; there is no customization + # of DictionaryProperties. + clean_dict, _ = super(HashesProperty, self).clean( + value, allow_custom, + ) + + has_custom = False for k, v in copy.deepcopy(clean_dict).items(): key = k.upper().replace('-', '') if key in HASHES_REGEX: @@ -446,25 +460,32 @@ class HashesProperty(DictionaryProperty): if k != vocab_key: clean_dict[vocab_key] = clean_dict[k] del clean_dict[k] - return clean_dict + + else: + has_custom = True + + if not allow_custom and has_custom: + raise CustomContentError("custom hash found: " + k) + + return clean_dict, has_custom class BinaryProperty(Property): - def clean(self, value): + def clean(self, value, allow_custom=False): try: base64.b64decode(value) except (binascii.Error, TypeError): raise ValueError("must contain a base64 encoded string") - return value + return value, False class HexProperty(Property): - def clean(self, value): + def clean(self, value, allow_custom=False): if not re.match(r"^([a-fA-F0-9]{2})+$", value): raise ValueError("must contain an even number of hexadecimal characters") - return value + return value, False class ReferenceProperty(Property): @@ -493,31 +514,52 @@ class ReferenceProperty(Property): super(ReferenceProperty, self).__init__(**kwargs) - def clean(self, value): + def clean(self, value, allow_custom): if isinstance(value, _STIXBase): value = value.id value = str(value) - possible_prefix = value[:value.index('--')] + _validate_id(value, self.spec_version, None) + + obj_type = value[:value.index('--')] if self.valid_types: + # allow_custom is not applicable to "whitelist" style object type + # constraints, so we ignore it. + has_custom = False + ref_valid_types = enumerate_types(self.valid_types, self.spec_version) - if possible_prefix in ref_valid_types: - required_prefix = possible_prefix - else: - raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (possible_prefix)) - elif self.invalid_types: + if obj_type not in ref_valid_types: + raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (obj_type)) + + else: + # A type "blacklist" was used to describe legal object types. + # We must enforce the type blacklist regardless of allow_custom. ref_invalid_types = enumerate_types(self.invalid_types, self.spec_version) - if possible_prefix not in ref_invalid_types: - required_prefix = possible_prefix - else: - raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix)) + if obj_type in ref_invalid_types: + raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (obj_type)) - _validate_id(value, self.spec_version, required_prefix) + # allow_custom=True only allows references to custom objects which + # are not otherwise blacklisted. So we need to figure out whether + # the referenced object is custom or not. No good way to do that + # at present... just check if unregistered and for the "x-" type + # prefix, for now? + type_maps = STIX2_OBJ_MAPS[self.spec_version] - return value + has_custom = obj_type not in type_maps["objects"] \ + and obj_type not in type_maps["observables"] \ + and obj_type not in ["relationship", "sighting"] + + has_custom = has_custom or obj_type.startswith("x-") + + if not allow_custom and has_custom: + raise CustomContentError( + "reference to custom object type: " + obj_type, + ) + + return value, has_custom def enumerate_types(types, spec_version): @@ -529,8 +571,7 @@ def enumerate_types(types, spec_version): once each of those words is being processed, that word will be removed from `return_types`, so as not to mistakenly allow objects to be created of types "SCO", "SDO", or "SRO" """ - return_types = [] - return_types += types + return_types = types[:] if "SDO" in types: return_types.remove("SDO") @@ -550,10 +591,10 @@ SELECTOR_REGEX = re.compile(r"^([a-z0-9_-]{3,250}(\.(\[\d+\]|[a-z0-9_-]{1,250})) class SelectorProperty(Property): - def clean(self, value): + def clean(self, value, allow_custom=False): if not SELECTOR_REGEX.match(value): raise ValueError("must adhere to selector syntax.") - return value + return value, False class ObjectReferenceProperty(StringProperty): @@ -571,12 +612,20 @@ class EmbeddedObjectProperty(Property): self.type = type super(EmbeddedObjectProperty, self).__init__(**kwargs) - def clean(self, value): - if type(value) is dict: - value = self.type(**value) + def clean(self, value, allow_custom): + if isinstance(value, dict): + value = self.type(allow_custom=allow_custom, **value) elif not isinstance(value, self.type): raise ValueError("must be of type {}.".format(self.type.__name__)) - return value + + has_custom = False + if isinstance(value, _STIXBase): + has_custom = value.has_custom + + if not allow_custom and has_custom: + raise CustomContentError("custom content encountered") + + return value, has_custom class EnumProperty(StringProperty): @@ -587,12 +636,14 @@ class EnumProperty(StringProperty): self.allowed = allowed super(EnumProperty, self).__init__(**kwargs) - def clean(self, value): - cleaned_value = super(EnumProperty, self).clean(value) - if cleaned_value not in self.allowed: + def clean(self, value, allow_custom): + cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom) + has_custom = cleaned_value not in self.allowed + + if not allow_custom and has_custom: raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value)) - return cleaned_value + return cleaned_value, has_custom class PatternProperty(StringProperty): @@ -603,12 +654,11 @@ class ObservableProperty(Property): """Property for holding Cyber Observable Objects. """ - def __init__(self, spec_version=DEFAULT_VERSION, allow_custom=False, *args, **kwargs): - self.allow_custom = allow_custom + def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs): self.spec_version = spec_version super(ObservableProperty, self).__init__(*args, **kwargs) - def clean(self, value): + def clean(self, value, allow_custom): try: dictified = _get_dict(value) # get deep copy since we are going modify the dict and might @@ -622,28 +672,43 @@ class ObservableProperty(Property): valid_refs = dict((k, v['type']) for (k, v) in dictified.items()) + has_custom = False for key, obj in dictified.items(): parsed_obj = parse_observable( obj, valid_refs, - allow_custom=self.allow_custom, + allow_custom=allow_custom, version=self.spec_version, ) + + if isinstance(parsed_obj, _STIXBase): + has_custom = has_custom or parsed_obj.has_custom + else: + # we get dicts for unregistered custom objects + has_custom = True + + if not allow_custom and has_custom: + if parsed_obj.has_custom: + raise CustomContentError( + "customized {} observable found".format( + parsed_obj["type"], + ), + ) + dictified[key] = parsed_obj - return dictified + return dictified, has_custom class ExtensionsProperty(DictionaryProperty): """Property for representing extensions on Observable objects. """ - def __init__(self, spec_version=DEFAULT_VERSION, allow_custom=False, enclosing_type=None, required=False): - self.allow_custom = allow_custom + def __init__(self, spec_version=DEFAULT_VERSION, enclosing_type=None, required=False): self.enclosing_type = enclosing_type super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required) - def clean(self, value): + def clean(self, value, allow_custom): try: dictified = _get_dict(value) # get deep copy since we are going modify the dict and might @@ -653,37 +718,47 @@ class ExtensionsProperty(DictionaryProperty): except ValueError: raise ValueError("The extensions property must contain a dictionary") + has_custom = False specific_type_map = STIX2_OBJ_MAPS[self.spec_version]['observable-extensions'].get(self.enclosing_type, {}) for key, subvalue in dictified.items(): if key in specific_type_map: cls = specific_type_map[key] if type(subvalue) is dict: - if self.allow_custom: - subvalue['allow_custom'] = True - dictified[key] = cls(**subvalue) - else: - dictified[key] = cls(**subvalue) + ext = cls(allow_custom=allow_custom, **subvalue) elif type(subvalue) is cls: # If already an instance of an _Extension class, assume it's valid - dictified[key] = subvalue + ext = subvalue else: raise ValueError("Cannot determine extension type.") + + has_custom = has_custom or ext.has_custom + + if not allow_custom and has_custom: + raise CustomContentError( + "custom content found in {} extension".format( + key, + ), + ) + + dictified[key] = ext + else: - if self.allow_custom: + if allow_custom: + has_custom = True dictified[key] = subvalue else: raise CustomContentError("Can't parse unknown extension type: {}".format(key)) - return dictified + + return dictified, has_custom class STIXObjectProperty(Property): - def __init__(self, spec_version=DEFAULT_VERSION, allow_custom=False, *args, **kwargs): - self.allow_custom = allow_custom + def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs): self.spec_version = spec_version super(STIXObjectProperty, self).__init__(*args, **kwargs) - def clean(self, value): + def clean(self, value, allow_custom): # Any STIX Object (SDO, SRO, or Marking Definition) can be added to # a bundle with no further checks. if any( @@ -702,7 +777,11 @@ class STIXObjectProperty(Property): "containing objects of a different spec " "version.", ) - return value + + if not allow_custom and value.has_custom: + raise CustomContentError("custom content encountered") + + return value, value.has_custom try: dictified = _get_dict(value) except ValueError: @@ -718,6 +797,22 @@ class STIXObjectProperty(Property): "containing objects of a different spec version.", ) - parsed_obj = parse(dictified, allow_custom=self.allow_custom) + parsed_obj = parse(dictified, allow_custom=allow_custom) - return parsed_obj + if isinstance(parsed_obj, _STIXBase): + has_custom = parsed_obj.has_custom + else: + # we get dicts for unregistered custom objects + has_custom = True + + if not allow_custom and has_custom: + # parse() will ignore the caller's allow_custom=False request if + # the object type is registered and dictified has a + # "custom_properties" key. So we have to do another check here. + raise CustomContentError( + "customized {} object found".format( + parsed_obj["type"], + ), + ) + + return parsed_obj, has_custom diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index dab713e..b8a589c 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -4,7 +4,10 @@ import pytest import pytz import stix2 -from stix2.exceptions import ExtraPropertiesError, STIXError +from stix2.base import _STIXBase +from stix2.exceptions import ( + ExtraPropertiesError, STIXError, CustomContentError, +) from stix2.properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, HexProperty, IntegerProperty, ListProperty, Property, @@ -16,8 +19,8 @@ def test_property(): p = Property() assert p.required is False - assert p.clean('foo') == 'foo' - assert p.clean(3) == 3 + assert p.clean('foo') == ('foo', False) + assert p.clean(3) == (3, False) def test_basic_clean(): @@ -47,7 +50,7 @@ def test_property_default(): assert p.default() == 77 -def test_property_fixed(): +def test_fixed_property(): p = Property(fixed="2.0") assert p.clean("2.0") @@ -65,16 +68,16 @@ def test_property_fixed_and_required(): Property(default=lambda: 3, required=True) -def test_list_property(): +def test_list_property_property_type(): p = ListProperty(StringProperty) - assert p.clean(['abc', 'xyz']) + assert p.clean(['abc', 'xyz'], False) with pytest.raises(ValueError): - p.clean([]) + p.clean([], False) def test_list_property_property_type_custom(): - class TestObj(stix2.base._STIXBase): + class TestObj(_STIXBase): _type = "test" _properties = { "foo": StringProperty(), @@ -86,20 +89,24 @@ def test_list_property_property_type_custom(): TestObj(foo="xyz"), ] - assert p.clean(objs_custom) + assert p.clean(objs_custom, True) + + with pytest.raises(CustomContentError): + p.clean(objs_custom, False) dicts_custom = [ {"foo": "abc", "bar": 123}, {"foo": "xyz"}, ] - # no opportunity to set allow_custom=True when using dicts + assert p.clean(dicts_custom, True) + with pytest.raises(ExtraPropertiesError): - p.clean(dicts_custom) + p.clean(dicts_custom, False) def test_list_property_object_type(): - class TestObj(stix2.base._STIXBase): + class TestObj(_STIXBase): _type = "test" _properties = { "foo": StringProperty(), @@ -107,14 +114,14 @@ def test_list_property_object_type(): p = ListProperty(TestObj) objs = [TestObj(foo="abc"), TestObj(foo="xyz")] - assert p.clean(objs) + assert p.clean(objs, False) dicts = [{"foo": "abc"}, {"foo": "xyz"}] - assert p.clean(dicts) + assert p.clean(dicts, False) def test_list_property_object_type_custom(): - class TestObj(stix2.base._STIXBase): + class TestObj(_STIXBase): _type = "test" _properties = { "foo": StringProperty(), @@ -126,16 +133,20 @@ def test_list_property_object_type_custom(): TestObj(foo="xyz"), ] - assert p.clean(objs_custom) + assert p.clean(objs_custom, True) + + with pytest.raises(CustomContentError): + p.clean(objs_custom, False) dicts_custom = [ {"foo": "abc", "bar": 123}, {"foo": "xyz"}, ] - # no opportunity to set allow_custom=True when using dicts + assert p.clean(dicts_custom, True) + with pytest.raises(ExtraPropertiesError): - p.clean(dicts_custom) + p.clean(dicts_custom, False) def test_list_property_bad_element_type(): @@ -144,7 +155,7 @@ def test_list_property_bad_element_type(): def test_list_property_bad_value_type(): - class TestObj(stix2.base._STIXBase): + class TestObj(_STIXBase): _type = "test" _properties = { "foo": StringProperty(), @@ -152,7 +163,7 @@ def test_list_property_bad_value_type(): list_prop = ListProperty(TestObj) with pytest.raises(ValueError): - list_prop.clean([1]) + list_prop.clean([1], False) def test_string_property(): @@ -296,7 +307,7 @@ def test_boolean_property_invalid(value): ) def test_timestamp_property_valid(value): ts_prop = TimestampProperty() - assert ts_prop.clean(value) == dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc) + assert ts_prop.clean(value) == (dt.datetime(2017, 1, 1, 12, 34, 56, tzinfo=pytz.utc), False) def test_timestamp_property_invalid(): @@ -332,15 +343,21 @@ def test_hex_property(): ) def test_enum_property_valid(value): enum_prop = EnumProperty(value) - assert enum_prop.clean('b') + assert enum_prop.clean('b', False) def test_enum_property_clean(): enum_prop = EnumProperty(['1']) - assert enum_prop.clean(1) == '1' + assert enum_prop.clean(1, False) == ('1', False) def test_enum_property_invalid(): enum_prop = EnumProperty(['a', 'b', 'c']) with pytest.raises(ValueError): - enum_prop.clean('z') + enum_prop.clean('z', False) + + +def test_enum_property_custom(): + enum_prop = EnumProperty(['a', 'b', 'c']) + result = enum_prop.clean("z", True) + assert result == ("z", True) diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py index 433bf81..84f97a5 100644 --- a/stix2/test/test_workbench.py +++ b/stix2/test/test_workbench.py @@ -369,6 +369,7 @@ def test_workbench_custom_property_object_in_observable_extension(): x_foo='bar', ) artifact = File( + allow_custom=True, name='test', extensions={'ntfs-ext': ntfs}, ) @@ -390,7 +391,6 @@ def test_workbench_custom_property_dict_in_observable_extension(): name='test', extensions={ 'ntfs-ext': { - 'allow_custom': True, 'sid': 1, 'x_foo': 'bar', }, diff --git a/stix2/test/v20/test_bundle.py b/stix2/test/v20/test_bundle.py index ac5d239..990a554 100644 --- a/stix2/test/v20/test_bundle.py +++ b/stix2/test/v20/test_bundle.py @@ -224,7 +224,7 @@ def test_stix_object_property(): prop = stix2.properties.STIXObjectProperty(spec_version='2.0') identity = stix2.v20.Identity(name="test", identity_class="individual") - assert prop.clean(identity) is identity + assert prop.clean(identity, False) == (identity, False) def test_bundle_with_different_spec_objects(): diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index a83bf24..8d237e0 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -157,10 +157,11 @@ def test_custom_properties_dict_in_bundled_object(): 'x_foo': 'bar', }, } - bundle = stix2.v20.Bundle(custom_identity) - assert bundle.objects[0].x_foo == "bar" - assert '"x_foo": "bar"' in str(bundle) + # must not succeed: allow_custom was not set to True when creating + # the bundle, so it must reject the customized identity object. + with pytest.raises(InvalidValueError): + stix2.v20.Bundle(custom_identity) def test_custom_property_in_observed_data(): @@ -188,6 +189,7 @@ def test_custom_property_object_in_observable_extension(): x_foo='bar', ) artifact = stix2.v20.File( + allow_custom=True, name='test', extensions={'ntfs-ext': ntfs}, ) @@ -220,7 +222,6 @@ def test_custom_property_dict_in_observable_extension(): name='test', extensions={ 'ntfs-ext': { - 'allow_custom': True, 'sid': 1, 'x_foo': 'bar', }, @@ -384,6 +385,47 @@ def test_custom_object_invalid_type_name(): assert "Invalid type name 'x_new_object':" in str(excinfo.value) +def test_custom_subobject_dict(): + obj_dict = { + "type": "bundle", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "name": "alice", + "identity_class": "individual", + "x_foo": 123, + }, + ], + } + + obj = stix2.parse(obj_dict, allow_custom=True) + assert obj["objects"][0]["x_foo"] == 123 + assert obj.has_custom + + with pytest.raises(InvalidValueError): + stix2.parse(obj_dict, allow_custom=False) + + +def test_custom_subobject_obj(): + ident = stix2.v20.Identity( + name="alice", identity_class=123, x_foo=123, allow_custom=True, + ) + + obj_dict = { + "type": "bundle", + "spec_version": "2.0", + "objects": [ident], + } + + obj = stix2.parse(obj_dict, allow_custom=True) + assert obj["objects"][0]["x_foo"] == 123 + assert obj.has_custom + + with pytest.raises(InvalidValueError): + stix2.parse(obj_dict, allow_custom=False) + + def test_parse_custom_object_type(): nt_string = """{ "type": "x-new-type", @@ -898,6 +940,35 @@ def test_parse_observable_with_custom_extension(): assert parsed.extensions['x-new-ext'].property2 == 12 +def test_parse_observable_with_custom_extension_property(): + input_str = """{ + "type": "observed-data", + "first_observed": "1976-09-09T01:50:24.000Z", + "last_observed": "1988-01-18T15:22:10.000Z", + "number_observed": 5, + "objects": { + "0": { + "type": "file", + "name": "cats.png", + "extensions": { + "raster-image-ext": { + "image_height": 1024, + "image_width": 768, + "x-foo": false + } + } + } + } + }""" + + parsed = stix2.parse(input_str, version='2.0', allow_custom=True) + assert parsed.has_custom + assert parsed["objects"]["0"]["extensions"]["raster-image-ext"]["x-foo"] is False + + with pytest.raises(InvalidValueError): + stix2.parse(input_str, version="2.0", allow_custom=False) + + def test_custom_and_spec_extension_mix(): """ Try to make sure that when allow_custom=True, encountering a custom diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index b03879c..3ff207a 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -3,14 +3,14 @@ import uuid import pytest import stix2 -import stix2.base from stix2.exceptions import ( AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, + ExtraPropertiesError, ParseError, ) from stix2.properties import ( DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, - HashesProperty, IDProperty, ListProperty, ReferenceProperty, - STIXObjectProperty, + HashesProperty, ListProperty, ObservableProperty, ReferenceProperty, + STIXObjectProperty, IDProperty, ) from stix2.v20.common import MarkingProperty @@ -27,7 +27,7 @@ MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' ], ) def test_id_property_valid(value): - assert ID_PROP.clean(value) == value + assert ID_PROP.clean(value) == (value, False) CONSTANT_IDS = [ @@ -54,7 +54,7 @@ CONSTANT_IDS.extend(constants.RELATIONSHIP_IDS) @pytest.mark.parametrize("value", CONSTANT_IDS) def test_id_property_valid_for_type(value): type = value.split('--', 1)[0] - assert IDProperty(type=type, spec_version="2.0").clean(value) == value + assert IDProperty(type=type, spec_version="2.0").clean(value) == (value, False) def test_id_property_wrong_type(): @@ -80,29 +80,63 @@ def test_id_property_not_a_valid_hex_uuid(value): def test_id_property_default(): default = ID_PROP.default() - assert ID_PROP.clean(default) == default + assert ID_PROP.clean(default) == (default, False) def test_reference_property(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") - assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") + assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000", False) with pytest.raises(ValueError): - ref_prop.clean("foo") + ref_prop.clean("foo", False) # This is not a valid V4 UUID with pytest.raises(ValueError): - ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000") + ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000", False) -def test_reference_property_specific_type(): +def test_reference_property_whitelist_type(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") with pytest.raises(ValueError): - ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") == \ - "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" + with pytest.raises(ValueError): + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + +def test_reference_property_blacklist_type(): + ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.0") + result = ref_prop.clean( + "malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) @pytest.mark.parametrize( @@ -183,7 +217,8 @@ def test_property_list_of_dictionary(): ) def test_hashes_property_valid(value): hash_prop = HashesProperty() - assert hash_prop.clean(value) + _, has_custom = hash_prop.clean(value, False) + assert not has_custom @pytest.mark.parametrize( @@ -196,7 +231,21 @@ def test_hashes_property_invalid(value): hash_prop = HashesProperty() with pytest.raises(ValueError): - hash_prop.clean(value) + hash_prop.clean(value, False) + + +def test_hashes_property_custom(): + value = { + "sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", + "abc-123": "aaaaaaaaaaaaaaaaaaaaa", + } + + hash_prop = HashesProperty() + result = hash_prop.clean(value, True) + assert result == (value, True) + + with pytest.raises(CustomContentError): + hash_prop.clean(value, False) def test_embedded_property(): @@ -206,25 +255,103 @@ def test_embedded_property(): content_disposition="inline", body="Cats are funny!", ) - assert emb_prop.clean(mime) + result = emb_prop.clean(mime, False) + assert result == (mime, False) + + result = emb_prop.clean(mime, True) + assert result == (mime, False) with pytest.raises(ValueError): - emb_prop.clean("string") + emb_prop.clean("string", False) + + +def test_embedded_property_dict(): + emb_prop = EmbeddedObjectProperty(type=stix2.v20.EmailMIMEComponent) + mime = { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!", + } + + result = emb_prop.clean(mime, False) + assert isinstance(result[0], stix2.v20.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert not result[1] + + result = emb_prop.clean(mime, True) + assert isinstance(result[0], stix2.v20.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert not result[1] + + +def test_embedded_property_custom(): + emb_prop = EmbeddedObjectProperty(type=stix2.v20.EmailMIMEComponent) + mime = stix2.v20.EmailMIMEComponent( + content_type="text/plain; charset=utf-8", + content_disposition="inline", + body="Cats are funny!", + foo=123, + allow_custom=True, + ) + + with pytest.raises(CustomContentError): + emb_prop.clean(mime, False) + + result = emb_prop.clean(mime, True) + assert result == (mime, True) + + +def test_embedded_property_dict_custom(): + emb_prop = EmbeddedObjectProperty(type=stix2.v20.EmailMIMEComponent) + mime = { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!", + "foo": 123, + } + + with pytest.raises(ExtraPropertiesError): + emb_prop.clean(mime, False) + + result = emb_prop.clean(mime, True) + assert isinstance(result[0], stix2.v20.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert result[1] def test_extension_property_valid(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') - assert ext_prop({ - 'windows-pebinary-ext': { - 'pe_type': 'exe', - }, - }) + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + }, + }, False, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v20.WindowsPEBinaryExt, + ) + assert not result[1] + + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + }, + }, True, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v20.WindowsPEBinaryExt, + ) + assert not result[1] def test_extension_property_invalid1(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') with pytest.raises(ValueError): - ext_prop.clean(1) + ext_prop.clean(1, False) def test_extension_property_invalid2(): @@ -236,8 +363,47 @@ def test_extension_property_invalid2(): 'pe_type': 'exe', }, }, + False, ) + result = ext_prop.clean( + { + 'foobar-ext': { + 'pe_type': 'exe', + }, + }, True, + ) + assert result == ({"foobar-ext": {"pe_type": "exe"}}, True) + + +def test_extension_property_invalid3(): + ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='file') + with pytest.raises(ExtraPropertiesError): + ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + 'abc': 123, + }, + }, + False, + ) + + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + 'abc': 123, + }, + }, True, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v20.WindowsPEBinaryExt, + ) + assert result[0]["windows-pebinary-ext"]["abc"] == 123 + assert result[1] + def test_extension_property_invalid_type(): ext_prop = ExtensionsProperty(spec_version="2.0", enclosing_type='indicator') @@ -248,6 +414,7 @@ def test_extension_property_invalid_type(): 'pe_type': 'exe', }, }, + False, ) assert "Can't parse unknown extension" in str(excinfo.value) @@ -272,6 +439,116 @@ def test_stix_property_not_compliant_spec(): stix_prop = STIXObjectProperty(spec_version="2.0") with pytest.raises(ValueError) as excinfo: - stix_prop.clean(indicator) + stix_prop.clean(indicator, False) assert "Spec version 2.0 bundles don't yet support containing objects of a different spec version." in str(excinfo.value) + + +def test_observable_property_obj(): + prop = ObservableProperty(spec_version="2.0") + + obs = stix2.v20.File(name="data.dat") + obs_dict = { + "0": obs, + } + + result = prop.clean(obs_dict, False) + assert result[0]["0"] == obs + assert not result[1] + + result = prop.clean(obs_dict, True) + assert result[0]["0"] == obs + assert not result[1] + + +def test_observable_property_dict(): + prop = ObservableProperty(spec_version="2.0") + + obs_dict = { + "0": { + "type": "file", + "name": "data.dat", + }, + } + + result = prop.clean(obs_dict, False) + assert isinstance(result[0]["0"], stix2.v20.File) + assert result[0]["0"]["name"] == "data.dat" + assert not result[1] + + result = prop.clean(obs_dict, True) + assert isinstance(result[0]["0"], stix2.v20.File) + assert result[0]["0"]["name"] == "data.dat" + assert not result[1] + + +def test_observable_property_obj_custom(): + prop = ObservableProperty(spec_version="2.0") + + obs = stix2.v20.File(name="data.dat", foo=True, allow_custom=True) + obs_dict = { + "0": obs, + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obs_dict, False) + + result = prop.clean(obs_dict, True) + assert result[0]["0"] == obs + assert result[1] + + +def test_observable_property_dict_custom(): + prop = ObservableProperty(spec_version="2.0") + + obs_dict = { + "0": { + "type": "file", + "name": "data.dat", + "foo": True, + }, + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obs_dict, False) + + result = prop.clean(obs_dict, True) + assert isinstance(result[0]["0"], stix2.v20.File) + assert result[0]["0"]["foo"] + assert result[1] + + +def test_stix_object_property_custom_prop(): + prop = STIXObjectProperty(spec_version="2.0") + + obj_dict = { + "type": "identity", + "name": "alice", + "identity_class": "supergirl", + "foo": "bar", + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obj_dict, False) + + result = prop.clean(obj_dict, True) + assert isinstance(result[0], stix2.v20.Identity) + assert result[0]["foo"] == "bar" + assert result[1] + + +def test_stix_object_property_custom_obj(): + prop = STIXObjectProperty(spec_version="2.0") + + obj_dict = { + "type": "something", + "abc": 123, + "xyz": ["a", 1], + } + + with pytest.raises(ParseError): + prop.clean(obj_dict, False) + + result = prop.clean(obj_dict, True) + assert result[0] == {"type": "something", "abc": 123, "xyz": ["a", 1]} + assert result[1] diff --git a/stix2/test/v21/test_bundle.py b/stix2/test/v21/test_bundle.py index 1cf30d0..a4777d8 100644 --- a/stix2/test/v21/test_bundle.py +++ b/stix2/test/v21/test_bundle.py @@ -235,7 +235,7 @@ def test_stix_object_property(): prop = stix2.properties.STIXObjectProperty(spec_version='2.1') identity = stix2.v21.Identity(name="test", identity_class="individual") - assert prop.clean(identity) is identity + assert prop.clean(identity, False) == (identity, False) def test_bundle_obj_id_found(): diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 36e3548..1f86a27 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -206,8 +206,10 @@ def test_custom_properties_dict_in_bundled_object(): 'x_foo': 'bar', }, } - bundle = stix2.v21.Bundle(custom_identity) + with pytest.raises(InvalidValueError): + stix2.v21.Bundle(custom_identity) + bundle = stix2.v21.Bundle(custom_identity, allow_custom=True) assert bundle.objects[0].x_foo == "bar" assert '"x_foo": "bar"' in str(bundle) @@ -251,6 +253,7 @@ def test_custom_property_object_in_observable_extension(): x_foo='bar', ) artifact = stix2.v21.File( + allow_custom=True, name='test', extensions={'ntfs-ext': ntfs}, ) @@ -283,7 +286,6 @@ def test_custom_property_dict_in_observable_extension(): name='test', extensions={ 'ntfs-ext': { - 'allow_custom': True, 'sid': 1, 'x_foo': 'bar', }, @@ -506,6 +508,48 @@ def test_custom_object_invalid_type_name(): assert "Invalid type name '7x-new-object':" in str(excinfo.value) +def test_custom_subobject_dict(): + obj_dict = { + "type": "bundle", + "id": "bundle--78d99c4a-4eda-4c59-b264-60807f05d799", + "objects": [ + { + "type": "identity", + "spec_version": "2.1", + "name": "alice", + "identity_class": "individual", + "x_foo": 123, + }, + ], + } + + obj = stix2.parse(obj_dict, allow_custom=True) + assert obj["objects"][0]["x_foo"] == 123 + assert obj.has_custom + + with pytest.raises(InvalidValueError): + stix2.parse(obj_dict, allow_custom=False) + + +def test_custom_subobject_obj(): + ident = stix2.v21.Identity( + name="alice", identity_class=123, x_foo=123, allow_custom=True, + ) + + obj_dict = { + "type": "bundle", + "id": "bundle--78d99c4a-4eda-4c59-b264-60807f05d799", + "objects": [ident], + } + + obj = stix2.parse(obj_dict, allow_custom=True) + assert obj["objects"][0]["x_foo"] == 123 + assert obj.has_custom + + with pytest.raises(InvalidValueError): + stix2.parse(obj_dict, allow_custom=False) + + def test_parse_custom_object_type(): nt_string = """{ "type": "x-new-type", @@ -1117,6 +1161,37 @@ def test_parse_observable_with_custom_extension(): assert parsed.extensions['x-new-ext'].property2 == 12 +def test_parse_observable_with_custom_extension_property(): + input_str = """{ + "type": "observed-data", + "spec_version": "2.1", + "first_observed": "1976-09-09T01:50:24.000Z", + "last_observed": "1988-01-18T15:22:10.000Z", + "number_observed": 5, + "objects": { + "0": { + "type": "file", + "spec_version": "2.1", + "name": "cats.png", + "extensions": { + "raster-image-ext": { + "image_height": 1024, + "image_width": 768, + "x-foo": false + } + } + } + } + }""" + + parsed = stix2.parse(input_str, version='2.1', allow_custom=True) + assert parsed.has_custom + assert parsed["objects"]["0"]["extensions"]["raster-image-ext"]["x-foo"] is False + + with pytest.raises(InvalidValueError): + stix2.parse(input_str, version="2.1", allow_custom=False) + + def test_custom_and_spec_extension_mix(): """ Try to make sure that when allow_custom=True, encountering a custom @@ -1337,3 +1412,42 @@ def test_register_duplicate_observable_extension(): class NewExtension2(): pass assert "cannot be registered again" in str(excinfo.value) + + +def test_register_duplicate_marking(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v21.CustomMarking( + 'x-new-obj', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + assert "cannot be registered again" in str(excinfo.value) + + +def test_allow_custom_propagation(): + obj_dict = { + "type": "bundle", + "objects": [ + { + "type": "file", + "spec_version": "2.1", + "name": "data.dat", + "extensions": { + "archive-ext": { + "contains_refs": [ + "file--3d4da5f6-31d8-4a66-a172-f31af9bf5238", + "file--4bb16def-cdfc-40d1-b6a4-815de6c60b74", + ], + "x_foo": "bar", + }, + }, + }, + ], + } + + # allow_custom=False at the top level should catch the custom property way + # down in the SCO extension. + with pytest.raises(InvalidValueError): + stix2.parse(obj_dict, allow_custom=False) diff --git a/stix2/test/v21/test_indicator.py b/stix2/test/v21/test_indicator.py index 2b22418..5f99ac5 100644 --- a/stix2/test/v21/test_indicator.py +++ b/stix2/test/v21/test_indicator.py @@ -223,6 +223,7 @@ def test_indicator_with_custom_embedded_objs(): valid_from=epoch, indicator_types=['malicious-activity'], external_references=[ext_ref], + allow_custom=True, ) assert ind.indicator_types == ['malicious-activity'] diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 36ff858..9914bd1 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -3,11 +3,12 @@ import pytest import stix2 from stix2.exceptions import ( AtLeastOnePropertyError, CustomContentError, DictionaryKeyError, + ExtraPropertiesError, ParseError, ) from stix2.properties import ( DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, - HashesProperty, IDProperty, ListProperty, ReferenceProperty, - StringProperty, TypeProperty, + HashesProperty, IDProperty, ListProperty, ObservableProperty, + ReferenceProperty, STIXObjectProperty, StringProperty, TypeProperty, ) from stix2.v21.common import MarkingProperty @@ -50,7 +51,7 @@ MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' ], ) def test_id_property_valid(value): - assert ID_PROP.clean(value) == value + assert ID_PROP.clean(value) == (value, False) CONSTANT_IDS = [ @@ -77,7 +78,7 @@ CONSTANT_IDS.extend(constants.RELATIONSHIP_IDS) @pytest.mark.parametrize("value", CONSTANT_IDS) def test_id_property_valid_for_type(value): type = value.split('--', 1)[0] - assert IDProperty(type=type, spec_version="2.1").clean(value) == value + assert IDProperty(type=type, spec_version="2.1").clean(value) == (value, False) def test_id_property_wrong_type(): @@ -100,29 +101,63 @@ def test_id_property_not_a_valid_hex_uuid(value): def test_id_property_default(): default = ID_PROP.default() - assert ID_PROP.clean(default) == default + assert ID_PROP.clean(default) == (default, False) def test_reference_property(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") - assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000") + assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000", False) with pytest.raises(ValueError): - ref_prop.clean("foo") + ref_prop.clean("foo", False) # This is not a valid RFC 4122 UUID with pytest.raises(ValueError): - ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000") + ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000", False) -def test_reference_property_specific_type(): +def test_reference_property_whitelist_type(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") with pytest.raises(ValueError): - ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf") == \ - "my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf" + with pytest.raises(ValueError): + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + +def test_reference_property_blacklist_type(): + ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.1") + result = ref_prop.clean( + "location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) @pytest.mark.parametrize( @@ -213,7 +248,11 @@ def test_property_list_of_dictionary(): ) def test_hashes_property_valid(value): hash_prop = HashesProperty() - assert hash_prop.clean(value) + _, has_custom = hash_prop.clean(value, False) + assert not has_custom + + _, has_custom = hash_prop.clean(value, True) + assert not has_custom @pytest.mark.parametrize( @@ -227,7 +266,24 @@ def test_hashes_property_invalid(value): hash_prop = HashesProperty() with pytest.raises(ValueError): - hash_prop.clean(value) + hash_prop.clean(value, False) + + with pytest.raises(ValueError): + hash_prop.clean(value, True) + + +def test_hashes_property_custom(): + value = { + "sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", + "abc-123": "aaaaaaaaaaaaaaaaaaaaa", + } + + hash_prop = HashesProperty() + result = hash_prop.clean(value, True) + assert result == (value, True) + + with pytest.raises(CustomContentError): + hash_prop.clean(value, False) def test_embedded_property(): @@ -237,25 +293,103 @@ def test_embedded_property(): content_disposition="inline", body="Cats are funny!", ) - assert emb_prop.clean(mime) + result = emb_prop.clean(mime, False) + assert result == (mime, False) + + result = emb_prop.clean(mime, True) + assert result == (mime, False) with pytest.raises(ValueError): - emb_prop.clean("string") + emb_prop.clean("string", False) + + +def test_embedded_property_dict(): + emb_prop = EmbeddedObjectProperty(type=stix2.v21.EmailMIMEComponent) + mime = { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!", + } + + result = emb_prop.clean(mime, False) + assert isinstance(result[0], stix2.v21.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert not result[1] + + result = emb_prop.clean(mime, True) + assert isinstance(result[0], stix2.v21.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert not result[1] + + +def test_embedded_property_custom(): + emb_prop = EmbeddedObjectProperty(type=stix2.v21.EmailMIMEComponent) + mime = stix2.v21.EmailMIMEComponent( + content_type="text/plain; charset=utf-8", + content_disposition="inline", + body="Cats are funny!", + foo=123, + allow_custom=True, + ) + + with pytest.raises(CustomContentError): + emb_prop.clean(mime, False) + + result = emb_prop.clean(mime, True) + assert result == (mime, True) + + +def test_embedded_property_dict_custom(): + emb_prop = EmbeddedObjectProperty(type=stix2.v21.EmailMIMEComponent) + mime = { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!", + "foo": 123, + } + + with pytest.raises(ExtraPropertiesError): + emb_prop.clean(mime, False) + + result = emb_prop.clean(mime, True) + assert isinstance(result[0], stix2.v21.EmailMIMEComponent) + assert result[0]["body"] == "Cats are funny!" + assert result[1] def test_extension_property_valid(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') - assert ext_prop({ - 'windows-pebinary-ext': { - 'pe_type': 'exe', - }, - }) + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + }, + }, False, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v21.WindowsPEBinaryExt, + ) + assert not result[1] + + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + }, + }, True, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v21.WindowsPEBinaryExt, + ) + assert not result[1] def test_extension_property_invalid1(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='file') with pytest.raises(ValueError): - ext_prop.clean(1) + ext_prop.clean(1, False) def test_extension_property_invalid2(): @@ -267,8 +401,47 @@ def test_extension_property_invalid2(): 'pe_type': 'exe', }, }, + False, ) + result = ext_prop.clean( + { + 'foobar-ext': { + 'pe_type': 'exe', + }, + }, True, + ) + assert result == ({"foobar-ext": {"pe_type": "exe"}}, True) + + +def test_extension_property_invalid3(): + ext_prop = ExtensionsProperty(spec_version="2.1", enclosing_type='file') + with pytest.raises(ExtraPropertiesError): + ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + 'abc': 123, + }, + }, + False, + ) + + result = ext_prop.clean( + { + 'windows-pebinary-ext': { + 'pe_type': 'exe', + 'abc': 123, + }, + }, True, + ) + + assert isinstance( + result[0]["windows-pebinary-ext"], stix2.v21.WindowsPEBinaryExt, + ) + assert result[0]["windows-pebinary-ext"]["abc"] == 123 + assert result[1] + def test_extension_property_invalid_type(): ext_prop = ExtensionsProperty(spec_version='2.1', enclosing_type='indicator') @@ -279,6 +452,7 @@ def test_extension_property_invalid_type(): 'pe_type': 'exe', }, }, + False, ) assert "Can't parse unknown extension" in str(excinfo.value) @@ -295,3 +469,115 @@ def test_marking_property_error(): mark_prop.clean('my-marking') assert str(excinfo.value) == "must be a Statement, TLP Marking or a registered marking." + + +def test_observable_property_obj(): + prop = ObservableProperty(spec_version="2.1") + + obs = stix2.v21.File(name="data.dat") + obs_dict = { + "0": obs, + } + + result = prop.clean(obs_dict, False) + assert result[0]["0"] == obs + assert not result[1] + + result = prop.clean(obs_dict, True) + assert result[0]["0"] == obs + assert not result[1] + + +def test_observable_property_dict(): + prop = ObservableProperty(spec_version="2.1") + + obs_dict = { + "0": { + "type": "file", + "name": "data.dat", + }, + } + + result = prop.clean(obs_dict, False) + assert isinstance(result[0]["0"], stix2.v21.File) + assert result[0]["0"]["name"] == "data.dat" + assert not result[1] + + result = prop.clean(obs_dict, True) + assert isinstance(result[0]["0"], stix2.v21.File) + assert result[0]["0"]["name"] == "data.dat" + assert not result[1] + + +def test_observable_property_obj_custom(): + prop = ObservableProperty(spec_version="2.1") + + obs = stix2.v21.File(name="data.dat", foo=True, allow_custom=True) + obs_dict = { + "0": obs, + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obs_dict, False) + + result = prop.clean(obs_dict, True) + assert result[0]["0"] == obs + assert result[1] + + +def test_observable_property_dict_custom(): + prop = ObservableProperty(spec_version="2.1") + + obs_dict = { + "0": { + "type": "file", + "name": "data.dat", + "foo": True, + }, + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obs_dict, False) + + result = prop.clean(obs_dict, True) + assert isinstance(result[0]["0"], stix2.v21.File) + assert result[0]["0"]["foo"] + assert result[1] + + +def test_stix_object_property_custom_prop(): + prop = STIXObjectProperty(spec_version="2.1") + + obj_dict = { + "type": "identity", + "spec_version": "2.1", + "name": "alice", + "identity_class": "supergirl", + "foo": "bar", + } + + with pytest.raises(ExtraPropertiesError): + prop.clean(obj_dict, False) + + result = prop.clean(obj_dict, True) + assert isinstance(result[0], stix2.v21.Identity) + assert result[0].has_custom + assert result[0]["foo"] == "bar" + assert result[1] + + +def test_stix_object_property_custom_obj(): + prop = STIXObjectProperty(spec_version="2.1") + + obj_dict = { + "type": "something", + "abc": 123, + "xyz": ["a", 1], + } + + with pytest.raises(ParseError): + prop.clean(obj_dict, False) + + result = prop.clean(obj_dict, True) + assert result[0] == {"type": "something", "abc": 123, "xyz": ["a", 1]} + assert result[1] diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index 6a663d6..c5ca25d 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -35,9 +35,6 @@ class Bundle(_STIXBase20): kwargs['objects'] = obj_list + kwargs.get('objects', []) - self._allow_custom = kwargs.get('allow_custom', False) - self._properties['objects'].contained.allow_custom = kwargs.get('allow_custom', False) - super(Bundle, self).__init__(**kwargs) def get_obj(self, obj_uuid): diff --git a/stix2/v20/common.py b/stix2/v20/common.py index 6695c9a..fe151fb 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -104,9 +104,9 @@ class MarkingProperty(Property): marking-definition objects. """ - def clean(self, value): + def clean(self, value, allow_custom=False): if type(value) in OBJ_MAP_MARKING.values(): - return value + return value, False else: raise ValueError("must be a Statement, TLP Marking or a registered marking.") diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index e1f6410..538558a 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -219,12 +219,6 @@ class ObservedData(_DomainObject): ('granular_markings', ListProperty(GranularMarking)), ]) - def __init__(self, *args, **kwargs): - self._allow_custom = kwargs.get('allow_custom', False) - self._properties['objects'].allow_custom = kwargs.get('allow_custom', False) - - super(ObservedData, self).__init__(*args, **kwargs) - class Report(_DomainObject): """For more detailed information on this object's properties, see diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index 5497da5..7ad056d 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -32,9 +32,6 @@ class Bundle(_STIXBase21): kwargs['objects'] = obj_list + kwargs.get('objects', []) - self._allow_custom = kwargs.get('allow_custom', False) - self._properties['objects'].contained.allow_custom = kwargs.get('allow_custom', False) - super(Bundle, self).__init__(**kwargs) def get_obj(self, obj_uuid): diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 2980276..af0a758 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -137,9 +137,9 @@ class MarkingProperty(Property): marking-definition objects. """ - def clean(self, value): + def clean(self, value, allow_custom=False): if type(value) in OBJ_MAP_MARKING.values(): - return value + return value, False else: raise ValueError("must be a Statement, TLP Marking or a registered marking.") diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 35a878e..d0b6077 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -567,8 +567,6 @@ class ObservedData(_DomainObject): ]) def __init__(self, *args, **kwargs): - self._allow_custom = kwargs.get('allow_custom', False) - self._properties['objects'].allow_custom = kwargs.get('allow_custom', False) if "objects" in kwargs: warnings.warn( diff --git a/stix2/versioning.py b/stix2/versioning.py index a6dc0bd..02a1e13 100644 --- a/stix2/versioning.py +++ b/stix2/versioning.py @@ -192,8 +192,9 @@ def new_version(data, allow_custom=None, **kwargs): or dict. :param allow_custom: Whether to allow custom properties on the new object. If True, allow them (regardless of whether the original had custom - properties); if False disallow them; if None, propagate the preference - from the original object. + properties); if False disallow them; if None, auto-detect from the + object: if it has custom properties, allow them in the new version, + otherwise don't allow them. :param kwargs: The properties to change. Setting to None requests property removal. :return: The new object. @@ -271,7 +272,7 @@ def new_version(data, allow_custom=None, **kwargs): # it for dicts. if isinstance(data, stix2.base._STIXBase): if allow_custom is None: - new_obj_inner["allow_custom"] = data._allow_custom + new_obj_inner["allow_custom"] = data.has_custom else: new_obj_inner["allow_custom"] = allow_custom From 387ce7e7cbfff8ea4190a71e6fbf90efff5fb32d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 19 Jun 2020 11:40:36 -0400 Subject: [PATCH 02/21] Fix Report SDO: I'd fixed ReferenceProperty to work the way I thought it should, but forgot to fix Report to use ReferenceProperty in the way I thought it should! Oops. Added some tests to ensure Report is working property with custom ID types in object_refs. --- stix2/test/v20/test_properties.py | 5 +++++ stix2/test/v20/test_report.py | 28 ++++++++++++++++++++++++++++ stix2/test/v21/test_properties.py | 5 +++++ stix2/test/v21/test_report.py | 26 +++++++++++++++++++++++++- stix2/v20/sdo.py | 2 +- stix2/v21/sdo.py | 2 +- 6 files changed, 65 insertions(+), 3 deletions(-) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 3ff207a..9e4b6e1 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -138,6 +138,11 @@ def test_reference_property_blacklist_type(): "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + @pytest.mark.parametrize( "d", [ diff --git a/stix2/test/v20/test_report.py b/stix2/test/v20/test_report.py index 53707ce..2de9363 100644 --- a/stix2/test/v20/test_report.py +++ b/stix2/test/v20/test_report.py @@ -4,6 +4,7 @@ import pytest import pytz import stix2 +from stix2.exceptions import InvalidValueError from .constants import ( CAMPAIGN_ID, IDENTITY_ID, INDICATOR_ID, INDICATOR_KWARGS, RELATIONSHIP_ID, @@ -133,3 +134,30 @@ def test_parse_report(data): assert rept.name == "The Black Vine Cyberespionage Group" # TODO: Add other examples + + +def test_report_on_custom(): + with pytest.raises(InvalidValueError): + stix2.v20.Report( + name="my report", + labels=["a label"], + published="2016-01-20T17:00:00Z", + object_refs=[ + "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + ] + ) + + report = stix2.v20.Report( + name="my report", + labels=["a label"], + published="2016-01-20T17:00:00Z", + object_refs=[ + "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + ], + allow_custom=True, + ) + + assert "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" \ + in report.object_refs diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 9914bd1..d2e2694 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -159,6 +159,11 @@ def test_reference_property_blacklist_type(): "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + @pytest.mark.parametrize( "d", [ diff --git a/stix2/test/v21/test_report.py b/stix2/test/v21/test_report.py index e54e11d..69af096 100644 --- a/stix2/test/v21/test_report.py +++ b/stix2/test/v21/test_report.py @@ -9,6 +9,7 @@ from .constants import ( CAMPAIGN_ID, IDENTITY_ID, INDICATOR_ID, INDICATOR_KWARGS, RELATIONSHIP_ID, REPORT_ID, ) +from stix2.exceptions import InvalidValueError EXPECTED = """{ "type": "report", @@ -135,4 +136,27 @@ def test_parse_report(data): assert rept.report_types == ["campaign"] assert rept.name == "The Black Vine Cyberespionage Group" -# TODO: Add other examples + +def test_report_on_custom(): + with pytest.raises(InvalidValueError): + stix2.v21.Report( + name="my report", + published="2016-01-20T17:00:00Z", + object_refs=[ + "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + ] + ) + + report = stix2.v21.Report( + name="my report", + published="2016-01-20T17:00:00Z", + object_refs=[ + "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + ], + allow_custom=True, + ) + + assert "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" \ + in report.object_refs diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 538558a..3c854e0 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -235,7 +235,7 @@ class Report(_DomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty, required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index d0b6077..3b81e43 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -646,7 +646,7 @@ class Report(_DomainObject): ('description', StringProperty()), ('report_types', ListProperty(StringProperty)), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), From c7dd58ed8916c477c51217b384c5be1199c77394 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 19 Jun 2020 18:48:38 -0400 Subject: [PATCH 03/21] Further ReferenceProperty refinements: make allow_custom=True work when a whitelist of generic category types is used. Disallow hybrid constraints (both generic and specific at the same time). Add more unit tests. --- stix2/properties.py | 127 ++++++++++++++++-------- stix2/test/v20/test_properties.py | 85 ++++++++++++++++ stix2/test/v21/test_malware_analysis.py | 35 +++++++ stix2/test/v21/test_properties.py | 85 ++++++++++++++++ 4 files changed, 293 insertions(+), 39 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 85d4e08..31c1d51 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -490,27 +490,44 @@ class HexProperty(Property): class ReferenceProperty(Property): + _OBJECT_CATEGORIES = {"SDO", "SCO", "SRO"} + _WHITELIST, _BLACKLIST = range(2) + def __init__(self, valid_types=None, invalid_types=None, spec_version=DEFAULT_VERSION, **kwargs): """ references sometimes must be to a specific object type """ self.spec_version = spec_version - # These checks need to be done prior to the STIX object finishing construction - # and thus we can't use base.py's _check_mutually_exclusive_properties() - # in the typical location of _check_object_constraints() in sdo.py - if valid_types and invalid_types: - raise MutuallyExclusivePropertiesError(self.__class__, ['invalid_types', 'valid_types']) - elif valid_types is None and invalid_types is None: - raise MissingPropertiesError(self.__class__, ['invalid_types', 'valid_types']) + if (valid_types is not None and invalid_types is not None) or \ + (valid_types is None and invalid_types is None): + raise ValueError( + "Exactly one of 'valid_types' and 'invalid_types' must be " + "given" + ) - if valid_types and type(valid_types) is not list: + if valid_types and not isinstance(valid_types, list): valid_types = [valid_types] - elif invalid_types and type(invalid_types) is not list: + elif invalid_types and not isinstance(invalid_types, list): invalid_types = [invalid_types] - self.valid_types = valid_types - self.invalid_types = invalid_types + self.types = set(valid_types or invalid_types) + self.auth_type = self._WHITELIST if valid_types else self._BLACKLIST + + # Handling both generic and non-generic types in the same constraint + # complicates life... let's keep it simple unless we really need the + # complexity. + self.generic_constraint = any( + t in self._OBJECT_CATEGORIES for t in self.types + ) + + if self.generic_constraint and any( + t not in self._OBJECT_CATEGORIES for t in self.types + ): + raise ValueError( + "Generic type categories and specific types may not both be " + "given" + ) super(ReferenceProperty, self).__init__(**kwargs) @@ -523,26 +540,56 @@ class ReferenceProperty(Property): obj_type = value[:value.index('--')] - if self.valid_types: + types = self.types + auth_type = self.auth_type + if allow_custom and auth_type == self._WHITELIST and \ + self.generic_constraint: + # If allowing customization and using a whitelist, and if generic + # "category" types were given, we need to allow custom object types + # of those categories. Unless registered, it's impossible to know + # whether a given type is within a given category. So we take a + # permissive approach and allow any type which is not known to be + # in the wrong category. I.e. flip the whitelist set to a + # blacklist of a complementary set. + types = self._OBJECT_CATEGORIES - types + auth_type = self._BLACKLIST + + if auth_type == self._WHITELIST: # allow_custom is not applicable to "whitelist" style object type # constraints, so we ignore it. has_custom = False - ref_valid_types = enumerate_types(self.valid_types, self.spec_version) + if self.generic_constraint: + type_ok = _type_in_generic_set( + obj_type, types, self.spec_version + ) + else: + type_ok = obj_type in types - if obj_type not in ref_valid_types: - raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (obj_type)) + if not type_ok: + raise ValueError( + "The type-specifying prefix '%s' for this property is not " + "valid" % obj_type + ) else: - # A type "blacklist" was used to describe legal object types. - # We must enforce the type blacklist regardless of allow_custom. - ref_invalid_types = enumerate_types(self.invalid_types, self.spec_version) + # A type "blacklist" was used to describe legal object types. We + # must enforce the type blacklist regardless of allow_custom. + if self.generic_constraint: + type_ok = not _type_in_generic_set( + obj_type, types, self.spec_version + ) + else: + type_ok = obj_type not in types - if obj_type in ref_invalid_types: - raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (obj_type)) + if not type_ok: + raise ValueError( + "The type-specifying prefix '%s' for this property is not " + "valid" % obj_type + ) # allow_custom=True only allows references to custom objects which - # are not otherwise blacklisted. So we need to figure out whether + # are not otherwise blacklisted. We need to figure out whether # the referenced object is custom or not. No good way to do that # at present... just check if unregistered and for the "x-" type # prefix, for now? @@ -562,28 +609,30 @@ class ReferenceProperty(Property): return value, has_custom -def enumerate_types(types, spec_version): +def _type_in_generic_set(type_, type_set, spec_version): """ - `types` is meant to be a list; it may contain specific object types and/or - the any of the words "SCO", "SDO", or "SRO" - - Since "SCO", "SDO", and "SRO" are general types that encompass various specific object types, - once each of those words is being processed, that word will be removed from `return_types`, - so as not to mistakenly allow objects to be created of types "SCO", "SDO", or "SRO" + Determine if type_ is in the given set, with respect to the given STIX + version. This handles special generic category values "SDO", "SCO", + "SRO", so it's not a simple set containment check. The type_set is + implicitly "OR"d. """ - return_types = types[:] + type_maps = STIX2_OBJ_MAPS[spec_version] - if "SDO" in types: - return_types.remove("SDO") - return_types += STIX2_OBJ_MAPS[spec_version]['objects'].keys() - if "SCO" in types: - return_types.remove("SCO") - return_types += STIX2_OBJ_MAPS[spec_version]['observables'].keys() - if "SRO" in types: - return_types.remove("SRO") - return_types += ['relationship', 'sighting'] + result = False + for type_id in type_set: + if type_id == "SDO": + result = type_ in type_maps["objects"] + elif type_id == "SCO": + result = type_ in type_maps["observables"] + elif type_id == "SRO": + result = type_ in ["relationship", "sighting"] + else: + raise ValueError("Unrecognized generic type category: " + type_id) - return return_types + if result: + break + + return result SELECTOR_REGEX = re.compile(r"^([a-z0-9_-]{3,250}(\.(\[\d+\]|[a-z0-9_-]{1,250}))*|id)$") diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 9e4b6e1..e899f81 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -111,6 +111,37 @@ def test_reference_property_whitelist_type(): assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) +def test_reference_property_whitelist_generic_type(): + ref_prop = ReferenceProperty( + valid_types=["SCO", "SRO"], spec_version="2.0" + ) + + result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False + ) + assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + def test_reference_property_blacklist_type(): ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.0") result = ref_prop.clean( @@ -144,6 +175,60 @@ def test_reference_property_blacklist_type(): ) +def test_reference_property_blacklist_generic_type(): + ref_prop = ReferenceProperty( + invalid_types=["SDO", "SRO"], spec_version="2.0" + ) + + result = ref_prop.clean( + "file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + +def test_reference_property_hybrid_constraint_type(): + with pytest.raises(ValueError): + ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.0") + + with pytest.raises(ValueError): + ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.0") + + @pytest.mark.parametrize( "d", [ {'description': 'something'}, diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index bf4bbe6..e5ad628 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -84,3 +84,38 @@ def test_malware_analysis_constraint(): stix2.v21.MalwareAnalysis( product="Acme Malware Analyzer", ) + + +def test_malware_analysis_custom_sco_refs(): + ma = stix2.v21.MalwareAnalysis( + product="super scanner", + analysis_sco_refs=[ + "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", + "some-object--f6bfc147-e844-4578-ae01-847979890239" + ], + allow_custom=True, + ) + + assert "some-object--f6bfc147-e844-4578-ae01-847979890239" in \ + ma["analysis_sco_refs"] + assert ma.has_custom + + with pytest.raises(stix2.exceptions.InvalidValueError): + stix2.v21.MalwareAnalysis( + product="super scanner", + analysis_sco_refs=[ + "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", + "some-object--f6bfc147-e844-4578-ae01-847979890239" + ] + ) + + with pytest.raises(stix2.exceptions.InvalidValueError): + stix2.v21.MalwareAnalysis( + product="super scanner", + analysis_sco_refs=[ + "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", + # standard object type; wrong category (not SCO) + "identity--56977a19-49ef-49d7-b259-f733fa4b7bbc" + ], + allow_custom=True, + ) diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index d2e2694..1015789 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -132,6 +132,37 @@ def test_reference_property_whitelist_type(): assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) +def test_reference_property_whitelist_generic_type(): + ref_prop = ReferenceProperty( + valid_types=["SCO", "SRO"], spec_version="2.1" + ) + + result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False + ) + assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + def test_reference_property_blacklist_type(): ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.1") result = ref_prop.clean( @@ -165,6 +196,60 @@ def test_reference_property_blacklist_type(): ) +def test_reference_property_blacklist_generic_type(): + ref_prop = ReferenceProperty( + invalid_types=["SDO", "SRO"], spec_version="2.1" + ) + + result = ref_prop.clean( + "file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + + with pytest.raises(ValueError): + ref_prop.clean( + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + + +def test_reference_property_hybrid_constraint_type(): + with pytest.raises(ValueError): + ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.1") + + with pytest.raises(ValueError): + ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.1") + + @pytest.mark.parametrize( "d", [ {'description': 'something'}, From 03c265c3a36b46d22f7728365df7d7bc3d2a646b Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 22 Jun 2020 16:58:28 -0400 Subject: [PATCH 04/21] Add a check in ReferenceProperty constructor for an impossible to satisfy type constraint: empty whitelist. It would be silly for anyone to do that, but I should check just in case I guess. --- stix2/properties.py | 3 +++ stix2/test/v20/test_properties.py | 5 +++++ stix2/test/v21/test_properties.py | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/stix2/properties.py b/stix2/properties.py index 31c1d51..80c9e0a 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -511,6 +511,9 @@ class ReferenceProperty(Property): elif invalid_types and not isinstance(invalid_types, list): invalid_types = [invalid_types] + if valid_types is not None and len(valid_types) == 0: + raise ValueError("Impossible type constraint: empty whitelist") + self.types = set(valid_types or invalid_types) self.auth_type = self._WHITELIST if valid_types else self._BLACKLIST diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index e899f81..5c57ed0 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -229,6 +229,11 @@ def test_reference_property_hybrid_constraint_type(): ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.0") +def test_reference_property_impossible_constraint(): + with pytest.raises(ValueError): + ReferenceProperty(valid_types=[], spec_version="2.0") + + @pytest.mark.parametrize( "d", [ {'description': 'something'}, diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 1015789..f963944 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -250,6 +250,11 @@ def test_reference_property_hybrid_constraint_type(): ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.1") +def test_reference_property_impossible_constraint(): + with pytest.raises(ValueError): + ReferenceProperty(valid_types=[], spec_version="2.1") + + @pytest.mark.parametrize( "d", [ {'description': 'something'}, From 986404b7b7c68fb761d037e5f35fc297f4b63f1f Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 24 Jun 2020 19:39:47 -0400 Subject: [PATCH 05/21] In the test_properties.py test suites, I thought code like assert prop.clean(...) doesn't test well enough since clean() methods on this branch produce 2-tuples, and you should test what's in the tuple, not just that it returned something non-empty. So I fixed it in several places to test the tuple contents. --- stix2/test/test_properties.py | 22 +++++++++++++++------- stix2/test/v20/test_properties.py | 1 + stix2/test/v21/test_properties.py | 17 ----------------- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index b8a589c..60a42e6 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -71,7 +71,9 @@ def test_property_fixed_and_required(): def test_list_property_property_type(): p = ListProperty(StringProperty) - assert p.clean(['abc', 'xyz'], False) + result = p.clean(['abc', 'xyz'], False) + assert result == (['abc', 'xyz'], False) + with pytest.raises(ValueError): p.clean([], False) @@ -89,7 +91,8 @@ def test_list_property_property_type_custom(): TestObj(foo="xyz"), ] - assert p.clean(objs_custom, True) + result = p.clean(objs_custom, True) + assert result == (objs_custom, True) with pytest.raises(CustomContentError): p.clean(objs_custom, False) @@ -99,7 +102,8 @@ def test_list_property_property_type_custom(): {"foo": "xyz"}, ] - assert p.clean(dicts_custom, True) + result = p.clean(dicts_custom, True) + assert result == (objs_custom, True) with pytest.raises(ExtraPropertiesError): p.clean(dicts_custom, False) @@ -114,10 +118,12 @@ def test_list_property_object_type(): p = ListProperty(TestObj) objs = [TestObj(foo="abc"), TestObj(foo="xyz")] - assert p.clean(objs, False) + result = p.clean(objs, False) + assert result == (objs, False) dicts = [{"foo": "abc"}, {"foo": "xyz"}] - assert p.clean(dicts, False) + result = p.clean(dicts, False) + assert result == (objs, False) def test_list_property_object_type_custom(): @@ -133,7 +139,8 @@ def test_list_property_object_type_custom(): TestObj(foo="xyz"), ] - assert p.clean(objs_custom, True) + result = p.clean(objs_custom, True) + assert result == (objs_custom, True) with pytest.raises(CustomContentError): p.clean(objs_custom, False) @@ -143,7 +150,8 @@ def test_list_property_object_type_custom(): {"foo": "xyz"}, ] - assert p.clean(dicts_custom, True) + result = p.clean(dicts_custom, True) + assert result == (objs_custom, True) with pytest.raises(ExtraPropertiesError): p.clean(dicts_custom, False) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 5c57ed0..af1ffec 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -16,6 +16,7 @@ from stix2.v20.common import MarkingProperty from . import constants + ID_PROP = IDProperty('my-type', spec_version="2.0") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index f963944..aa3624b 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -23,23 +23,6 @@ def test_dictionary_property(): p.clean({}) -def test_string_property(): - prop = StringProperty() - - assert prop.clean('foobar') - assert prop.clean(1) - assert prop.clean([1, 2, 3]) - - -def test_type_property(): - prop = TypeProperty('my-type') - - assert prop.clean('my-type') - with pytest.raises(ValueError): - prop.clean('not-my-type') - assert prop.clean(prop.default()) - - ID_PROP = IDProperty('my-type', spec_version="2.1") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' From 414694d8fa5684128cf1b2742b292ade8e18b662 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Sat, 27 Jun 2020 20:03:33 -0400 Subject: [PATCH 06/21] Add support in ReferenceProperty for hybrid object type constraints (i.e. both generic type categories and specific types). Also: - more expansion/refinement of reference property unit tests - bugfix: SROs are in OBJ_MAP too, it's not just SDOs! Oops... - pre-commit stylistic fixes --- stix2/properties.py | 111 +++++++++------------- stix2/test/v20/test_properties.py | 116 ++++++++++++++++------- stix2/test/v20/test_report.py | 8 +- stix2/test/v21/test_malware_analysis.py | 8 +- stix2/test/v21/test_properties.py | 118 +++++++++++++++++------- stix2/test/v21/test_report.py | 10 +- 6 files changed, 222 insertions(+), 149 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 80c9e0a..735bc79 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -8,10 +8,7 @@ import re import uuid from .base import _STIXBase -from .exceptions import ( - CustomContentError, DictionaryKeyError, MissingPropertiesError, - MutuallyExclusivePropertiesError, STIXError, -) +from .exceptions import CustomContentError, DictionaryKeyError, STIXError from .parsing import parse, parse_observable from .registry import STIX2_OBJ_MAPS from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime @@ -503,7 +500,7 @@ class ReferenceProperty(Property): (valid_types is None and invalid_types is None): raise ValueError( "Exactly one of 'valid_types' and 'invalid_types' must be " - "given" + "given", ) if valid_types and not isinstance(valid_types, list): @@ -517,21 +514,6 @@ class ReferenceProperty(Property): self.types = set(valid_types or invalid_types) self.auth_type = self._WHITELIST if valid_types else self._BLACKLIST - # Handling both generic and non-generic types in the same constraint - # complicates life... let's keep it simple unless we really need the - # complexity. - self.generic_constraint = any( - t in self._OBJECT_CATEGORIES for t in self.types - ) - - if self.generic_constraint and any( - t not in self._OBJECT_CATEGORIES for t in self.types - ): - raise ValueError( - "Generic type categories and specific types may not both be " - "given" - ) - super(ReferenceProperty, self).__init__(**kwargs) def clean(self, value, allow_custom): @@ -543,10 +525,19 @@ class ReferenceProperty(Property): obj_type = value[:value.index('--')] - types = self.types + # Only comes into play when inverting a hybrid whitelist. + # E.g. if the possible generic categories are A, B, C, then the + # inversion of whitelist constraint "A or x" (where x is a specific + # type) is something like "[not (B or C)] or x". In other words, we + # invert the generic categories to produce a blacklist, but leave the + # specific categories alone; they essentially become exceptions to our + # blacklist. + blacklist_exceptions = set() + + generics = self.types & self._OBJECT_CATEGORIES + specifics = self.types - generics auth_type = self.auth_type - if allow_custom and auth_type == self._WHITELIST and \ - self.generic_constraint: + if allow_custom and auth_type == self._WHITELIST and generics: # If allowing customization and using a whitelist, and if generic # "category" types were given, we need to allow custom object types # of those categories. Unless registered, it's impossible to know @@ -554,60 +545,40 @@ class ReferenceProperty(Property): # permissive approach and allow any type which is not known to be # in the wrong category. I.e. flip the whitelist set to a # blacklist of a complementary set. - types = self._OBJECT_CATEGORIES - types auth_type = self._BLACKLIST + generics = self._OBJECT_CATEGORIES - generics + blacklist_exceptions, specifics = specifics, blacklist_exceptions if auth_type == self._WHITELIST: - # allow_custom is not applicable to "whitelist" style object type - # constraints, so we ignore it. - has_custom = False - - if self.generic_constraint: - type_ok = _type_in_generic_set( - obj_type, types, self.spec_version - ) - else: - type_ok = obj_type in types - - if not type_ok: - raise ValueError( - "The type-specifying prefix '%s' for this property is not " - "valid" % obj_type - ) + type_ok = _type_in_generic_set( + obj_type, generics, self.spec_version + ) or obj_type in specifics else: - # A type "blacklist" was used to describe legal object types. We - # must enforce the type blacklist regardless of allow_custom. - if self.generic_constraint: - type_ok = not _type_in_generic_set( - obj_type, types, self.spec_version + type_ok = ( + not _type_in_generic_set( + obj_type, generics, self.spec_version, ) - else: - type_ok = obj_type not in types + and obj_type not in specifics + ) or obj_type in blacklist_exceptions - if not type_ok: - raise ValueError( - "The type-specifying prefix '%s' for this property is not " - "valid" % obj_type - ) + if not type_ok: + raise ValueError( + "The type-specifying prefix '%s' for this property is not " + "valid" % obj_type, + ) - # allow_custom=True only allows references to custom objects which - # are not otherwise blacklisted. We need to figure out whether - # the referenced object is custom or not. No good way to do that - # at present... just check if unregistered and for the "x-" type - # prefix, for now? - type_maps = STIX2_OBJ_MAPS[self.spec_version] + # We need to figure out whether the referenced object is custom or + # not. No good way to do that at present... just check if + # unregistered and for the "x-" type prefix, for now? + has_custom = not _type_in_generic_set( + obj_type, self._OBJECT_CATEGORIES, self.spec_version, + ) or obj_type.startswith("x-") - has_custom = obj_type not in type_maps["objects"] \ - and obj_type not in type_maps["observables"] \ - and obj_type not in ["relationship", "sighting"] - - has_custom = has_custom or obj_type.startswith("x-") - - if not allow_custom and has_custom: - raise CustomContentError( - "reference to custom object type: " + obj_type, - ) + if not allow_custom and has_custom: + raise CustomContentError( + "reference to custom object type: " + obj_type, + ) return value, has_custom @@ -624,7 +595,9 @@ def _type_in_generic_set(type_, type_set, spec_version): result = False for type_id in type_set: if type_id == "SDO": - result = type_ in type_maps["objects"] + result = type_ in type_maps["objects"] and type_ not in [ + "relationship", "sighting", + ] # sigh elif type_id == "SCO": result = type_ in type_maps["observables"] elif type_id == "SRO": diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index af1ffec..4dbc437 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -84,19 +84,21 @@ def test_id_property_default(): assert ID_PROP.clean(default) == (default, False) -def test_reference_property(): - ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") +def test_reference_property_whitelist_standard_type(): + ref_prop = ReferenceProperty(valid_types="identity", spec_version="2.0") + result = ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000", False) with pytest.raises(ValueError): - ref_prop.clean("foo", False) + ref_prop.clean("foo--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - # This is not a valid V4 UUID with pytest.raises(ValueError): - ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000", False) + ref_prop.clean("foo--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) -def test_reference_property_whitelist_type(): +def test_reference_property_whitelist_custom_type(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.0") with pytest.raises(ValueError): @@ -105,16 +107,18 @@ def test_reference_property_whitelist_type(): with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + # This is the whitelisted type, but it's still custom, and + # customization is disallowed here. + ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) def test_reference_property_whitelist_generic_type(): ref_prop = ReferenceProperty( - valid_types=["SCO", "SRO"], spec_version="2.0" + valid_types=["SCO", "SRO"], spec_version="2.0", ) result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) @@ -124,12 +128,19 @@ def test_reference_property_whitelist_generic_type(): assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) result = ref_prop.clean( - "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, ) assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) result = ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # The prop assumes some-type is a custom type of one of the generic + # type categories. + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) @@ -143,7 +154,7 @@ def test_reference_property_whitelist_generic_type(): ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) -def test_reference_property_blacklist_type(): +def test_reference_property_blacklist_standard_type(): ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.0") result = ref_prop.clean( "malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, @@ -155,6 +166,11 @@ def test_reference_property_blacklist_type(): ) assert result == ("malware--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + result = ref_prop.clean( "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) @@ -170,15 +186,10 @@ def test_reference_property_blacklist_type(): "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) - with pytest.raises(CustomContentError): - ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, - ) - def test_reference_property_blacklist_generic_type(): ref_prop = ReferenceProperty( - invalid_types=["SDO", "SRO"], spec_version="2.0" + invalid_types=["SDO", "SRO"], spec_version="2.0", ) result = ref_prop.clean( @@ -191,16 +202,16 @@ def test_reference_property_blacklist_generic_type(): ) assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + result = ref_prop.clean( "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - with pytest.raises(ValueError): - ref_prop.clean( - "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, - ) - with pytest.raises(ValueError): ref_prop.clean( "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, @@ -208,7 +219,7 @@ def test_reference_property_blacklist_generic_type(): with pytest.raises(ValueError): ref_prop.clean( - "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) with pytest.raises(ValueError): @@ -216,18 +227,57 @@ def test_reference_property_blacklist_generic_type(): "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, ) - with pytest.raises(CustomContentError): + with pytest.raises(ValueError): ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) -def test_reference_property_hybrid_constraint_type(): - with pytest.raises(ValueError): - ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.0") +def test_reference_property_whitelist_hybrid_type(): + p = ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.0") + + result = p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(CustomContentError): + # although whitelisted, "a" is a custom type + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) with pytest.raises(ValueError): - ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.0") + p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # should just assume "b" is a custom SCO type. + result = p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + +def test_reference_property_blacklist_hybrid_type(): + p = ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.0") + + with pytest.raises(ValueError): + p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(CustomContentError): + p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # should just assume "b" is a custom type which is not an SCO + result = p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) def test_reference_property_impossible_constraint(): diff --git a/stix2/test/v20/test_report.py b/stix2/test/v20/test_report.py index 2de9363..29d181c 100644 --- a/stix2/test/v20/test_report.py +++ b/stix2/test/v20/test_report.py @@ -144,8 +144,8 @@ def test_report_on_custom(): published="2016-01-20T17:00:00Z", object_refs=[ "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", - "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" - ] + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7", + ], ) report = stix2.v20.Report( @@ -154,10 +154,10 @@ def test_report_on_custom(): published="2016-01-20T17:00:00Z", object_refs=[ "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", - "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7", ], allow_custom=True, ) assert "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" \ - in report.object_refs + in report.object_refs diff --git a/stix2/test/v21/test_malware_analysis.py b/stix2/test/v21/test_malware_analysis.py index e5ad628..ed8cce5 100644 --- a/stix2/test/v21/test_malware_analysis.py +++ b/stix2/test/v21/test_malware_analysis.py @@ -91,7 +91,7 @@ def test_malware_analysis_custom_sco_refs(): product="super scanner", analysis_sco_refs=[ "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", - "some-object--f6bfc147-e844-4578-ae01-847979890239" + "some-object--f6bfc147-e844-4578-ae01-847979890239", ], allow_custom=True, ) @@ -105,8 +105,8 @@ def test_malware_analysis_custom_sco_refs(): product="super scanner", analysis_sco_refs=[ "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", - "some-object--f6bfc147-e844-4578-ae01-847979890239" - ] + "some-object--f6bfc147-e844-4578-ae01-847979890239", + ], ) with pytest.raises(stix2.exceptions.InvalidValueError): @@ -115,7 +115,7 @@ def test_malware_analysis_custom_sco_refs(): analysis_sco_refs=[ "file--6e8c78cf-4bcc-4729-9265-86a97bfc91ba", # standard object type; wrong category (not SCO) - "identity--56977a19-49ef-49d7-b259-f733fa4b7bbc" + "identity--56977a19-49ef-49d7-b259-f733fa4b7bbc", ], allow_custom=True, ) diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index aa3624b..413aa0d 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -8,7 +8,7 @@ from stix2.exceptions import ( from stix2.properties import ( DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, HashesProperty, IDProperty, ListProperty, ObservableProperty, - ReferenceProperty, STIXObjectProperty, StringProperty, TypeProperty, + ReferenceProperty, STIXObjectProperty, StringProperty, ) from stix2.v21.common import MarkingProperty @@ -87,19 +87,21 @@ def test_id_property_default(): assert ID_PROP.clean(default) == (default, False) -def test_reference_property(): - ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") +def test_reference_property_whitelist_standard_type(): + ref_prop = ReferenceProperty(valid_types="identity", spec_version="2.1") + result = ref_prop.clean( + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + assert result == ("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert ref_prop.clean("my-type--00000000-0000-4000-8000-000000000000", False) with pytest.raises(ValueError): - ref_prop.clean("foo", False) + ref_prop.clean("foo--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - # This is not a valid RFC 4122 UUID with pytest.raises(ValueError): - ref_prop.clean("my-type--00000000-0000-0000-0000-000000000000", False) + ref_prop.clean("foo--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) -def test_reference_property_whitelist_type(): +def test_reference_property_whitelist_custom_type(): ref_prop = ReferenceProperty(valid_types="my-type", spec_version="2.1") with pytest.raises(ValueError): @@ -108,16 +110,18 @@ def test_reference_property_whitelist_type(): with pytest.raises(ValueError): ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + # This is the whitelisted type, but it's still custom, and + # customization is disallowed here. + ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) - assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + result = ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) def test_reference_property_whitelist_generic_type(): ref_prop = ReferenceProperty( - valid_types=["SCO", "SRO"], spec_version="2.1" + valid_types=["SCO", "SRO"], spec_version="2.1", ) result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) @@ -127,12 +131,19 @@ def test_reference_property_whitelist_generic_type(): assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) result = ref_prop.clean( - "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, ) assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) result = ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True + "sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + ) + assert result == ("sighting--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # The prop assumes some-type is a custom type of one of the generic + # type categories. + result = ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) @@ -146,7 +157,7 @@ def test_reference_property_whitelist_generic_type(): ref_prop.clean("identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) -def test_reference_property_blacklist_type(): +def test_reference_property_blacklist_standard_type(): ref_prop = ReferenceProperty(invalid_types="identity", spec_version="2.1") result = ref_prop.clean( "location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, @@ -158,6 +169,11 @@ def test_reference_property_blacklist_type(): ) assert result == ("location--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + result = ref_prop.clean( "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) @@ -173,15 +189,10 @@ def test_reference_property_blacklist_type(): "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) - with pytest.raises(CustomContentError): - ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, - ) - def test_reference_property_blacklist_generic_type(): ref_prop = ReferenceProperty( - invalid_types=["SDO", "SRO"], spec_version="2.1" + invalid_types=["SDO", "SRO"], spec_version="2.1", ) result = ref_prop.clean( @@ -194,16 +205,16 @@ def test_reference_property_blacklist_generic_type(): ) assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + with pytest.raises(CustomContentError): + ref_prop.clean( + "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + ) + result = ref_prop.clean( "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) assert result == ("some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) - with pytest.raises(ValueError): - ref_prop.clean( - "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, - ) - with pytest.raises(ValueError): ref_prop.clean( "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, @@ -211,7 +222,7 @@ def test_reference_property_blacklist_generic_type(): with pytest.raises(ValueError): ref_prop.clean( - "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, + "identity--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) with pytest.raises(ValueError): @@ -219,18 +230,57 @@ def test_reference_property_blacklist_generic_type(): "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, ) - with pytest.raises(CustomContentError): + with pytest.raises(ValueError): ref_prop.clean( - "some-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False, + "relationship--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True, ) -def test_reference_property_hybrid_constraint_type(): - with pytest.raises(ValueError): - ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.1") +def test_reference_property_whitelist_hybrid_type(): + p = ReferenceProperty(valid_types=["a", "SCO"], spec_version="2.1") + + result = p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(CustomContentError): + # although whitelisted, "a" is a custom type + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) with pytest.raises(ValueError): - ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.1") + p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # should just assume "b" is a custom SCO type. + result = p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + +def test_reference_property_blacklist_hybrid_type(): + p = ReferenceProperty(invalid_types=["a", "SCO"], spec_version="2.1") + + with pytest.raises(ValueError): + p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + p.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(ValueError): + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + p.clean("a--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(CustomContentError): + p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + # should just assume "b" is a custom type which is not an SCO + result = p.clean("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("b--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) def test_reference_property_impossible_constraint(): diff --git a/stix2/test/v21/test_report.py b/stix2/test/v21/test_report.py index 69af096..1dc55be 100644 --- a/stix2/test/v21/test_report.py +++ b/stix2/test/v21/test_report.py @@ -4,12 +4,12 @@ import pytest import pytz import stix2 +from stix2.exceptions import InvalidValueError from .constants import ( CAMPAIGN_ID, IDENTITY_ID, INDICATOR_ID, INDICATOR_KWARGS, RELATIONSHIP_ID, REPORT_ID, ) -from stix2.exceptions import InvalidValueError EXPECTED = """{ "type": "report", @@ -144,8 +144,8 @@ def test_report_on_custom(): published="2016-01-20T17:00:00Z", object_refs=[ "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", - "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" - ] + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7", + ], ) report = stix2.v21.Report( @@ -153,10 +153,10 @@ def test_report_on_custom(): published="2016-01-20T17:00:00Z", object_refs=[ "indicator--a740531e-63ff-4e49-a9e1-a0a3eed0e3e7", - "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" + "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7", ], allow_custom=True, ) assert "some-type--2672975a-ce1e-4473-a1c6-0d79868930c7" \ - in report.object_refs + in report.object_refs From 62ee2b2b0a4c489e358e93d58d639bc99d63a586 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 6 Jul 2020 22:43:25 -0400 Subject: [PATCH 07/21] Pre-commit changes. Also remove a dupe test function which slipped through due to a rather complex rebase... not too surprising I missed it. --- stix2/test/test_properties.py | 3 +-- stix2/test/v20/test_properties.py | 5 ++--- stix2/test/v21/test_custom.py | 12 ------------ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 60a42e6..ea9a003 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -3,10 +3,9 @@ import datetime as dt import pytest import pytz -import stix2 from stix2.base import _STIXBase from stix2.exceptions import ( - ExtraPropertiesError, STIXError, CustomContentError, + CustomContentError, ExtraPropertiesError, STIXError, ) from stix2.properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 4dbc437..0158f9a 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -9,14 +9,13 @@ from stix2.exceptions import ( ) from stix2.properties import ( DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, - HashesProperty, ListProperty, ObservableProperty, ReferenceProperty, - STIXObjectProperty, IDProperty, + HashesProperty, IDProperty, ListProperty, ObservableProperty, + ReferenceProperty, STIXObjectProperty, ) from stix2.v20.common import MarkingProperty from . import constants - ID_PROP = IDProperty('my-type', spec_version="2.0") MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7' diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 1f86a27..e51d76a 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1414,18 +1414,6 @@ def test_register_duplicate_observable_extension(): assert "cannot be registered again" in str(excinfo.value) -def test_register_duplicate_marking(): - with pytest.raises(DuplicateRegistrationError) as excinfo: - @stix2.v21.CustomMarking( - 'x-new-obj', [ - ('property1', stix2.properties.StringProperty(required=True)), - ], - ) - class NewObj2(): - pass - assert "cannot be registered again" in str(excinfo.value) - - def test_allow_custom_propagation(): obj_dict = { "type": "bundle", From c590de8ea5e38635d5aacdc7dd3b45c44eb0a651 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 9 Jul 2020 20:13:53 -0400 Subject: [PATCH 08/21] Add proper customization enforcement for open vocabs. This adds a new OpenVocabProperty class. It also requires a redesign of HashesProperty and redoes general library support for hash algorithms. --- stix2/hashes.py | 95 +++++ stix2/properties.py | 122 ++++-- stix2/test/test_properties.py | 91 ++++- stix2/test/v20/conftest.py | 16 +- stix2/test/v20/test_datastore_filters.py | 2 +- stix2/test/v20/test_datastore_memory.py | 18 +- stix2/test/v20/test_pickle.py | 2 +- stix2/test/v20/test_properties.py | 44 +-- stix2/test/v20/test_versioning.py | 8 +- stix2/test/v21/conftest.py | 16 +- stix2/test/v21/test_datastore_filters.py | 2 +- stix2/test/v21/test_datastore_memory.py | 2 +- stix2/test/v21/test_deterministic_ids.py | 3 +- stix2/test/v21/test_environment.py | 10 +- stix2/test/v21/test_location.py | 15 +- stix2/test/v21/test_malware.py | 6 +- stix2/test/v21/test_pickle.py | 2 +- stix2/test/v21/test_properties.py | 53 +-- stix2/test/v21/test_threat_actor.py | 4 +- stix2/v20/common.py | 3 +- stix2/v20/observables.py | 15 +- stix2/v20/sdo.py | 39 +- stix2/v20/vocab.py | 178 +++++++++ stix2/v21/common.py | 3 +- stix2/v21/observables.py | 124 ++----- stix2/v21/sdo.py | 72 ++-- stix2/v21/vocab.py | 452 +++++++++++++++++++++++ 27 files changed, 1046 insertions(+), 351 deletions(-) create mode 100644 stix2/hashes.py create mode 100644 stix2/v20/vocab.py create mode 100644 stix2/v21/vocab.py diff --git a/stix2/hashes.py b/stix2/hashes.py new file mode 100644 index 0000000..198cef6 --- /dev/null +++ b/stix2/hashes.py @@ -0,0 +1,95 @@ +""" +Library support for hash algorithms, independent of STIX specs. +""" + +import enum +import re + + +class Hash(enum.Enum): + """ + Instances represent a hash algorithm, independent of STIX spec version. + Different spec versions may have different requirements for naming; this + allows us to refer to and use hash algorithms in a spec-agnostic way. + """ + MD5 = 0 + MD6 = 1 + RIPEMD160 = 2 + SHA1 = 3 + SHA224 = 4 + SHA256 = 5 + SHA384 = 6 + SHA512 = 7 + SHA3224 = 8 + SHA3256 = 9 + SHA3384 = 10 + SHA3512 = 11 + SSDEEP = 12 + WHIRLPOOL = 13 + TLSH = 14 + + +# Regexes used to sanity check hash values. Could also be combined with the +# enum values themselves using enum definition tricks, but... this seems +# simpler. +_HASH_REGEXES = { + Hash.MD5: r"^[a-f0-9]{32}$", + Hash.MD6: r"^[a-f0-9]{32}|[a-f0-9]{40}|[a-f0-9]{56}|[a-f0-9]{64}|[a-f0-9]{96}|[a-f0-9]{128}$", + Hash.RIPEMD160: r"^[a-f0-9]{40}$", + Hash.SHA1: r"^[a-f0-9]{40}$", + Hash.SHA224: r"^[a-f0-9]{56}$", + Hash.SHA256: r"^[a-f0-9]{64}$", + Hash.SHA384: r"^[a-f0-9]{96}$", + Hash.SHA512: r"^[a-f0-9]{128}$", + Hash.SHA3224: r"^[a-f0-9]{56}$", + Hash.SHA3256: r"^[a-f0-9]{64}$", + Hash.SHA3384: r"^[a-f0-9]{96}$", + Hash.SHA3512: r"^[a-f0-9]{128}$", + Hash.SSDEEP: r"^[a-z0-9/+:.]{1,128}$", + Hash.WHIRLPOOL: r"^[a-f0-9]{128}$", + Hash.TLSH: r"^[a-f0-9]{70}$", +} + + +# compile all the regexes; be case-insensitive +for hash_, re_str in _HASH_REGEXES.items(): + _HASH_REGEXES[hash_] = re.compile(re_str, re.I) + + +def infer_hash_algorithm(name): + """ + Given a hash algorithm name, try to figure out which hash algorithm it + refers to. This primarily enables some user flexibility in naming hash + algorithms when creating STIX content. + + :param name: A hash algorithm name + :return: A Hash enum value if the name was recognized, or None if it was + not recognized. + """ + enum_name = name.replace("-", "").upper() + + try: + enum_obj = Hash[enum_name] + except KeyError: + enum_obj = None + + return enum_obj + + +def check_hash(hash_, value): + """ + Sanity check the given hash value, against the given hash algorithm. + + :param hash_: The hash algorithm, as one of the Hash enums + :param value: A hash value as string + :return: True if the value seems okay; False if not + """ + + # I guess there's no need to require a regex mapping for the algorithm... + # Just assume it's okay if we have no way to check it. + result = True + regex = _HASH_REGEXES.get(hash_) + if regex: + result = bool(regex.match(value)) + + return result diff --git a/stix2/properties.py b/stix2/properties.py index 735bc79..ef7246c 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -7,6 +7,9 @@ import inspect import re import uuid +import stix2 +import stix2.hashes + from .base import _STIXBase from .exceptions import CustomContentError, DictionaryKeyError, STIXError from .parsing import parse, parse_observable @@ -417,54 +420,65 @@ class DictionaryProperty(Property): return dictified, False -HASHES_REGEX = { - "MD5": (r"^[a-fA-F0-9]{32}$", "MD5"), - "MD6": (r"^[a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{56}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128}$", "MD6"), - "RIPEMD160": (r"^[a-fA-F0-9]{40}$", "RIPEMD-160"), - "SHA1": (r"^[a-fA-F0-9]{40}$", "SHA-1"), - "SHA224": (r"^[a-fA-F0-9]{56}$", "SHA-224"), - "SHA256": (r"^[a-fA-F0-9]{64}$", "SHA-256"), - "SHA384": (r"^[a-fA-F0-9]{96}$", "SHA-384"), - "SHA512": (r"^[a-fA-F0-9]{128}$", "SHA-512"), - "SHA3224": (r"^[a-fA-F0-9]{56}$", "SHA3-224"), - "SHA3256": (r"^[a-fA-F0-9]{64}$", "SHA3-256"), - "SHA3384": (r"^[a-fA-F0-9]{96}$", "SHA3-384"), - "SHA3512": (r"^[a-fA-F0-9]{128}$", "SHA3-512"), - "SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "SSDEEP"), - "WHIRLPOOL": (r"^[a-fA-F0-9]{128}$", "WHIRLPOOL"), - "TLSH": (r"^[a-fA-F0-9]{70}$", "TLSH"), -} - - class HashesProperty(DictionaryProperty): + def __init__(self, spec_hash_names, **kwargs): + super().__init__(**kwargs) + + self.__spec_hash_names = spec_hash_names + + # Map hash algorithm enum to the given spec mandated name, for those + # names which are recognized as hash algorithms by this library. + self.__alg_to_spec_name = {} + for spec_hash_name in spec_hash_names: + alg = stix2.hashes.infer_hash_algorithm(spec_hash_name) + if alg: + self.__alg_to_spec_name[alg] = spec_hash_name + def clean(self, value, allow_custom): # ignore the has_custom return value here; there is no customization # of DictionaryProperties. - clean_dict, _ = super(HashesProperty, self).clean( - value, allow_custom, - ) + clean_dict, _ = super().clean(value, allow_custom) + + spec_dict = {} has_custom = False - for k, v in copy.deepcopy(clean_dict).items(): - key = k.upper().replace('-', '') - if key in HASHES_REGEX: - vocab_key = HASHES_REGEX[key][1] - if vocab_key == "SSDEEP" and self.spec_version == "2.0": - vocab_key = vocab_key.lower() - if not re.match(HASHES_REGEX[key][0], v): - raise ValueError("'{0}' is not a valid {1} hash".format(v, vocab_key)) - if k != vocab_key: - clean_dict[vocab_key] = clean_dict[k] - del clean_dict[k] + for hash_k, hash_v in clean_dict.items(): + hash_alg = stix2.hashes.infer_hash_algorithm(hash_k) + + if hash_alg: + # Library-supported hash algorithm: sanity check the value. + if not stix2.hashes.check_hash(hash_alg, hash_v): + raise ValueError( + "'{0}' is not a valid {1} hash".format( + hash_v, hash_alg.name, + ), + ) + + spec_name = self.__alg_to_spec_name.get(hash_alg) + if not spec_name: + # There is library support for the hash algorithm, but it's + # not in the spec. So it's custom. Just use the user's + # name as-is. + has_custom = True + spec_name = hash_k else: - has_custom = True + # Unrecognized hash algorithm; use as-is. Hash algorithm name + # must be an exact match from spec, or it will be considered + # custom. + spec_name = hash_k + if spec_name not in self.__spec_hash_names: + has_custom = True if not allow_custom and has_custom: - raise CustomContentError("custom hash found: " + k) + raise CustomContentError( + "custom hash algorithm: " + hash_k, + ) - return clean_dict, has_custom + spec_dict[spec_name] = hash_v + + return spec_dict, has_custom class BinaryProperty(Property): @@ -654,19 +668,49 @@ class EmbeddedObjectProperty(Property): class EnumProperty(StringProperty): + """ + Used for enumeration type properties. Properties of this type do not allow + customization. + """ def __init__(self, allowed, **kwargs): - if type(allowed) is not list: - allowed = list(allowed) + if isinstance(allowed, str): + allowed = [allowed] self.allowed = allowed super(EnumProperty, self).__init__(**kwargs) def clean(self, value, allow_custom): cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom) + + if cleaned_value not in self.allowed: + raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value)) + + return cleaned_value, False + + +class OpenVocabProperty(StringProperty): + """ + Used for open vocab type properties. + """ + + def __init__(self, allowed, **kwargs): + super(OpenVocabProperty, self).__init__(**kwargs) + + if isinstance(allowed, str): + allowed = [allowed] + self.allowed = allowed + + def clean(self, value, allow_custom): + cleaned_value, _ = super(OpenVocabProperty, self).clean( + value, allow_custom, + ) + has_custom = cleaned_value not in self.allowed if not allow_custom and has_custom: - raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value)) + raise CustomContentError( + "custom value in open vocab: '{}'".format(cleaned_value), + ) return cleaned_value, has_custom diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index ea9a003..f19d1b5 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -9,8 +9,9 @@ from stix2.exceptions import ( ) from stix2.properties import ( BinaryProperty, BooleanProperty, EmbeddedObjectProperty, EnumProperty, - FloatProperty, HexProperty, IntegerProperty, ListProperty, Property, - StringProperty, TimestampProperty, TypeProperty, + FloatProperty, HashesProperty, HexProperty, IntegerProperty, ListProperty, + OpenVocabProperty, Property, StringProperty, TimestampProperty, + TypeProperty, ) @@ -363,8 +364,86 @@ def test_enum_property_invalid(): with pytest.raises(ValueError): enum_prop.clean('z', False) + with pytest.raises(ValueError): + enum_prop.clean('z', True) -def test_enum_property_custom(): - enum_prop = EnumProperty(['a', 'b', 'c']) - result = enum_prop.clean("z", True) - assert result == ("z", True) + +@pytest.mark.parametrize( + "vocab", [ + ['a', 'b', 'c'], + ('a', 'b', 'c'), + 'b', + ], +) +def test_openvocab_property(vocab): + ov_prop = OpenVocabProperty(vocab) + + assert ov_prop.clean("b", False) == ("b", False) + assert ov_prop.clean("b", True) == ("b", False) + + with pytest.raises(CustomContentError): + ov_prop.clean("d", False) + + assert ov_prop.clean("d", True) == ("d", True) + + +@pytest.mark.parametrize( + "value", [ + {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"}, + [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')], + ], +) +def test_hashes_property_valid(value): + hash_prop = HashesProperty(["sha256", "md5", "ripemd160"]) + _, has_custom = hash_prop.clean(value, False) + assert not has_custom + + +@pytest.mark.parametrize( + "value", [ + {"MD5": "a"}, + {"SHA-256": "2dfb1bcc980200c6706feee399d41b3f"}, + ], +) +def test_hashes_property_invalid(value): + hash_prop = HashesProperty(["sha256", "md5"]) + + with pytest.raises(ValueError): + hash_prop.clean(value, False) + + +def test_hashes_property_custom(): + value = { + "sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", + "abc-123": "aaaaaaaaaaaaaaaaaaaaa", + } + expected_cleaned_value = { + # cleaning transforms recognized hash algorithm names to the spec- + # mandated name. + "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", + "abc-123": "aaaaaaaaaaaaaaaaaaaaa", + } + + hash_prop = HashesProperty(["SHA-256"]) + result = hash_prop.clean(value, True) + assert result == (expected_cleaned_value, True) + + with pytest.raises(CustomContentError): + hash_prop.clean(value, False) + + +def test_hashes_no_library_support(): + prop = HashesProperty(["foo"]) + + result = prop.clean({"foo": "bar"}, False) + assert result == ({"foo": "bar"}, False) + + result = prop.clean({"foo": "bar"}, True) + assert result == ({"foo": "bar"}, False) + + with pytest.raises(CustomContentError): + # require exact name match for unsupported hash algorithms + prop.clean({"FOO": "bar"}, False) + + result = prop.clean({"FOO": "bar"}, True) + assert result == ({"FOO": "bar"}, True) diff --git a/stix2/test/v20/conftest.py b/stix2/test/v20/conftest.py index 6bb8fae..0e4bc4c 100644 --- a/stix2/test/v20/conftest.py +++ b/stix2/test/v20/conftest.py @@ -55,7 +55,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -67,7 +67,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -79,7 +79,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.936Z", "name": "Malicious site hosting downloader", @@ -91,7 +91,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -103,7 +103,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -156,7 +156,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-31T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -168,7 +168,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -180,7 +180,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", diff --git a/stix2/test/v20/test_datastore_filters.py b/stix2/test/v20/test_datastore_filters.py index e8945d1..bcbd08f 100644 --- a/stix2/test/v20/test_datastore_filters.py +++ b/stix2/test/v20/test_datastore_filters.py @@ -20,7 +20,7 @@ stix_objs = [ "created": "2014-05-08T09:00:00.000Z", "id": "indicator--a932fcc6-e032-476c-826f-cb970a5a1ade", "labels": [ - "file-hash-watchlist", + "compromised", ], "modified": "2014-05-08T09:00:00.000Z", "name": "File hash for Poison Ivy variant", diff --git a/stix2/test/v20/test_datastore_memory.py b/stix2/test/v20/test_datastore_memory.py index 3357e2c..23b3966 100644 --- a/stix2/test/v20/test_datastore_memory.py +++ b/stix2/test/v20/test_datastore_memory.py @@ -19,7 +19,7 @@ IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -31,7 +31,7 @@ IND2 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -43,7 +43,7 @@ IND3 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.936Z", "name": "Malicious site hosting downloader", @@ -55,7 +55,7 @@ IND4 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -67,7 +67,7 @@ IND5 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -79,7 +79,7 @@ IND6 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-31T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -91,7 +91,7 @@ IND7 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -103,7 +103,7 @@ IND8 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "labels": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -285,7 +285,7 @@ def test_memory_store_object_creator_of_present(mem_store): iden = Identity( id=IDENTITY_ID, name="Foo Corp.", - identity_class="corporation", + identity_class="organization", ) mem_store.add(camp) diff --git a/stix2/test/v20/test_pickle.py b/stix2/test/v20/test_pickle.py index 416341c..36e4337 100644 --- a/stix2/test/v20/test_pickle.py +++ b/stix2/test/v20/test_pickle.py @@ -13,7 +13,7 @@ def test_pickling(): id=IDENTITY_ID, name="alice", description="this is a pickle test", - identity_class="some_class", + identity_class="individual", ) pickle.loads(pickle.dumps(identity)) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index 0158f9a..d760363 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -8,9 +8,8 @@ from stix2.exceptions import ( ExtraPropertiesError, ParseError, ) from stix2.properties import ( - DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, - HashesProperty, IDProperty, ListProperty, ObservableProperty, - ReferenceProperty, STIXObjectProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, IDProperty, + ListProperty, ObservableProperty, ReferenceProperty, STIXObjectProperty, ) from stix2.v20.common import MarkingProperty @@ -354,45 +353,6 @@ def test_property_list_of_dictionary(): assert test_obj.property1[0]['foo'] == 'bar' -@pytest.mark.parametrize( - "value", [ - {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"}, - [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')], - ], -) -def test_hashes_property_valid(value): - hash_prop = HashesProperty() - _, has_custom = hash_prop.clean(value, False) - assert not has_custom - - -@pytest.mark.parametrize( - "value", [ - {"MD5": "a"}, - {"SHA-256": "2dfb1bcc980200c6706feee399d41b3f"}, - ], -) -def test_hashes_property_invalid(value): - hash_prop = HashesProperty() - - with pytest.raises(ValueError): - hash_prop.clean(value, False) - - -def test_hashes_property_custom(): - value = { - "sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", - "abc-123": "aaaaaaaaaaaaaaaaaaaaa", - } - - hash_prop = HashesProperty() - result = hash_prop.clean(value, True) - assert result == (value, True) - - with pytest.raises(CustomContentError): - hash_prop.clean(value, False) - - def test_embedded_property(): emb_prop = EmbeddedObjectProperty(type=stix2.v20.EmailMIMEComponent) mime = stix2.v20.EmailMIMEComponent( diff --git a/stix2/test/v20/test_versioning.py b/stix2/test/v20/test_versioning.py index 665ac4c..d3973f0 100644 --- a/stix2/test/v20/test_versioning.py +++ b/stix2/test/v20/test_versioning.py @@ -433,7 +433,7 @@ def test_version_marking(): def test_version_disable_custom(): m = stix2.v20.Malware( - name="foo", labels=["label"], description="Steals your identity!", + name="foo", labels=["spyware"], description="Steals your identity!", x_custom=123, allow_custom=True, ) @@ -450,7 +450,7 @@ def test_version_disable_custom(): def test_version_enable_custom(): m = stix2.v20.Malware( - name="foo", labels=["label"], description="Steals your identity!", + name="foo", labels=["spyware"], description="Steals your identity!", ) # Add a custom property to an object for which it was previously disallowed @@ -464,7 +464,7 @@ def test_version_enable_custom(): def test_version_propagate_custom(): m = stix2.v20.Malware( - name="foo", labels=["label"], + name="foo", labels=["spyware"], ) # Remember custom-not-allowed setting from original; produce error @@ -476,7 +476,7 @@ def test_version_propagate_custom(): assert m2.description == "Steals your identity!" m_custom = stix2.v20.Malware( - name="foo", labels=["label"], x_custom=123, allow_custom=True, + name="foo", labels=["spyware"], x_custom=123, allow_custom=True, ) # Remember custom-allowed setting from original; should work diff --git a/stix2/test/v21/conftest.py b/stix2/test/v21/conftest.py index 6efcf39..9e37d92 100644 --- a/stix2/test/v21/conftest.py +++ b/stix2/test/v21/conftest.py @@ -66,7 +66,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -80,7 +80,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -94,7 +94,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.936Z", "name": "Malicious site hosting downloader", @@ -108,7 +108,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -122,7 +122,7 @@ def stix_objs1(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -183,7 +183,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000001", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-31T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -197,7 +197,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", @@ -211,7 +211,7 @@ def stix_objs2(): "created": "2017-01-27T13:49:53.935Z", "id": "indicator--00000000-0000-4000-8000-000000000002", "indicator_types": [ - "url-watchlist", + "malicious-activity", ], "modified": "2017-01-27T13:49:53.935Z", "name": "Malicious site hosting downloader", diff --git a/stix2/test/v21/test_datastore_filters.py b/stix2/test/v21/test_datastore_filters.py index a6a50a7..0e531fd 100644 --- a/stix2/test/v21/test_datastore_filters.py +++ b/stix2/test/v21/test_datastore_filters.py @@ -24,7 +24,7 @@ stix_objs = [ "created": "2014-05-08T09:00:00.000Z", "id": "indicator--a932fcc6-e032-476c-826f-cb970a5a1ade", "indicator_types": [ - "file-hash-watchlist", + "compromised", ], "modified": "2014-05-08T09:00:00.000Z", "name": "File hash for Poison Ivy variant", diff --git a/stix2/test/v21/test_datastore_memory.py b/stix2/test/v21/test_datastore_memory.py index 870f82e..f2c9a8c 100644 --- a/stix2/test/v21/test_datastore_memory.py +++ b/stix2/test/v21/test_datastore_memory.py @@ -300,7 +300,7 @@ def test_memory_store_object_creator_of_present(mem_store): iden = Identity( id=IDENTITY_ID, name="Foo Corp.", - identity_class="corporation", + identity_class="organization", ) mem_store.add(camp) diff --git a/stix2/test/v21/test_deterministic_ids.py b/stix2/test/v21/test_deterministic_ids.py index 56b2e8a..f8a60e4 100644 --- a/stix2/test/v21/test_deterministic_ids.py +++ b/stix2/test/v21/test_deterministic_ids.py @@ -14,6 +14,7 @@ from stix2.properties import ( TypeProperty, ) import stix2.v21 +from stix2.v21.vocab import HASHING_ALGORITHM SCO_DET_ID_NAMESPACE = uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7") @@ -175,7 +176,7 @@ def test_empty_hash(): spec_version='2.1', enclosing_type=_type, ), ), - ('hashes', HashesProperty()), + ('hashes', HashesProperty(HASHING_ALGORITHM)), )) _id_contributing_properties = ['hashes'] diff --git a/stix2/test/v21/test_environment.py b/stix2/test/v21/test_environment.py index 7f6b71c..883ff01 100644 --- a/stix2/test/v21/test_environment.py +++ b/stix2/test/v21/test_environment.py @@ -490,7 +490,7 @@ def test_object_similarity_on_same_identity2(): IDEN_KWARGS = dict( name="John Smith", identity_class="individual", - sectors=["government", "critical-infrastructure"], + sectors=["government", "infrastructure"], ) iden1 = stix2.v21.Identity(id=IDENTITY_ID, **IDEN_KWARGS) iden2 = stix2.v21.Identity(id=IDENTITY_ID, **IDEN_KWARGS) @@ -723,7 +723,7 @@ def test_object_similarity_different_spec_version_raises(): def test_object_similarity_zero_match(): IND_KWARGS = dict( - indicator_types=["malicious-activity", "bar"], + indicator_types=["anomalous-activity"], pattern="[ipv4-addr:value = '192.168.1.1']", pattern_type="stix", valid_from="2019-01-01T12:34:56Z", @@ -743,14 +743,14 @@ def test_object_similarity_zero_match(): ind1 = stix2.v21.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) ind2 = stix2.v21.Indicator(id=INDICATOR_ID, **IND_KWARGS) env = stix2.Environment().object_similarity(ind1, ind2, **weights) - assert round(env) == 8 + assert round(env) == 0 env = stix2.Environment().object_similarity(ind2, ind1, **weights) - assert round(env) == 8 + assert round(env) == 0 def test_object_similarity_different_spec_version(): IND_KWARGS = dict( - labels=["APTX"], + labels=["malicious-activity"], pattern="[ipv4-addr:value = '192.168.1.1']", ) weights = { diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index cf9c101..e912fba 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -36,7 +36,7 @@ EXPECTED_LOCATION_2 = """{ "id": "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64", "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", - "region": "north-america" + "region": "northern-america" } """ @@ -47,8 +47,7 @@ EXPECTED_LOCATION_2_REPR = "Location(" + " ".join( id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64', created='2016-04-06T20:03:00.000Z', modified='2016-04-06T20:03:00.000Z', - region='north-america'""".split(), -) + ")" + region='northern-america'""".split()) + ")" def test_location_with_some_required_properties(): @@ -76,7 +75,7 @@ def test_location_with_some_required_properties(): "id": LOCATION_ID, "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", - "region": "north-america", + "region": "northern-america", }, ], ) @@ -88,7 +87,7 @@ def test_parse_location(data): assert location.id == LOCATION_ID assert location.created == dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) assert location.modified == dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) - assert location.region == 'north-america' + assert location.region == 'northern-america' rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(location)) assert rep == EXPECTED_LOCATION_2_REPR @@ -302,6 +301,7 @@ def test_google_map_url_multiple_props_no_long_lat_provided(): region="North America", country="United States of America", street_address="1410 Museum Campus Drive, Chicago, IL 60605", + allow_custom=True, ) loc_url = loc.to_maps_url() @@ -312,7 +312,7 @@ def test_google_map_url_multiple_props_and_long_lat_provided(): expected_url = "https://www.google.com/maps/search/?api=1&query=41.862401%2C-87.616001" loc = stix2.v21.Location( - region="North America", + region="northern-america", country="United States of America", street_address="1410 Museum Campus Drive, Chicago, IL 60605", latitude=41.862401, @@ -354,6 +354,7 @@ def test_bing_map_url_multiple_props_no_long_lat_provided(): region="North America", country="United States of America", street_address="1410 Museum Campus Drive, Chicago, IL 60605", + allow_custom=True, ) loc_url = loc.to_maps_url("Bing Maps") @@ -364,7 +365,7 @@ def test_bing_map_url_multiple_props_and_long_lat_provided(): expected_url = "https://bing.com/maps/default.aspx?where1=41.862401%2C-87.616001&lvl=16" loc = stix2.v21.Location( - region="North America", + region="northern-america", country="United States of America", street_address="1410 Museum Campus Drive, Chicago, IL 60605", latitude=41.862401, diff --git a/stix2/test/v21/test_malware.py b/stix2/test/v21/test_malware.py index f111826..0425204 100644 --- a/stix2/test/v21/test_malware.py +++ b/stix2/test/v21/test_malware.py @@ -181,7 +181,7 @@ def test_malware_family_no_name(): "id": MALWARE_ID, "spec_version": "2.1", "is_family": True, - "malware_types": ["a type"], + "malware_types": ["spyware"], }) @@ -191,7 +191,7 @@ def test_malware_non_family_no_name(): "id": MALWARE_ID, "spec_version": "2.1", "is_family": False, - "malware_types": ["something"], + "malware_types": ["spyware"], }) @@ -207,7 +207,7 @@ def test_malware_with_os_refs(): "id": MALWARE_ID, "spec_version": "2.1", "is_family": False, - "malware_types": ["something"], + "malware_types": ["spyware"], "operating_system_refs": [software], }) diff --git a/stix2/test/v21/test_pickle.py b/stix2/test/v21/test_pickle.py index faef4c4..55e8139 100644 --- a/stix2/test/v21/test_pickle.py +++ b/stix2/test/v21/test_pickle.py @@ -13,7 +13,7 @@ def test_pickling(): id=IDENTITY_ID, name="alice", description="this is a pickle test", - identity_class="some_class", + identity_class="individual", ) pickle.loads(pickle.dumps(identity)) diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 413aa0d..7ccf6f2 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -6,9 +6,9 @@ from stix2.exceptions import ( ExtraPropertiesError, ParseError, ) from stix2.properties import ( - DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, - HashesProperty, IDProperty, ListProperty, ObservableProperty, - ReferenceProperty, STIXObjectProperty, StringProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, IDProperty, + ListProperty, ObservableProperty, ReferenceProperty, STIXObjectProperty, + StringProperty, ) from stix2.v21.common import MarkingProperty @@ -367,53 +367,6 @@ def test_property_list_of_dictionary(): assert test_obj.property1[0]['foo'] == 'bar' -@pytest.mark.parametrize( - "value", [ - {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"}, - [('MD5', '2dfb1bcc980200c6706feee399d41b3f'), ('RIPEMD-160', 'b3a8cd8a27c90af79b3c81754f267780f443dfef')], - [('TLSH', '6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8')], - ], -) -def test_hashes_property_valid(value): - hash_prop = HashesProperty() - _, has_custom = hash_prop.clean(value, False) - assert not has_custom - - _, has_custom = hash_prop.clean(value, True) - assert not has_custom - - -@pytest.mark.parametrize( - "value", [ - {"MD5": "a"}, - {"SHA-256": "2dfb1bcc980200c6706feee399d41b3f"}, - {"TLSH": "6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F"}, - ], -) -def test_hashes_property_invalid(value): - hash_prop = HashesProperty() - - with pytest.raises(ValueError): - hash_prop.clean(value, False) - - with pytest.raises(ValueError): - hash_prop.clean(value, True) - - -def test_hashes_property_custom(): - value = { - "sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b", - "abc-123": "aaaaaaaaaaaaaaaaaaaaa", - } - - hash_prop = HashesProperty() - result = hash_prop.clean(value, True) - assert result == (value, True) - - with pytest.raises(CustomContentError): - hash_prop.clean(value, False) - - def test_embedded_property(): emb_prop = EmbeddedObjectProperty(type=stix2.v21.EmailMIMEComponent) mime = stix2.v21.EmailMIMEComponent( diff --git a/stix2/test/v21/test_threat_actor.py b/stix2/test/v21/test_threat_actor.py index 6a782ef..899c300 100644 --- a/stix2/test/v21/test_threat_actor.py +++ b/stix2/test/v21/test_threat_actor.py @@ -76,7 +76,7 @@ def test_seen_ordering_constraint(): with pytest.raises(ValueError): stix2.v21.ThreatActor( name="Bad Person", - threat_actor_types=["bad person", "evil person"], + threat_actor_types=["hacker", "criminal"], first_seen="2010-04-21T09:31:11Z", last_seen="2009-02-06T03:39:31Z", ) @@ -84,7 +84,7 @@ def test_seen_ordering_constraint(): # equal timestamps is okay. stix2.v21.ThreatActor( name="Bad Person", - threat_actor_types=["bad person", "evil person"], + threat_actor_types=["hacker", "criminal"], first_seen="2010-04-21T09:31:11Z", last_seen="2010-04-21T09:31:11Z", ) diff --git a/stix2/v20/common.py b/stix2/v20/common.py index fe151fb..acff911 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -12,6 +12,7 @@ from ..properties import ( ) from ..utils import NOW, _get_dict from .base import _STIXBase20 +from .vocab import HASHING_ALGORITHM def _should_set_millisecond(cr, marking_type): @@ -38,7 +39,7 @@ class ExternalReference(_STIXBase20): ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version='2.0')), ('external_id', StringProperty()), ]) diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 56262b0..793b683 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -17,6 +17,7 @@ from ..properties import ( ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from .base import _Extension, _Observable, _STIXBase20 +from .vocab import HASHING_ALGORITHM class Artifact(_Observable): @@ -30,7 +31,7 @@ class Artifact(_Observable): ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -173,7 +174,7 @@ class AlternateDataStream(_STIXBase20): _properties = OrderedDict([ ('name', StringProperty(required=True)), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('size', IntegerProperty()), ]) @@ -256,7 +257,7 @@ class WindowsPEOptionalHeaderType(_STIXBase20): ('size_of_heap_commit', IntegerProperty()), ('loader_flags_hex', HexProperty()), ('number_of_rva_and_sizes', IntegerProperty()), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ]) def _check_object_constraints(self): @@ -273,7 +274,7 @@ class WindowsPESection(_STIXBase20): ('name', StringProperty(required=True)), ('size', IntegerProperty()), ('entropy', FloatProperty()), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ]) @@ -293,7 +294,7 @@ class WindowsPEBinaryExt(_Extension): ('number_of_symbols', IntegerProperty()), ('size_of_optional_header', IntegerProperty()), ('characteristics_hex', HexProperty()), - ('file_header_hashes', HashesProperty(spec_version='2.0')), + ('file_header_hashes', HashesProperty(HASHING_ALGORITHM)), ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), ]) @@ -307,7 +308,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('size', IntegerProperty()), ('name', StringProperty()), ('name_enc', StringProperty()), @@ -771,7 +772,7 @@ class X509Certificate(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('is_self_signed', BooleanProperty()), - ('hashes', HashesProperty(spec_version='2.0')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('version', StringProperty()), ('serial_number', StringProperty()), ('signature_algorithm', StringProperty()), diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 3c854e0..d08063e 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -9,12 +9,17 @@ from ..custom import _custom_object_builder from ..exceptions import InvalidValueError from ..properties import ( BooleanProperty, IDProperty, IntegerProperty, ListProperty, - ObservableProperty, PatternProperty, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty, + ObservableProperty, OpenVocabProperty, PatternProperty, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW from .base import _DomainObject from .common import ExternalReference, GranularMarking, KillChainPhase +from .vocab import ( + ATTACK_MOTIVATION, ATTACK_RESOURCE_LEVEL, IDENTITY_CLASS, INDICATOR_LABEL, + INDUSTRY_SECTOR, MALWARE_LABEL, REPORT_LABEL, THREAT_ACTOR_LABEL, + THREAT_ACTOR_ROLE, THREAT_ACTOR_SOPHISTICATION, TOOL_LABEL, +) class AttackPattern(_DomainObject): @@ -102,8 +107,8 @@ class Identity(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('identity_class', StringProperty(required=True)), - ('sectors', ListProperty(StringProperty)), + ('identity_class', OpenVocabProperty(IDENTITY_CLASS, required=True)), + ('sectors', ListProperty(OpenVocabProperty(INDUSTRY_SECTOR))), ('contact_information', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -132,7 +137,7 @@ class Indicator(_DomainObject): ('valid_until', TimestampProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty, required=True)), + ('labels', ListProperty(OpenVocabProperty(INDICATOR_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), @@ -163,8 +168,8 @@ class IntrusionSet(_DomainObject): ('last_seen', TimestampProperty()), ('goals', ListProperty(StringProperty)), ('resource_level', StringProperty()), - ('primary_motivation', StringProperty()), - ('secondary_motivations', ListProperty(StringProperty)), + ('primary_motivation', OpenVocabProperty(ATTACK_MOTIVATION)), + ('secondary_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('external_references', ListProperty(ExternalReference)), @@ -189,7 +194,7 @@ class Malware(_DomainObject): ('description', StringProperty()), ('kill_chain_phases', ListProperty(KillChainPhase)), ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty, required=True)), + ('labels', ListProperty(OpenVocabProperty(MALWARE_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), @@ -237,7 +242,7 @@ class Report(_DomainObject): ('published', TimestampProperty(required=True)), ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty, required=True)), + ('labels', ListProperty(OpenVocabProperty(REPORT_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), @@ -259,15 +264,15 @@ class ThreatActor(_DomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('aliases', ListProperty(StringProperty)), - ('roles', ListProperty(StringProperty)), + ('roles', ListProperty(OpenVocabProperty(THREAT_ACTOR_ROLE))), ('goals', ListProperty(StringProperty)), - ('sophistication', StringProperty()), - ('resource_level', StringProperty()), - ('primary_motivation', StringProperty()), - ('secondary_motivations', ListProperty(StringProperty)), - ('personal_motivations', ListProperty(StringProperty)), + ('sophistication', OpenVocabProperty(THREAT_ACTOR_SOPHISTICATION)), + ('resource_level', OpenVocabProperty(ATTACK_RESOURCE_LEVEL)), + ('primary_motivation', OpenVocabProperty(ATTACK_MOTIVATION)), + ('secondary_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), + ('personal_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty, required=True)), + ('labels', ListProperty(OpenVocabProperty(THREAT_ACTOR_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), @@ -291,7 +296,7 @@ class Tool(_DomainObject): ('kill_chain_phases', ListProperty(KillChainPhase)), ('tool_version', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), - ('labels', ListProperty(StringProperty, required=True)), + ('labels', ListProperty(OpenVocabProperty(TOOL_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.0'))), ('granular_markings', ListProperty(GranularMarking)), diff --git a/stix2/v20/vocab.py b/stix2/v20/vocab.py new file mode 100644 index 0000000..b0f9a70 --- /dev/null +++ b/stix2/v20/vocab.py @@ -0,0 +1,178 @@ +""" +STIX 2.0 open vocabularies and enums +""" + +ATTACK_MOTIVATION = [ + "accidental", + "coercion", + "dominance", + "ideology", + "notoriety", + "organizational-gain", + "personal-gain", + "personal-satisfaction", + "revenge", + "unpredictable", +] + + +ATTACK_RESOURCE_LEVEL = [ + "individual", + "club", + "contest", + "team", + "organization", + "government", +] + + +HASHING_ALGORITHM = [ + "MD5", + "MD6", + "RIPEMD-160", + "SHA-1", + "SHA-224", + "SHA-256", + "SHA-384", + "SHA-512", + "SHA3-224", + "SHA3-256", + "SHA3-384", + "SHA3-512", + "ssdeep", + "WHIRLPOOL", +] + + +IDENTITY_CLASS = [ + "individual", + "group", + "organization", + "class", + "unknown", +] + + +INDICATOR_LABEL = [ + "anomalous-activity", + "anonymization", + "benign", + "compromised", + "malicious-activity", + "attribution", +] + + +INDUSTRY_SECTOR = [ + "agriculture", + "aerospace", + "automotive", + "communications", + "construction", + "defence", + "education", + "energy", + "entertainment", + "financial-services", + "government-national", + "government-regional", + "government-local", + "government-public-services", + "healthcare", + "hospitality-leisure", + "infrastructure", + "insurance", + "manufacturing", + "mining", + "non-profit", + "pharmaceuticals", + "retail", + "technology", + "telecommunications", + "transportation", + "utilities", +] + + +MALWARE_LABEL = [ + "adware", + "backdoor", + "bot", + "ddos", + "dropper", + "exploit-kit", + "keylogger", + "ransomware", + "remote-access-trojan", + "resource-exploitation", + "rogue-security-software", + "rootkit", + "screen-capture", + "spyware", + "trojan", + "virus", + "worm", +] + + +REPORT_LABEL = [ + "threat-report", + "attack-pattern", + "campaign", + "identity", + "indicator", + "intrusion-set", + "malware", + "observed-data", + "threat-actor", + "tool", + "vulnerability", +] + + +THREAT_ACTOR_LABEL = [ + "activist", + "competitor", + "crime-syndicate", + "criminal", + "hacker", + "insider-accidental", + "insider-disgruntled", + "nation-state", + "sensationalist", + "spy", + "terrorist", +] + + +THREAT_ACTOR_ROLE = [ + "agent", + "director", + "independent", + "infrastructure-architect", + "infrastructure-operator", + "malware-author", + "sponsor", +] + + +THREAT_ACTOR_SOPHISTICATION = [ + "none", + "minimal", + "intermediate", + "advanced", + "expert", + "innovator", + "strategic", +] + + +TOOL_LABEL = [ + "denial-of-service", + "exploitation", + "information-gathering", + "network-capture", + "credential-exploitation", + "remote-access", + "vulnerability-scanning", +] diff --git a/stix2/v21/common.py b/stix2/v21/common.py index af0a758..49fdf87 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -13,6 +13,7 @@ from ..properties import ( ) from ..utils import NOW, _get_dict from .base import _STIXBase21 +from .vocab import HASHING_ALGORITHM class ExternalReference(_STIXBase21): @@ -24,7 +25,7 @@ class ExternalReference(_STIXBase21): ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('external_id', StringProperty()), ]) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 430afea..9963d49 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -14,10 +14,17 @@ from ..properties import ( BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, + OpenVocabProperty, ReferenceProperty, StringProperty, TimestampProperty, + TypeProperty, ) from .base import _Extension, _Observable, _STIXBase21 from .common import GranularMarking +from .vocab import ( + ACCOUNT_TYPE, ENCRYPTION_ALGORITHM, HASHING_ALGORITHM, + NETWORK_SOCKET_ADDRESS_FAMILY, NETWORK_SOCKET_TYPE, + WINDOWS_INTEGRITY_LEVEL, WINDOWS_PEBINARY_TYPE, WINDOWS_REGISTRY_DATATYPE, + WINDOWS_SERVICE_START_TYPE, WINDOWS_SERVICE_STATUS, WINDOWS_SERVICE_TYPE, +) class Artifact(_Observable): @@ -33,8 +40,8 @@ class Artifact(_Observable): ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(spec_version='2.1')), - ('encryption_algorithm', StringProperty()), + ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('encryption_algorithm', EnumProperty(ENCRYPTION_ALGORITHM)), ('decryption_key', StringProperty()), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), @@ -212,7 +219,7 @@ class AlternateDataStream(_STIXBase21): _properties = OrderedDict([ ('name', StringProperty(required=True)), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('size', IntegerProperty()), ]) @@ -294,7 +301,7 @@ class WindowsPEOptionalHeaderType(_STIXBase21): ('size_of_heap_commit', IntegerProperty()), ('loader_flags_hex', HexProperty()), ('number_of_rva_and_sizes', IntegerProperty()), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ]) def _check_object_constraints(self): @@ -311,7 +318,7 @@ class WindowsPESection(_STIXBase21): ('name', StringProperty(required=True)), ('size', IntegerProperty(min=0)), ('entropy', FloatProperty()), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ]) @@ -322,7 +329,7 @@ class WindowsPEBinaryExt(_Extension): _type = 'windows-pebinary-ext' _properties = OrderedDict([ - ('pe_type', StringProperty(required=True)), # open_vocab + ('pe_type', OpenVocabProperty(WINDOWS_PEBINARY_TYPE, required=True)), ('imphash', StringProperty()), ('machine_hex', HexProperty()), ('number_of_sections', IntegerProperty(min=0)), @@ -331,7 +338,7 @@ class WindowsPEBinaryExt(_Extension): ('number_of_symbols', IntegerProperty(min=0)), ('size_of_optional_header', IntegerProperty(min=0)), ('characteristics_hex', HexProperty()), - ('file_header_hashes', HashesProperty(spec_version='2.1')), + ('file_header_hashes', HashesProperty(HASHING_ALGORITHM)), ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), ]) @@ -347,7 +354,7 @@ class File(_Observable): ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('size', IntegerProperty(min=0)), ('name', StringProperty()), ('name_enc', StringProperty()), @@ -487,34 +494,11 @@ class SocketExt(_Extension): _type = 'socket-ext' _properties = OrderedDict([ - ( - 'address_family', EnumProperty( - allowed=[ - "AF_UNSPEC", - "AF_INET", - "AF_IPX", - "AF_APPLETALK", - "AF_NETBIOS", - "AF_INET6", - "AF_IRDA", - "AF_BTH", - ], required=True, - ), - ), + ('address_family', EnumProperty(NETWORK_SOCKET_ADDRESS_FAMILY, required=True)), ('is_blocking', BooleanProperty()), ('is_listening', BooleanProperty()), ('options', DictionaryProperty(spec_version='2.1')), - ( - 'socket_type', EnumProperty( - allowed=[ - "SOCK_STREAM", - "SOCK_DGRAM", - "SOCK_RAW", - "SOCK_RDM", - "SOCK_SEQPACKET", - ], - ), - ), + ('socket_type', EnumProperty(NETWORK_SOCKET_TYPE)), ('socket_descriptor', IntegerProperty(min=0)), ('socket_handle', IntegerProperty()), ]) @@ -613,16 +597,7 @@ class WindowsProcessExt(_Extension): ('owner_sid', StringProperty()), ('window_title', StringProperty()), ('startup_info', DictionaryProperty(spec_version='2.1')), - ( - 'integrity_level', EnumProperty( - allowed=[ - "low", - "medium", - "high", - "system", - ], - ), - ), + ('integrity_level', EnumProperty(WINDOWS_INTEGRITY_LEVEL)), ]) @@ -637,41 +612,10 @@ class WindowsServiceExt(_Extension): ('descriptions', ListProperty(StringProperty)), ('display_name', StringProperty()), ('group_name', StringProperty()), - ( - 'start_type', EnumProperty( - allowed=[ - "SERVICE_AUTO_START", - "SERVICE_BOOT_START", - "SERVICE_DEMAND_START", - "SERVICE_DISABLED", - "SERVICE_SYSTEM_ALERT", - ], - ), - ), + ('start_type', EnumProperty(WINDOWS_SERVICE_START_TYPE)), ('service_dll_refs', ListProperty(ReferenceProperty(valid_types='file', spec_version="2.1"))), - ( - 'service_type', EnumProperty( - allowed=[ - "SERVICE_KERNEL_DRIVER", - "SERVICE_FILE_SYSTEM_DRIVER", - "SERVICE_WIN32_OWN_PROCESS", - "SERVICE_WIN32_SHARE_PROCESS", - ], - ), - ), - ( - 'service_status', EnumProperty( - allowed=[ - "SERVICE_CONTINUE_PENDING", - "SERVICE_PAUSE_PENDING", - "SERVICE_PAUSED", - "SERVICE_RUNNING", - "SERVICE_START_PENDING", - "SERVICE_STOP_PENDING", - "SERVICE_STOPPED", - ], - ), - ), + ('service_type', EnumProperty(WINDOWS_SERVICE_TYPE)), + ('service_status', EnumProperty(WINDOWS_SERVICE_STATUS)), ]) @@ -789,7 +733,7 @@ class UserAccount(_Observable): ('user_id', StringProperty()), ('credential', StringProperty()), ('account_login', StringProperty()), - ('account_type', StringProperty()), # open vocab + ('account_type', OpenVocabProperty(ACCOUNT_TYPE)), ('display_name', StringProperty()), ('is_service_account', BooleanProperty()), ('is_privileged', BooleanProperty()), @@ -817,25 +761,7 @@ class WindowsRegistryValueType(_STIXBase21): _properties = OrderedDict([ ('name', StringProperty()), ('data', StringProperty()), - ( - 'data_type', EnumProperty( - allowed=[ - "REG_NONE", - "REG_SZ", - "REG_EXPAND_SZ", - "REG_BINARY", - "REG_DWORD", - "REG_DWORD_BIG_ENDIAN", - "REG_LINK", - "REG_MULTI_SZ", - "REG_RESOURCE_LIST", - "REG_FULL_RESOURCE_DESCRIPTION", - "REG_RESOURCE_REQUIREMENTS_LIST", - "REG_QWORD", - "REG_INVALID_TYPE", - ], - ), - ), + ('data_type', EnumProperty(WINDOWS_REGISTRY_DATATYPE)), ]) @@ -900,7 +826,7 @@ class X509Certificate(_Observable): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), - ('hashes', HashesProperty(spec_version='2.1')), + ('hashes', HashesProperty(HASHING_ALGORITHM)), ('version', StringProperty()), ('serial_number', StringProperty()), ('signature_algorithm', StringProperty()), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 3b81e43..bba531e 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -13,12 +13,20 @@ from ..exceptions import ( ) from ..properties import ( BooleanProperty, EnumProperty, FloatProperty, IDProperty, IntegerProperty, - ListProperty, ObservableProperty, PatternProperty, ReferenceProperty, - StringProperty, TimestampProperty, TypeProperty, + ListProperty, ObservableProperty, OpenVocabProperty, PatternProperty, + ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW from .base import _DomainObject from .common import ExternalReference, GranularMarking, KillChainPhase +from .vocab import ( + ATTACK_MOTIVATION, ATTACK_RESOURCE_LEVEL, GROUPING_CONTEXT, IDENTITY_CLASS, + IMPLEMENTATION_LANGUAGE, INDICATOR_TYPE, INDUSTRY_SECTOR, + INFRASTRUCTURE_TYPE, MALWARE_CAPABILITIES, MALWARE_RESULT, MALWARE_TYPE, + OPINION, PATTERN_TYPE, PROCESSOR_ARCHITECTURE, REGION, REPORT_TYPE, + THREAT_ACTOR_ROLE, THREAT_ACTOR_SOPHISTICATION, THREAT_ACTOR_TYPE, + TOOL_TYPE, +) class AttackPattern(_DomainObject): @@ -127,7 +135,7 @@ class Grouping(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), - ('context', StringProperty(required=True)), + ('context', OpenVocabProperty(GROUPING_CONTEXT, required=True)), ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -155,8 +163,8 @@ class Identity(_DomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('roles', ListProperty(StringProperty)), - ('identity_class', StringProperty()), - ('sectors', ListProperty(StringProperty)), + ('identity_class', OpenVocabProperty(IDENTITY_CLASS)), + ('sectors', ListProperty(OpenVocabProperty(INDUSTRY_SECTOR))), ('contact_information', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -183,9 +191,9 @@ class Indicator(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), - ('indicator_types', ListProperty(StringProperty)), + ('indicator_types', ListProperty(OpenVocabProperty(INDICATOR_TYPE))), ('pattern', PatternProperty(required=True)), - ('pattern_type', StringProperty(required=True)), + ('pattern_type', OpenVocabProperty(PATTERN_TYPE, required=True)), ('pattern_version', StringProperty()), ('valid_from', TimestampProperty(default=lambda: NOW)), ('valid_until', TimestampProperty()), @@ -242,7 +250,7 @@ class Infrastructure(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('infrastructure_types', ListProperty(StringProperty)), + ('infrastructure_types', ListProperty(OpenVocabProperty(INFRASTRUCTURE_TYPE))), ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('first_seen', TimestampProperty()), @@ -286,9 +294,9 @@ class IntrusionSet(_DomainObject): ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('goals', ListProperty(StringProperty)), - ('resource_level', StringProperty()), - ('primary_motivation', StringProperty()), - ('secondary_motivations', ListProperty(StringProperty)), + ('resource_level', OpenVocabProperty(ATTACK_RESOURCE_LEVEL)), + ('primary_motivation', OpenVocabProperty(ATTACK_MOTIVATION)), + ('secondary_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -327,7 +335,7 @@ class Location(_DomainObject): ('latitude', FloatProperty(min=-90.0, max=90.0)), ('longitude', FloatProperty(min=-180.0, max=180.0)), ('precision', FloatProperty(min=0.0)), - ('region', StringProperty()), + ('region', OpenVocabProperty(REGION)), ('country', StringProperty()), ('administrative_area', StringProperty()), ('city', StringProperty()), @@ -431,16 +439,16 @@ class Malware(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), - ('malware_types', ListProperty(StringProperty)), + ('malware_types', ListProperty(OpenVocabProperty(MALWARE_TYPE))), ('is_family', BooleanProperty(required=True)), ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), ('operating_system_refs', ListProperty(ReferenceProperty(valid_types='software', spec_version='2.1'))), - ('architecture_execution_envs', ListProperty(StringProperty)), - ('implementation_languages', ListProperty(StringProperty)), - ('capabilities', ListProperty(StringProperty)), + ('architecture_execution_envs', ListProperty(OpenVocabProperty(PROCESSOR_ARCHITECTURE))), + ('implementation_languages', ListProperty(OpenVocabProperty(IMPLEMENTATION_LANGUAGE))), + ('capabilities', ListProperty(OpenVocabProperty(MALWARE_CAPABILITIES))), ('sample_refs', ListProperty(ReferenceProperty(valid_types=['artifact', 'file'], spec_version='2.1'))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -494,7 +502,7 @@ class MalwareAnalysis(_DomainObject): ('analysis_started', TimestampProperty()), ('analysis_ended', TimestampProperty()), ('result_name', StringProperty()), - ('result', StringProperty()), + ('result', OpenVocabProperty(MALWARE_RESULT)), ('analysis_sco_refs', ListProperty(ReferenceProperty(valid_types="SCO", spec_version='2.1'))), ('sample_ref', ReferenceProperty(valid_types="SCO", spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), @@ -607,17 +615,7 @@ class Opinion(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('explanation', StringProperty()), ('authors', ListProperty(StringProperty)), - ( - 'opinion', EnumProperty( - allowed=[ - 'strongly-disagree', - 'disagree', - 'neutral', - 'agree', - 'strongly-agree', - ], required=True, - ), - ), + ('opinion', EnumProperty(OPINION, required=True)), ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -644,7 +642,7 @@ class Report(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('report_types', ListProperty(StringProperty)), + ('report_types', ListProperty(OpenVocabProperty(REPORT_TYPE))), ('published', TimestampProperty(required=True)), ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), @@ -672,17 +670,17 @@ class ThreatActor(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('threat_actor_types', ListProperty(StringProperty)), + ('threat_actor_types', ListProperty(OpenVocabProperty(THREAT_ACTOR_TYPE))), ('aliases', ListProperty(StringProperty)), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), - ('roles', ListProperty(StringProperty)), + ('roles', ListProperty(OpenVocabProperty(THREAT_ACTOR_ROLE))), ('goals', ListProperty(StringProperty)), - ('sophistication', StringProperty()), - ('resource_level', StringProperty()), - ('primary_motivation', StringProperty()), - ('secondary_motivations', ListProperty(StringProperty)), - ('personal_motivations', ListProperty(StringProperty)), + ('sophistication', OpenVocabProperty(THREAT_ACTOR_SOPHISTICATION)), + ('resource_level', OpenVocabProperty(ATTACK_RESOURCE_LEVEL)), + ('primary_motivation', OpenVocabProperty(ATTACK_MOTIVATION)), + ('secondary_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), + ('personal_motivations', ListProperty(OpenVocabProperty(ATTACK_MOTIVATION))), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -718,7 +716,7 @@ class Tool(_DomainObject): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), - ('tool_types', ListProperty(StringProperty)), + ('tool_types', ListProperty(OpenVocabProperty(TOOL_TYPE))), ('aliases', ListProperty(StringProperty)), ('kill_chain_phases', ListProperty(KillChainPhase)), ('tool_version', StringProperty()), diff --git a/stix2/v21/vocab.py b/stix2/v21/vocab.py new file mode 100644 index 0000000..c18c320 --- /dev/null +++ b/stix2/v21/vocab.py @@ -0,0 +1,452 @@ +""" +STIX 2.1 open vocabularies and enums +""" + +ACCOUNT_TYPE = [ + "facebook", + "ldap", + "nis", + "openid", + "radius", + "skype", + "tacacs", + "twitter", + "unix", + "windows-local", + "windows-domain", +] + + +ATTACK_MOTIVATION = [ + "accidental", + "coercion", + "dominance", + "ideology", + "notoriety", + "organizational-gain", + "personal-gain", + "personal-satisfaction", + "revenge", + "unpredictable", +] + + +ATTACK_RESOURCE_LEVEL = [ + "individual", + "club", + "contest", + "team", + "organization", + "government", +] + + +ENCRYPTION_ALGORITHM = [ + "AES-256-GCM", + "ChaCha20-Poly1305", + "mime-type-indicated", +] + + +GROUPING_CONTEXT = [ + "suspicious-activity", + "malware-analysis", + "unspecified", +] + + +HASHING_ALGORITHM = [ + "MD5", + "SHA-1", + "SHA-256", + "SHA-512", + "SHA3-256", + "SHA3-512", + "SSDEEP", + "TLSH", +] + + +IDENTITY_CLASS = [ + "individual", + "group", + "system", + "organization", + "class", + "unknown", +] + + +IMPLEMENTATION_LANGUAGE = [ + "applescript", + "bash", + "c", + "c++", + "c#", + "go", + "java", + "javascript", + "lua", + "objective-c", + "perl", + "php", + "powershell", + "python", + "ruby", + "scala", + "swift", + "typescript", + "visual-basic", + "x86-32", + "x86-64", +] + + +INDICATOR_TYPE = [ + "anomalous-activity", + "anonymization", + "benign", + "compromised", + "malicious-activity", + "attribution", + "unknown", +] + + +INDUSTRY_SECTOR = [ + "agriculture", + "aerospace", + "automotive", + "chemical", + "commercial", + "communications", + "construction", + "defense", + "education", + "energy", + "entertainment", + "financial-services", + "government", + "emergency-services", + "government-national", + "government-regional", + "government-local", + "government-public-services", + "healthcare", + "hospitality-leisure", + "infrastructure", + "dams", + "nuclear", + "water", + "insurance", + "manufacturing", + "mining", + "non-profit", + "pharmaceuticals", + "retail", + "technology", + "telecommunications", + "transportation", + "utilities", +] + + +INFRASTRUCTURE_TYPE = [ + "amplification", + "anonymization", + "botnet", + "command-and-control", + "exfiltration", + "hosting-malware", + "hosting-target-lists", + "phishing", + "reconnaissance", + "staging", + "unknown", +] + + +MALWARE_RESULT = [ + "malicious", + "suspicious", + "benign", + "unknown", +] + + +MALWARE_CAPABILITIES = [ + "accesses-remote-machines", + "anti-debugging", + "anti-disassembly", + "anti-emulation", + "anti-memory-forensics", + "anti-sandbox", + "anti-vm", + "captures-input-peripherals", + "captures-output-peripherals", + "captures-system-state-data", + "cleans-traces-of-infection", + "commits-fraud", + "communicates-with-c2", + "compromises-data-availability", + "compromises-data-integrity", + "compromises-system-availability", + "controls-local-machine", + "degrades-security-software", + "degrades-system-updates", + "determines-c2-server", + "emails-spam", + "escalates-privileges", + "evades-av", + "exfiltrates-data", + "fingerprints-host", + "hides-artifacts", + "hides-executing-code", + "infects-files", + "infects-remote-machines", + "installs-other-components", + "persists-after-system-reboot", + "prevents-artifact-access", + "prevents-artifact-deletion", + "probes-network-environment", + "self-modifies", + "steals-authentication-credentials", + "violates-system-operational-integrity", +] + + +MALWARE_TYPE = [ + "adware", + "backdoor", + "bot", + "bootkit", + "ddos", + "downloader", + "dropper", + "exploit-kit", + "keylogger", + "ransomware", + "remote-access-trojan", + "resource-exploitation", + "rogue-security-software", + "rootkit", + "screen-capture", + "spyware", + "trojan", + "unknown", + "virus", + "webshell", + "wiper", + "worm", +] + + +NETWORK_SOCKET_ADDRESS_FAMILY = [ + "AF_UNSPEC", + "AF_INET", + "AF_IPX", + "AF_APPLETALK", + "AF_NETBIOS", + "AF_INET6", + "AF_IRDA", + "AF_BTH", +] + + +NETWORK_SOCKET_TYPE = [ + "SOCK_STREAM", + "SOCK_DGRAM", + "SOCK_RAW", + "SOCK_RDM", + "SOCK_SEQPACKET", +] + + +OPINION = [ + "strongly-disagree", + "disagree", + "neutral", + "agree", + "strongly-agree", +] + + +PATTERN_TYPE = [ + "stix", + "pcre", + "sigma", + "snort", + "suricata", + "yara", +] + + +PROCESSOR_ARCHITECTURE = [ + "alpha", + "arm", + "ia-64", + "mips", + "powerpc", + "sparc", + "x86", + "x86-64", +] + + +REGION = [ + "africa", + "eastern-africa", + "middle-africa", + "northern-africa", + "southern-africa", + "western-africa", + "americas", + "latin-america-caribbean", + "south-america", + "caribbean", + "central-america", + "northern-america", + "asia", + "central-asia", + "eastern-asia", + "southern-asia", + "south-eastern-asia", + "western-asia", + "europe", + "eastern-europe", + "northern-europe", + "southern-europe", + "western-europe", + "oceania", + "antarctica", + "australia-new-zealand", + "melanesia", + "micronesia", + "polynesia", +] + + +REPORT_TYPE = [ + "attack-pattern", + "campaign", + "identity", + "indicator", + "intrusion-set", + "malware", + "observed-data", + "threat-actor", + "threat-report", + "tool", + "vulnerability", +] + + +THREAT_ACTOR_TYPE = [ + "activist", + "competitor", + "crime-syndicate", + "criminal", + "hacker", + "insider-accidental", + "insider-disgruntled", + "nation-state", + "sensationalist", + "spy", + "terrorist", + "unknown", +] + + +THREAT_ACTOR_ROLE = [ + "agent", + "director", + "independent", + "infrastructure-architect", + "infrastructure-operator", + "malware-author", + "sponsor", +] + + +THREAT_ACTOR_SOPHISTICATION = [ + "none", + "minimal", + "intermediate", + "advanced", + "expert", + "innovator", + "strategic", +] + + +TOOL_TYPE = [ + "denial-of-service", + "exploitation", + "information-gathering", + "network-capture", + "credential-exploitation", + "remote-access", + "vulnerability-scanning", + "unknown", +] + + +WINDOWS_INTEGRITY_LEVEL = [ + "low", + "medium", + "high", + "system", +] + + +WINDOWS_PEBINARY_TYPE = [ + "dll", + "exe", + "sys", +] + + +WINDOWS_REGISTRY_DATATYPE = [ + "REG_NONE", + "REG_SZ", + "REG_EXPAND_SZ", + "REG_BINARY", + "REG_DWORD", + "REG_DWORD_BIG_ENDIAN", + "REG_DWORD_LITTLE_ENDIAN", + "REG_LINK", + "REG_MULTI_SZ", + "REG_RESOURCE_LIST", + "REG_FULL_RESOURCE_DESCRIPTION", + "REG_RESOURCE_REQUIREMENTS_LIST", + "REG_QWORD", + "REG_INVALID_TYPE", +] + + +WINDOWS_SERVICE_START_TYPE = [ + "SERVICE_AUTO_START", + "SERVICE_BOOT_START", + "SERVICE_DEMAND_START", + "SERVICE_DISABLED", + "SERVICE_SYSTEM_ALERT", +] + + +WINDOWS_SERVICE_TYPE = [ + "SERVICE_KERNEL_DRIVER", + "SERVICE_FILE_SYSTEM_DRIVER", + "SERVICE_WIN32_OWN_PROCESS", + "SERVICE_WIN32_SHARE_PROCESS", +] + + +WINDOWS_SERVICE_STATUS = [ + "SERVICE_CONTINUE_PENDING", + "SERVICE_PAUSE_PENDING", + "SERVICE_PAUSED", + "SERVICE_RUNNING", + "SERVICE_START_PENDING", + "SERVICE_STOP_PENDING", + "SERVICE_STOPPED", +] From a8a65599bfaba7a1617aaa8edea65384a4f6c6f2 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 10 Jul 2020 16:29:57 -0400 Subject: [PATCH 09/21] Add back spec_version kwarg to HashesProperty, since it was used to check key lengths. Added some unit tests for hash keys. Also added a library hash support test module I'd forgotten to add before. --- stix2/properties.py | 4 +-- stix2/test/test_hashes.py | 60 +++++++++++++++++++++++++++++++ stix2/test/v20/test_properties.py | 31 ++++++++++++++-- stix2/test/v21/test_properties.py | 32 +++++++++++++++-- 4 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 stix2/test/test_hashes.py diff --git a/stix2/properties.py b/stix2/properties.py index ef7246c..4514379 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -422,8 +422,8 @@ class DictionaryProperty(Property): class HashesProperty(DictionaryProperty): - def __init__(self, spec_hash_names, **kwargs): - super().__init__(**kwargs) + def __init__(self, spec_hash_names, spec_version=stix2.DEFAULT_VERSION, **kwargs): + super().__init__(spec_version=spec_version, **kwargs) self.__spec_hash_names = spec_hash_names diff --git a/stix2/test/test_hashes.py b/stix2/test/test_hashes.py new file mode 100644 index 0000000..f3445e1 --- /dev/null +++ b/stix2/test/test_hashes.py @@ -0,0 +1,60 @@ +from stix2.hashes import Hash, infer_hash_algorithm, check_hash +import pytest + + +@pytest.mark.parametrize("hash_name, expected_alg", [ + ("md5", Hash.MD5), + ("md6", Hash.MD6), + ("ripemd160", Hash.RIPEMD160), + ("sha1", Hash.SHA1), + ("sha224", Hash.SHA224), + ("sha256", Hash.SHA256), + ("sha384", Hash.SHA384), + ("sha512", Hash.SHA512), + ("sha3224", Hash.SHA3224), + ("sha3256", Hash.SHA3256), + ("sha3384", Hash.SHA3384), + ("sha3512", Hash.SHA3512), + ("ssdeep", Hash.SSDEEP), + ("whirlpool", Hash.WHIRLPOOL), + ("tlsh", Hash.TLSH), + ("xxxx", None), +]) +def test_hash_inference(hash_name, expected_alg): + alg = infer_hash_algorithm(hash_name) + assert alg == expected_alg + + # Try some other name variations + alg = infer_hash_algorithm(hash_name[0].upper() + hash_name[1:]) + assert alg == expected_alg + + alg = infer_hash_algorithm("-"+hash_name) + assert alg == expected_alg + + +@pytest.mark.parametrize("hash_alg, hash_value", [ + (Hash.MD5, "f9e40b9aa5464f3dae711ca524fceb63"), + (Hash.MD6, "f9e40b9aa5464f3dae711ca524fceb63"), + (Hash.RIPEMD160, "8ae5d2e6b1f3a514257f2469b637454931844aeb"), + (Hash.SHA1, "f2c7d4185880c0adcbb4a01d020a69498b16210e"), + (Hash.SHA224, "6743ed70cc26e750ad0108b6b8ad7fc2780c550f7d78adefa04dda05"), + (Hash.SHA256, "a2d1c2081aa932fe72307ab076b9739455bc7a21b3bed367bd9a86ae27af5a40"), + (Hash.SHA384, "bc846457de707f97bce93cca23b5ea58c0326fd8b79ef7b523ba1d0a792f22868732e53a5dcf2f9e3b89eecca9c9b4e3"), + (Hash.SHA512, "896e45c82f9d8ba917d4f95891c967b88304b0a67ccc59aac813ee7ab3bc700bf9ce559e283c35ddba619755f6b70bdff2a07dc9cd337576a143a2aa361d08b1"), + (Hash.SHA3224, "37cb283bc9f6ecf0f94e92d5bd4c1e061ae00d7ed85804d18f981f53"), + (Hash.SHA3256, "d5fc146e37d4fddaeaa57aa88390be5c9ca6bcb18ae1bf2346cbfc36d3310ea2"), + (Hash.SHA3384, "ac97414589b2ef59a87dc5277d156b6cfc8f6b92b7c0e889d8f38a235dd9c1ba4030321beddd13f29519390ba914f70f"), + (Hash.SHA3512, "8dc580ad3abc6305ce5ada7c5920c763720c7733c2a94d28dd5351ffbc162b6b6d21371d91d6559124159025172e19896e09889047aac4ef555cc55456e14b0a"), + (Hash.SSDEEP, "3:AXGBicFlgVNhBGcL6wCrFQEv:AXGHsNhxLsr2C"), + (Hash.WHIRLPOOL, "b752b6eeb497a8bebfc1be1649ca41d57fd1973bffc2261ca196b5474e0f353762f354c1d743581f61c51f4d86921360bc2e8ad35e830578b68b12e884a50894"), + (Hash.TLSH, "6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8"), + ("foo", "bar"), # unrecognized hash type is accepted as-is +]) +def test_hash_check(hash_alg, hash_value): + assert check_hash(hash_alg, hash_value) + assert check_hash(hash_alg, hash_value.upper()) # check case sensitivity + + +def test_hash_check_fail(): + for hash_alg in Hash: + assert not check_hash(hash_alg, "x"*200) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index d760363..2f13e8e 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -8,8 +8,9 @@ from stix2.exceptions import ( ExtraPropertiesError, ParseError, ) from stix2.properties import ( - DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, IDProperty, - ListProperty, ObservableProperty, ReferenceProperty, STIXObjectProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, + HashesProperty, IDProperty, ListProperty, ObservableProperty, + ReferenceProperty, STIXObjectProperty, ) from stix2.v20.common import MarkingProperty @@ -353,6 +354,32 @@ def test_property_list_of_dictionary(): assert test_obj.property1[0]['foo'] == 'bar' +@pytest.mark.parametrize( + "key", [ + "aaa", + "a"*256, + "a-1_b", + ], +) +def test_hash_property_valid_key(key): + p = HashesProperty(["foo"], spec_version="2.0") + result = p.clean({key: "bar"}, True) + assert result == ({key: "bar"}, True) + + +@pytest.mark.parametrize( + "key", [ + "aa", + "a"*257, + "funny%chars?", + ], +) +def test_hash_property_invalid_key(key): + p = HashesProperty(["foo"], spec_version="2.0") + with pytest.raises(DictionaryKeyError): + p.clean({key: "foo"}, True) + + def test_embedded_property(): emb_prop = EmbeddedObjectProperty(type=stix2.v20.EmailMIMEComponent) mime = stix2.v20.EmailMIMEComponent( diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 7ccf6f2..60cbeb0 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -6,9 +6,9 @@ from stix2.exceptions import ( ExtraPropertiesError, ParseError, ) from stix2.properties import ( - DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, IDProperty, - ListProperty, ObservableProperty, ReferenceProperty, STIXObjectProperty, - StringProperty, + DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty, + HashesProperty, IDProperty, ListProperty, ObservableProperty, + ReferenceProperty, STIXObjectProperty, StringProperty, ) from stix2.v21.common import MarkingProperty @@ -367,6 +367,32 @@ def test_property_list_of_dictionary(): assert test_obj.property1[0]['foo'] == 'bar' +@pytest.mark.parametrize( + "key", [ + "a", + "a"*250, + "a-1_b", + ], +) +def test_hash_property_valid_key(key): + p = HashesProperty(["foo"], spec_version="2.1") + result = p.clean({key: "bar"}, True) + assert result == ({key: "bar"}, True) + + +@pytest.mark.parametrize( + "key", [ + "", + "a"*251, + "funny%chars?", + ], +) +def test_hash_property_invalid_key(key): + p = HashesProperty(["foo"], spec_version="2.1") + with pytest.raises(DictionaryKeyError): + p.clean({key: "foo"}, True) + + def test_embedded_property(): emb_prop = EmbeddedObjectProperty(type=stix2.v21.EmailMIMEComponent) mime = stix2.v21.EmailMIMEComponent( From c8c4e89415ef273841b5ddc6a9b46cbd1af58f0d Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Fri, 10 Jul 2020 16:57:22 -0400 Subject: [PATCH 10/21] Add back usage of spec_version where HashesProperty is used for various STIX objects. Also pre-commit stylistic fixes... --- stix2/test/test_hashes.py | 79 +++++++++++++++++++++------------------ stix2/v20/observables.py | 14 +++---- stix2/v21/common.py | 2 +- stix2/v21/observables.py | 14 +++---- 4 files changed, 57 insertions(+), 52 deletions(-) diff --git a/stix2/test/test_hashes.py b/stix2/test/test_hashes.py index f3445e1..40ace38 100644 --- a/stix2/test/test_hashes.py +++ b/stix2/test/test_hashes.py @@ -1,25 +1,28 @@ -from stix2.hashes import Hash, infer_hash_algorithm, check_hash import pytest +from stix2.hashes import Hash, check_hash, infer_hash_algorithm -@pytest.mark.parametrize("hash_name, expected_alg", [ - ("md5", Hash.MD5), - ("md6", Hash.MD6), - ("ripemd160", Hash.RIPEMD160), - ("sha1", Hash.SHA1), - ("sha224", Hash.SHA224), - ("sha256", Hash.SHA256), - ("sha384", Hash.SHA384), - ("sha512", Hash.SHA512), - ("sha3224", Hash.SHA3224), - ("sha3256", Hash.SHA3256), - ("sha3384", Hash.SHA3384), - ("sha3512", Hash.SHA3512), - ("ssdeep", Hash.SSDEEP), - ("whirlpool", Hash.WHIRLPOOL), - ("tlsh", Hash.TLSH), - ("xxxx", None), -]) + +@pytest.mark.parametrize( + "hash_name, expected_alg", [ + ("md5", Hash.MD5), + ("md6", Hash.MD6), + ("ripemd160", Hash.RIPEMD160), + ("sha1", Hash.SHA1), + ("sha224", Hash.SHA224), + ("sha256", Hash.SHA256), + ("sha384", Hash.SHA384), + ("sha512", Hash.SHA512), + ("sha3224", Hash.SHA3224), + ("sha3256", Hash.SHA3256), + ("sha3384", Hash.SHA3384), + ("sha3512", Hash.SHA3512), + ("ssdeep", Hash.SSDEEP), + ("whirlpool", Hash.WHIRLPOOL), + ("tlsh", Hash.TLSH), + ("xxxx", None), + ], +) def test_hash_inference(hash_name, expected_alg): alg = infer_hash_algorithm(hash_name) assert alg == expected_alg @@ -32,24 +35,26 @@ def test_hash_inference(hash_name, expected_alg): assert alg == expected_alg -@pytest.mark.parametrize("hash_alg, hash_value", [ - (Hash.MD5, "f9e40b9aa5464f3dae711ca524fceb63"), - (Hash.MD6, "f9e40b9aa5464f3dae711ca524fceb63"), - (Hash.RIPEMD160, "8ae5d2e6b1f3a514257f2469b637454931844aeb"), - (Hash.SHA1, "f2c7d4185880c0adcbb4a01d020a69498b16210e"), - (Hash.SHA224, "6743ed70cc26e750ad0108b6b8ad7fc2780c550f7d78adefa04dda05"), - (Hash.SHA256, "a2d1c2081aa932fe72307ab076b9739455bc7a21b3bed367bd9a86ae27af5a40"), - (Hash.SHA384, "bc846457de707f97bce93cca23b5ea58c0326fd8b79ef7b523ba1d0a792f22868732e53a5dcf2f9e3b89eecca9c9b4e3"), - (Hash.SHA512, "896e45c82f9d8ba917d4f95891c967b88304b0a67ccc59aac813ee7ab3bc700bf9ce559e283c35ddba619755f6b70bdff2a07dc9cd337576a143a2aa361d08b1"), - (Hash.SHA3224, "37cb283bc9f6ecf0f94e92d5bd4c1e061ae00d7ed85804d18f981f53"), - (Hash.SHA3256, "d5fc146e37d4fddaeaa57aa88390be5c9ca6bcb18ae1bf2346cbfc36d3310ea2"), - (Hash.SHA3384, "ac97414589b2ef59a87dc5277d156b6cfc8f6b92b7c0e889d8f38a235dd9c1ba4030321beddd13f29519390ba914f70f"), - (Hash.SHA3512, "8dc580ad3abc6305ce5ada7c5920c763720c7733c2a94d28dd5351ffbc162b6b6d21371d91d6559124159025172e19896e09889047aac4ef555cc55456e14b0a"), - (Hash.SSDEEP, "3:AXGBicFlgVNhBGcL6wCrFQEv:AXGHsNhxLsr2C"), - (Hash.WHIRLPOOL, "b752b6eeb497a8bebfc1be1649ca41d57fd1973bffc2261ca196b5474e0f353762f354c1d743581f61c51f4d86921360bc2e8ad35e830578b68b12e884a50894"), - (Hash.TLSH, "6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8"), - ("foo", "bar"), # unrecognized hash type is accepted as-is -]) +@pytest.mark.parametrize( + "hash_alg, hash_value", [ + (Hash.MD5, "f9e40b9aa5464f3dae711ca524fceb63"), + (Hash.MD6, "f9e40b9aa5464f3dae711ca524fceb63"), + (Hash.RIPEMD160, "8ae5d2e6b1f3a514257f2469b637454931844aeb"), + (Hash.SHA1, "f2c7d4185880c0adcbb4a01d020a69498b16210e"), + (Hash.SHA224, "6743ed70cc26e750ad0108b6b8ad7fc2780c550f7d78adefa04dda05"), + (Hash.SHA256, "a2d1c2081aa932fe72307ab076b9739455bc7a21b3bed367bd9a86ae27af5a40"), + (Hash.SHA384, "bc846457de707f97bce93cca23b5ea58c0326fd8b79ef7b523ba1d0a792f22868732e53a5dcf2f9e3b89eecca9c9b4e3"), + (Hash.SHA512, "896e45c82f9d8ba917d4f95891c967b88304b0a67ccc59aac813ee7ab3bc700bf9ce559e283c35ddba619755f6b70bdff2a07dc9cd337576a143a2aa361d08b1"), + (Hash.SHA3224, "37cb283bc9f6ecf0f94e92d5bd4c1e061ae00d7ed85804d18f981f53"), + (Hash.SHA3256, "d5fc146e37d4fddaeaa57aa88390be5c9ca6bcb18ae1bf2346cbfc36d3310ea2"), + (Hash.SHA3384, "ac97414589b2ef59a87dc5277d156b6cfc8f6b92b7c0e889d8f38a235dd9c1ba4030321beddd13f29519390ba914f70f"), + (Hash.SHA3512, "8dc580ad3abc6305ce5ada7c5920c763720c7733c2a94d28dd5351ffbc162b6b6d21371d91d6559124159025172e19896e09889047aac4ef555cc55456e14b0a"), + (Hash.SSDEEP, "3:AXGBicFlgVNhBGcL6wCrFQEv:AXGHsNhxLsr2C"), + (Hash.WHIRLPOOL, "b752b6eeb497a8bebfc1be1649ca41d57fd1973bffc2261ca196b5474e0f353762f354c1d743581f61c51f4d86921360bc2e8ad35e830578b68b12e884a50894"), + (Hash.TLSH, "6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8"), + ("foo", "bar"), # unrecognized hash type is accepted as-is + ], +) def test_hash_check(hash_alg, hash_value): assert check_hash(hash_alg, hash_value) assert check_hash(hash_alg, hash_value.upper()) # check case sensitivity diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 793b683..af445a4 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -31,7 +31,7 @@ class Artifact(_Observable): ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -174,7 +174,7 @@ class AlternateDataStream(_STIXBase20): _properties = OrderedDict([ ('name', StringProperty(required=True)), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ('size', IntegerProperty()), ]) @@ -257,7 +257,7 @@ class WindowsPEOptionalHeaderType(_STIXBase20): ('size_of_heap_commit', IntegerProperty()), ('loader_flags_hex', HexProperty()), ('number_of_rva_and_sizes', IntegerProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ]) def _check_object_constraints(self): @@ -274,7 +274,7 @@ class WindowsPESection(_STIXBase20): ('name', StringProperty(required=True)), ('size', IntegerProperty()), ('entropy', FloatProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ]) @@ -294,7 +294,7 @@ class WindowsPEBinaryExt(_Extension): ('number_of_symbols', IntegerProperty()), ('size_of_optional_header', IntegerProperty()), ('characteristics_hex', HexProperty()), - ('file_header_hashes', HashesProperty(HASHING_ALGORITHM)), + ('file_header_hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), ]) @@ -308,7 +308,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ('size', IntegerProperty()), ('name', StringProperty()), ('name_enc', StringProperty()), @@ -772,7 +772,7 @@ class X509Certificate(_Observable): _properties = OrderedDict([ ('type', TypeProperty(_type, spec_version='2.0')), ('is_self_signed', BooleanProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.0")), ('version', StringProperty()), ('serial_number', StringProperty()), ('signature_algorithm', StringProperty()), diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 49fdf87..0622ef1 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -25,7 +25,7 @@ class ExternalReference(_STIXBase21): ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('external_id', StringProperty()), ]) diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 9963d49..bdd0640 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -40,7 +40,7 @@ class Artifact(_Observable): ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('encryption_algorithm', EnumProperty(ENCRYPTION_ALGORITHM)), ('decryption_key', StringProperty()), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), @@ -219,7 +219,7 @@ class AlternateDataStream(_STIXBase21): _properties = OrderedDict([ ('name', StringProperty(required=True)), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('size', IntegerProperty()), ]) @@ -301,7 +301,7 @@ class WindowsPEOptionalHeaderType(_STIXBase21): ('size_of_heap_commit', IntegerProperty()), ('loader_flags_hex', HexProperty()), ('number_of_rva_and_sizes', IntegerProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ]) def _check_object_constraints(self): @@ -318,7 +318,7 @@ class WindowsPESection(_STIXBase21): ('name', StringProperty(required=True)), ('size', IntegerProperty(min=0)), ('entropy', FloatProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ]) @@ -338,7 +338,7 @@ class WindowsPEBinaryExt(_Extension): ('number_of_symbols', IntegerProperty(min=0)), ('size_of_optional_header', IntegerProperty(min=0)), ('characteristics_hex', HexProperty()), - ('file_header_hashes', HashesProperty(HASHING_ALGORITHM)), + ('file_header_hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), ]) @@ -354,7 +354,7 @@ class File(_Observable): ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('size', IntegerProperty(min=0)), ('name', StringProperty()), ('name_enc', StringProperty()), @@ -826,7 +826,7 @@ class X509Certificate(_Observable): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), - ('hashes', HashesProperty(HASHING_ALGORITHM)), + ('hashes', HashesProperty(HASHING_ALGORITHM, spec_version="2.1")), ('version', StringProperty()), ('serial_number', StringProperty()), ('signature_algorithm', StringProperty()), From 6f60bed235e73f0d77d6e6860af142086cf2ca25 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 16:21:35 -0400 Subject: [PATCH 11/21] Make a minor post-rebase fix --- stix2/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/properties.py b/stix2/properties.py index 4514379..52e37e9 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -422,7 +422,7 @@ class DictionaryProperty(Property): class HashesProperty(DictionaryProperty): - def __init__(self, spec_hash_names, spec_version=stix2.DEFAULT_VERSION, **kwargs): + def __init__(self, spec_hash_names, spec_version=DEFAULT_VERSION, **kwargs): super().__init__(spec_version=spec_version, **kwargs) self.__spec_hash_names = spec_hash_names From c8b9eab53d7f78802a0c854ea67272e23482d4c0 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 16:49:14 -0400 Subject: [PATCH 12/21] Fixed some more silliness in properties.py... was it a rebase glitch? Something else I overlooked? Not sure. --- stix2/properties.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 52e37e9..55c5625 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -757,12 +757,11 @@ class ObservableProperty(Property): has_custom = True if not allow_custom and has_custom: - if parsed_obj.has_custom: - raise CustomContentError( - "customized {} observable found".format( - parsed_obj["type"], - ), - ) + raise CustomContentError( + "customized {} observable found".format( + parsed_obj["type"], + ), + ) dictified[key] = parsed_obj From 0ddf0d2ca8035035bd4ce34d3f446df6726e61a6 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 20:01:27 -0400 Subject: [PATCH 13/21] Clean up ReferenceProperty.clean(), to use new APIs which are available after the rebase. This simplifies the implementation. Also made utils.to_enum() a module public function, since I needed to use it outside that module. Misc pre-commit stylistic fixes. --- stix2/properties.py | 73 +++++++++------------- stix2/test/v21/test_location.py | 3 +- stix2/test/v21/test_timestamp_precision.py | 8 +-- stix2/utils.py | 10 +-- 4 files changed, 40 insertions(+), 54 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 55c5625..ba308c8 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -14,7 +14,10 @@ from .base import _STIXBase from .exceptions import CustomContentError, DictionaryKeyError, STIXError from .parsing import parse, parse_observable from .registry import STIX2_OBJ_MAPS -from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime +from .utils import ( + STIXTypeClass, _get_dict, get_class_hierarchy_names, get_type_from_id, + is_object, is_stix_type, parse_into_datetime, to_enum, +) from .version import DEFAULT_VERSION try: @@ -501,7 +504,6 @@ class HexProperty(Property): class ReferenceProperty(Property): - _OBJECT_CATEGORIES = {"SDO", "SCO", "SRO"} _WHITELIST, _BLACKLIST = range(2) def __init__(self, valid_types=None, invalid_types=None, spec_version=DEFAULT_VERSION, **kwargs): @@ -525,9 +527,22 @@ class ReferenceProperty(Property): if valid_types is not None and len(valid_types) == 0: raise ValueError("Impossible type constraint: empty whitelist") - self.types = set(valid_types or invalid_types) self.auth_type = self._WHITELIST if valid_types else self._BLACKLIST + # Divide type requirements into generic type classes and specific + # types. With respect to strings, values recognized as STIXTypeClass + # enum names are generic; all else are specifics. + self.generics = set() + self.specifics = set() + types = valid_types or invalid_types + for type_ in types: + try: + enum_value = to_enum(type_, STIXTypeClass) + except KeyError: + self.specifics.add(type_) + else: + self.generics.add(enum_value) + super(ReferenceProperty, self).__init__(**kwargs) def clean(self, value, allow_custom): @@ -537,7 +552,7 @@ class ReferenceProperty(Property): _validate_id(value, self.spec_version, None) - obj_type = value[:value.index('--')] + obj_type = get_type_from_id(value) # Only comes into play when inverting a hybrid whitelist. # E.g. if the possible generic categories are A, B, C, then the @@ -548,8 +563,8 @@ class ReferenceProperty(Property): # blacklist. blacklist_exceptions = set() - generics = self.types & self._OBJECT_CATEGORIES - specifics = self.types - generics + generics = self.generics + specifics = self.specifics auth_type = self.auth_type if allow_custom and auth_type == self._WHITELIST and generics: # If allowing customization and using a whitelist, and if generic @@ -560,20 +575,19 @@ class ReferenceProperty(Property): # in the wrong category. I.e. flip the whitelist set to a # blacklist of a complementary set. auth_type = self._BLACKLIST - generics = self._OBJECT_CATEGORIES - generics + generics = set(STIXTypeClass) - generics blacklist_exceptions, specifics = specifics, blacklist_exceptions if auth_type == self._WHITELIST: - type_ok = _type_in_generic_set( - obj_type, generics, self.spec_version + type_ok = is_stix_type( + obj_type, self.spec_version, *generics ) or obj_type in specifics else: type_ok = ( - not _type_in_generic_set( - obj_type, generics, self.spec_version, - ) - and obj_type not in specifics + not is_stix_type( + obj_type, self.spec_version, *generics + ) and obj_type not in specifics ) or obj_type in blacklist_exceptions if not type_ok: @@ -585,9 +599,8 @@ class ReferenceProperty(Property): # We need to figure out whether the referenced object is custom or # not. No good way to do that at present... just check if # unregistered and for the "x-" type prefix, for now? - has_custom = not _type_in_generic_set( - obj_type, self._OBJECT_CATEGORIES, self.spec_version, - ) or obj_type.startswith("x-") + has_custom = not is_object(obj_type, self.spec_version) \ + or obj_type.startswith("x-") if not allow_custom and has_custom: raise CustomContentError( @@ -597,34 +610,6 @@ class ReferenceProperty(Property): return value, has_custom -def _type_in_generic_set(type_, type_set, spec_version): - """ - Determine if type_ is in the given set, with respect to the given STIX - version. This handles special generic category values "SDO", "SCO", - "SRO", so it's not a simple set containment check. The type_set is - implicitly "OR"d. - """ - type_maps = STIX2_OBJ_MAPS[spec_version] - - result = False - for type_id in type_set: - if type_id == "SDO": - result = type_ in type_maps["objects"] and type_ not in [ - "relationship", "sighting", - ] # sigh - elif type_id == "SCO": - result = type_ in type_maps["observables"] - elif type_id == "SRO": - result = type_ in ["relationship", "sighting"] - else: - raise ValueError("Unrecognized generic type category: " + type_id) - - if result: - break - - return result - - SELECTOR_REGEX = re.compile(r"^([a-z0-9_-]{3,250}(\.(\[\d+\]|[a-z0-9_-]{1,250}))*|id)$") diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index e912fba..9f42255 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -47,7 +47,8 @@ EXPECTED_LOCATION_2_REPR = "Location(" + " ".join( id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64', created='2016-04-06T20:03:00.000Z', modified='2016-04-06T20:03:00.000Z', - region='northern-america'""".split()) + ")" + region='northern-america'""".split() +) + ")" def test_location_with_some_required_properties(): diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index 8cb9735..831bd7a 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -5,8 +5,8 @@ import pytest import stix2 from stix2.utils import ( - Precision, PrecisionConstraint, STIXdatetime, _to_enum, format_datetime, - parse_into_datetime, + Precision, PrecisionConstraint, STIXdatetime, format_datetime, + parse_into_datetime, to_enum, ) _DT = datetime.datetime.utcnow() @@ -27,7 +27,7 @@ _DT_STR = _DT.strftime("%Y-%m-%dT%H:%M:%S") ], ) def test_to_enum(value, enum_type, enum_default, enum_expected): - result = _to_enum(value, enum_type, enum_default) + result = to_enum(value, enum_type, enum_default) assert result == enum_expected @@ -41,7 +41,7 @@ def test_to_enum(value, enum_type, enum_default, enum_expected): ) def test_to_enum_errors(value, err_type): with pytest.raises(err_type): - _to_enum(value, Precision) + to_enum(value, Precision) @pytest.mark.xfail( diff --git a/stix2/utils.py b/stix2/utils.py index 08e272d..647a89f 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -45,7 +45,7 @@ class PrecisionConstraint(enum.Enum): # no need for a MAX constraint yet -def _to_enum(value, enum_type, enum_default=None): +def to_enum(value, enum_type, enum_default=None): """ Detect and convert strings to enums and None to a default enum. This allows use of strings and None in APIs, while enforcing the enum type: if @@ -88,11 +88,11 @@ class STIXdatetime(dt.datetime): """ def __new__(cls, *args, **kwargs): - precision = _to_enum( + precision = to_enum( kwargs.pop("precision", Precision.ANY), Precision, ) - precision_constraint = _to_enum( + precision_constraint = to_enum( kwargs.pop("precision_constraint", PrecisionConstraint.EXACT), PrecisionConstraint, ) @@ -233,8 +233,8 @@ def parse_into_datetime( :return: A STIXdatetime instance, which is a datetime but also carries the precision info necessary to properly JSON-serialize it. """ - precision = _to_enum(precision, Precision) - precision_constraint = _to_enum(precision_constraint, PrecisionConstraint) + precision = to_enum(precision, Precision) + precision_constraint = to_enum(precision_constraint, PrecisionConstraint) if isinstance(value, dt.date): if hasattr(value, 'hour'): From e735b537c7f3f90b49829c90f1ed9fa86b000b23 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 20:56:31 -0400 Subject: [PATCH 14/21] Remove a workaround which was used in the 2.0 and 2.1 Report objects, to structure object_refs type requirements as an empty blacklist, instead of a whitelist. I think it was originally necessary due to the older implementation of ReferenceProperty which was in place at the time. With the implementation change to invert whitelists with generics to blacklists when allow_custom=True, a "full" whitelist is internally converted to an empty blacklist anyway, so it winds up being the same thing. But I think the full whitelist looks better in the code, so I prefer that to the empty blacklist. --- stix2/v20/sdo.py | 2 +- stix2/v21/sdo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index d08063e..8565dcb 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -240,7 +240,7 @@ class Report(_DomainObject): ('name', StringProperty(required=True)), ('description', StringProperty()), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.0'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.0'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(OpenVocabProperty(REPORT_LABEL), required=True)), ('external_references', ListProperty(ExternalReference)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index bba531e..da90542 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -644,7 +644,7 @@ class Report(_DomainObject): ('description', StringProperty()), ('report_types', ListProperty(OpenVocabProperty(REPORT_TYPE))), ('published', TimestampProperty(required=True)), - ('object_refs', ListProperty(ReferenceProperty(invalid_types=[], spec_version='2.1'), required=True)), + ('object_refs', ListProperty(ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1'), required=True)), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), From 82148aa1ac73293638514f4ceef6c21b6f45b701 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 21:27:06 -0400 Subject: [PATCH 15/21] Pre-commit stylistic fix --- stix2/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/properties.py b/stix2/properties.py index ba308c8..e977240 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -600,7 +600,7 @@ class ReferenceProperty(Property): # not. No good way to do that at present... just check if # unregistered and for the "x-" type prefix, for now? has_custom = not is_object(obj_type, self.spec_version) \ - or obj_type.startswith("x-") + or obj_type.startswith("x-") if not allow_custom and has_custom: raise CustomContentError( From 33128908c749083ea02ff1b6038fd3e4446ccdbc Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 31 Mar 2021 21:31:59 -0400 Subject: [PATCH 16/21] Another pre-commit stylistic fix. Necessary since apparently the git action uses a more recent version of pre-commit than travis did, and the newer pre-commit requires different things than the old one did. --- stix2/test/v21/test_location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_location.py b/stix2/test/v21/test_location.py index 9f42255..e7da4f2 100644 --- a/stix2/test/v21/test_location.py +++ b/stix2/test/v21/test_location.py @@ -47,7 +47,7 @@ EXPECTED_LOCATION_2_REPR = "Location(" + " ".join( id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64', created='2016-04-06T20:03:00.000Z', modified='2016-04-06T20:03:00.000Z', - region='northern-america'""".split() + region='northern-america'""".split(), ) + ")" From f301bc144f9d3b425479a724723df4697a6a6e83 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 14 Apr 2021 12:48:32 -0400 Subject: [PATCH 17/21] Disable custom vocab enforcement. Custom values for open-vocab properties are now always accepted and not considered customizations (has_custom will be False). Just commented out the enforcement code and xfail'd the unit tests, which makes it easy to reinstate these behaviors later. It was decided this is too likely to break user code, and we don't currently have a way to disallow customizations in other places while still allowing custom open vocab values. Safest to always allow custom open vocab values, for now. --- stix2/properties.py | 16 +++++++++++----- stix2/test/test_properties.py | 4 ++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index e977240..b67f627 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -690,12 +690,18 @@ class OpenVocabProperty(StringProperty): value, allow_custom, ) - has_custom = cleaned_value not in self.allowed + # Disabled: it was decided that enforcing this is too strict (might + # break too much user code). Revisit when we have the capability for + # more granular config settings when creating objects. + # + # has_custom = cleaned_value not in self.allowed + # + # if not allow_custom and has_custom: + # raise CustomContentError( + # "custom value in open vocab: '{}'".format(cleaned_value), + # ) - if not allow_custom and has_custom: - raise CustomContentError( - "custom value in open vocab: '{}'".format(cleaned_value), - ) + has_custom = False return cleaned_value, has_custom diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index f19d1b5..d37319c 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -368,6 +368,10 @@ def test_enum_property_invalid(): enum_prop.clean('z', True) +@pytest.mark.xfail( + reason="Temporarily disabled custom open vocab enforcement", + strict=True +) @pytest.mark.parametrize( "vocab", [ ['a', 'b', 'c'], From 4dc8e49d3ca53b3db866fee4b411245fb57471bc Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 14 Apr 2021 12:58:09 -0400 Subject: [PATCH 18/21] pre-commit stylistic fix --- stix2/test/test_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index d37319c..5116a68 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -370,7 +370,7 @@ def test_enum_property_invalid(): @pytest.mark.xfail( reason="Temporarily disabled custom open vocab enforcement", - strict=True + strict=True, ) @pytest.mark.parametrize( "vocab", [ From 2b064872a437319c20de281d78b4881eff1299a2 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 15 Apr 2021 22:06:57 -0400 Subject: [PATCH 19/21] Add variables for each individual enum value in the vocab modules. --- stix2/v20/vocab.py | 408 +++++++++++------ stix2/v21/vocab.py | 1041 ++++++++++++++++++++++++++++++-------------- 2 files changed, 995 insertions(+), 454 deletions(-) diff --git a/stix2/v20/vocab.py b/stix2/v20/vocab.py index b0f9a70..765c796 100644 --- a/stix2/v20/vocab.py +++ b/stix2/v20/vocab.py @@ -2,177 +2,329 @@ STIX 2.0 open vocabularies and enums """ +ATTACK_MOTIVATION_ACCIDENTAL = "accidental" +ATTACK_MOTIVATION_COERCION = "coercion" +ATTACK_MOTIVATION_DOMINANCE = "dominance" +ATTACK_MOTIVATION_IDEOLOGY = "ideology" +ATTACK_MOTIVATION_NOTORIETY = "notoriety" +ATTACK_MOTIVATION_ORGANIZATIONAL_GAIN = "organizational-gain" +ATTACK_MOTIVATION_PERSONAL_GAIN = "personal-gain" +ATTACK_MOTIVATION_PERSONAL_SATISFACTION = "personal-satisfaction" +ATTACK_MOTIVATION_REVENGE = "revenge" +ATTACK_MOTIVATION_UNPREDICTABLE = "unpredictable" + + ATTACK_MOTIVATION = [ - "accidental", - "coercion", - "dominance", - "ideology", - "notoriety", - "organizational-gain", - "personal-gain", - "personal-satisfaction", - "revenge", - "unpredictable", + ATTACK_MOTIVATION_ACCIDENTAL, + ATTACK_MOTIVATION_COERCION, + ATTACK_MOTIVATION_DOMINANCE, + ATTACK_MOTIVATION_IDEOLOGY, + ATTACK_MOTIVATION_NOTORIETY, + ATTACK_MOTIVATION_ORGANIZATIONAL_GAIN, + ATTACK_MOTIVATION_PERSONAL_GAIN, + ATTACK_MOTIVATION_PERSONAL_SATISFACTION, + ATTACK_MOTIVATION_REVENGE, + ATTACK_MOTIVATION_UNPREDICTABLE, ] +ATTACK_RESOURCE_LEVEL_INDIVIDUAL = "individual" +ATTACK_RESOURCE_LEVEL_CLUB = "club" +ATTACK_RESOURCE_LEVEL_CONTEST = "contest" +ATTACK_RESOURCE_LEVEL_TEAM = "team" +ATTACK_RESOURCE_LEVEL_ORGANIZATION = "organization" +ATTACK_RESOURCE_LEVEL_GOVERNMENT = "government" + + ATTACK_RESOURCE_LEVEL = [ - "individual", - "club", - "contest", - "team", - "organization", - "government", + ATTACK_RESOURCE_LEVEL_INDIVIDUAL, + ATTACK_RESOURCE_LEVEL_CLUB, + ATTACK_RESOURCE_LEVEL_CONTEST, + ATTACK_RESOURCE_LEVEL_TEAM, + ATTACK_RESOURCE_LEVEL_ORGANIZATION, + ATTACK_RESOURCE_LEVEL_GOVERNMENT, ] +HASHING_ALGORITHM_MD5 = "MD5" +HASHING_ALGORITHM_MD6 = "MD6" +HASHING_ALGORITHM_RIPEMD_160 = "RIPEMD-160" +HASHING_ALGORITHM_SHA_1 = "SHA-1" +HASHING_ALGORITHM_SHA_224 = "SHA-224" +HASHING_ALGORITHM_SHA_256 = "SHA-256" +HASHING_ALGORITHM_SHA_384 = "SHA-384" +HASHING_ALGORITHM_SHA_512 = "SHA-512" +HASHING_ALGORITHM_SHA3_224 = "SHA3-224" +HASHING_ALGORITHM_SHA3_256 = "SHA3-256" +HASHING_ALGORITHM_SHA3_384 = "SHA3-384" +HASHING_ALGORITHM_SHA3_512 = "SHA3-512" +HASHING_ALGORITHM_SSDEEP = "ssdeep" +HASHING_ALGORITHM_WHIRLPOOL = "WHIRLPOOL" + + HASHING_ALGORITHM = [ - "MD5", - "MD6", - "RIPEMD-160", - "SHA-1", - "SHA-224", - "SHA-256", - "SHA-384", - "SHA-512", - "SHA3-224", - "SHA3-256", - "SHA3-384", - "SHA3-512", - "ssdeep", - "WHIRLPOOL", + HASHING_ALGORITHM_MD5, + HASHING_ALGORITHM_MD6, + HASHING_ALGORITHM_RIPEMD_160, + HASHING_ALGORITHM_SHA_1, + HASHING_ALGORITHM_SHA_224, + HASHING_ALGORITHM_SHA_256, + HASHING_ALGORITHM_SHA_384, + HASHING_ALGORITHM_SHA_512, + HASHING_ALGORITHM_SHA3_224, + HASHING_ALGORITHM_SHA3_256, + HASHING_ALGORITHM_SHA3_384, + HASHING_ALGORITHM_SHA3_512, + HASHING_ALGORITHM_SSDEEP, + HASHING_ALGORITHM_WHIRLPOOL, ] +IDENTITY_CLASS_INDIVIDUAL = "individual" +IDENTITY_CLASS_GROUP = "group" +IDENTITY_CLASS_ORGANIZATION = "organization" +IDENTITY_CLASS_CLASS = "class" +IDENTITY_CLASS_UNKNOWN = "unknown" + + IDENTITY_CLASS = [ - "individual", - "group", - "organization", - "class", - "unknown", + IDENTITY_CLASS_INDIVIDUAL, + IDENTITY_CLASS_GROUP, + IDENTITY_CLASS_ORGANIZATION, + IDENTITY_CLASS_CLASS, + IDENTITY_CLASS_UNKNOWN, ] +INDICATOR_LABEL_ANOMALOUS_ACTIVITY = "anomalous-activity" +INDICATOR_LABEL_ANONYMIZATION = "anonymization" +INDICATOR_LABEL_BENIGN = "benign" +INDICATOR_LABEL_COMPROMISED = "compromised" +INDICATOR_LABEL_MALICIOUS_ACTIVITY = "malicious-activity" +INDICATOR_LABEL_ATTRIBUTION = "attribution" + + INDICATOR_LABEL = [ - "anomalous-activity", - "anonymization", - "benign", - "compromised", - "malicious-activity", - "attribution", + INDICATOR_LABEL_ANOMALOUS_ACTIVITY, + INDICATOR_LABEL_ANONYMIZATION, + INDICATOR_LABEL_BENIGN, + INDICATOR_LABEL_COMPROMISED, + INDICATOR_LABEL_MALICIOUS_ACTIVITY, + INDICATOR_LABEL_ATTRIBUTION, ] +INDUSTRY_SECTOR_AGRICULTURE = "agriculture" +INDUSTRY_SECTOR_AEROSPACE = "aerospace" +INDUSTRY_SECTOR_AUTOMOTIVE = "automotive" +INDUSTRY_SECTOR_COMMUNICATIONS = "communications" +INDUSTRY_SECTOR_CONSTRUCTION = "construction" +INDUSTRY_SECTOR_DEFENCE = "defence" +INDUSTRY_SECTOR_EDUCATION = "education" +INDUSTRY_SECTOR_ENERGY = "energy" +INDUSTRY_SECTOR_ENTERTAINMENT = "entertainment" +INDUSTRY_SECTOR_FINANCIAL_SERVICES = "financial-services" +INDUSTRY_SECTOR_GOVERNMENT_NATIONAL = "government-national" +INDUSTRY_SECTOR_GOVERNMENT_REGIONAL = "government-regional" +INDUSTRY_SECTOR_GOVERNMENT_LOCAL = "government-local" +INDUSTRY_SECTOR_GOVERNMENT_PUBLIC_SERVICES = "government-public-services" +INDUSTRY_SECTOR_HEALTHCARE = "healthcare" +INDUSTRY_SECTOR_HOSPITALITY_LEISURE = "hospitality-leisure" +INDUSTRY_SECTOR_INFRASTRUCTURE = "infrastructure" +INDUSTRY_SECTOR_INSURANCE = "insurance" +INDUSTRY_SECTOR_MANUFACTURING = "manufacturing" +INDUSTRY_SECTOR_MINING = "mining" +INDUSTRY_SECTOR_NON_PROFIT = "non-profit" +INDUSTRY_SECTOR_PHARMACEUTICALS = "pharmaceuticals" +INDUSTRY_SECTOR_RETAIL = "retail" +INDUSTRY_SECTOR_TECHNOLOGY = "technology" +INDUSTRY_SECTOR_TELECOMMUNICATIONS = "telecommunications" +INDUSTRY_SECTOR_TRANSPORTATION = "transportation" +INDUSTRY_SECTOR_UTILITIES = "utilities" + + INDUSTRY_SECTOR = [ - "agriculture", - "aerospace", - "automotive", - "communications", - "construction", - "defence", - "education", - "energy", - "entertainment", - "financial-services", - "government-national", - "government-regional", - "government-local", - "government-public-services", - "healthcare", - "hospitality-leisure", - "infrastructure", - "insurance", - "manufacturing", - "mining", - "non-profit", - "pharmaceuticals", - "retail", - "technology", - "telecommunications", - "transportation", - "utilities", + INDUSTRY_SECTOR_AGRICULTURE, + INDUSTRY_SECTOR_AEROSPACE, + INDUSTRY_SECTOR_AUTOMOTIVE, + INDUSTRY_SECTOR_COMMUNICATIONS, + INDUSTRY_SECTOR_CONSTRUCTION, + INDUSTRY_SECTOR_DEFENCE, + INDUSTRY_SECTOR_EDUCATION, + INDUSTRY_SECTOR_ENERGY, + INDUSTRY_SECTOR_ENTERTAINMENT, + INDUSTRY_SECTOR_FINANCIAL_SERVICES, + INDUSTRY_SECTOR_GOVERNMENT_NATIONAL, + INDUSTRY_SECTOR_GOVERNMENT_REGIONAL, + INDUSTRY_SECTOR_GOVERNMENT_LOCAL, + INDUSTRY_SECTOR_GOVERNMENT_PUBLIC_SERVICES, + INDUSTRY_SECTOR_HEALTHCARE, + INDUSTRY_SECTOR_HOSPITALITY_LEISURE, + INDUSTRY_SECTOR_INFRASTRUCTURE, + INDUSTRY_SECTOR_INSURANCE, + INDUSTRY_SECTOR_MANUFACTURING, + INDUSTRY_SECTOR_MINING, + INDUSTRY_SECTOR_NON_PROFIT, + INDUSTRY_SECTOR_PHARMACEUTICALS, + INDUSTRY_SECTOR_RETAIL, + INDUSTRY_SECTOR_TECHNOLOGY, + INDUSTRY_SECTOR_TELECOMMUNICATIONS, + INDUSTRY_SECTOR_TRANSPORTATION, + INDUSTRY_SECTOR_UTILITIES, ] +MALWARE_LABEL_ADWARE = "adware" +MALWARE_LABEL_BACKDOOR = "backdoor" +MALWARE_LABEL_BOT = "bot" +MALWARE_LABEL_DDOS = "ddos" +MALWARE_LABEL_DROPPER = "dropper" +MALWARE_LABEL_EXPLOIT_KIT = "exploit-kit" +MALWARE_LABEL_KEYLOGGER = "keylogger" +MALWARE_LABEL_RANSOMWARE = "ransomware" +MALWARE_LABEL_REMOTE_ACCESS_TROJAN = "remote-access-trojan" +MALWARE_LABEL_RESOURCE_EXPLOITATION = "resource-exploitation" +MALWARE_LABEL_ROGUE_SECURITY_SOFTWARE = "rogue-security-software" +MALWARE_LABEL_ROOTKIT = "rootkit" +MALWARE_LABEL_SCREEN_CAPTURE = "screen-capture" +MALWARE_LABEL_SPYWARE = "spyware" +MALWARE_LABEL_TROJAN = "trojan" +MALWARE_LABEL_VIRUS = "virus" +MALWARE_LABEL_WORM = "worm" + + MALWARE_LABEL = [ - "adware", - "backdoor", - "bot", - "ddos", - "dropper", - "exploit-kit", - "keylogger", - "ransomware", - "remote-access-trojan", - "resource-exploitation", - "rogue-security-software", - "rootkit", - "screen-capture", - "spyware", - "trojan", - "virus", - "worm", + MALWARE_LABEL_ADWARE, + MALWARE_LABEL_BACKDOOR, + MALWARE_LABEL_BOT, + MALWARE_LABEL_DDOS, + MALWARE_LABEL_DROPPER, + MALWARE_LABEL_EXPLOIT_KIT, + MALWARE_LABEL_KEYLOGGER, + MALWARE_LABEL_RANSOMWARE, + MALWARE_LABEL_REMOTE_ACCESS_TROJAN, + MALWARE_LABEL_RESOURCE_EXPLOITATION, + MALWARE_LABEL_ROGUE_SECURITY_SOFTWARE, + MALWARE_LABEL_ROOTKIT, + MALWARE_LABEL_SCREEN_CAPTURE, + MALWARE_LABEL_SPYWARE, + MALWARE_LABEL_TROJAN, + MALWARE_LABEL_VIRUS, + MALWARE_LABEL_WORM, ] +REPORT_LABEL_THREAT_REPORT = "threat-report" +REPORT_LABEL_ATTACK_PATTERN = "attack-pattern" +REPORT_LABEL_CAMPAIGN = "campaign" +REPORT_LABEL_IDENTITY = "identity" +REPORT_LABEL_INDICATOR = "indicator" +REPORT_LABEL_INTRUSION_SET = "intrusion-set" +REPORT_LABEL_MALWARE = "malware" +REPORT_LABEL_OBSERVED_DATA = "observed-data" +REPORT_LABEL_THREAT_ACTOR = "threat-actor" +REPORT_LABEL_TOOL = "tool" +REPORT_LABEL_VULNERABILITY = "vulnerability" + + REPORT_LABEL = [ - "threat-report", - "attack-pattern", - "campaign", - "identity", - "indicator", - "intrusion-set", - "malware", - "observed-data", - "threat-actor", - "tool", - "vulnerability", + REPORT_LABEL_THREAT_REPORT, + REPORT_LABEL_ATTACK_PATTERN, + REPORT_LABEL_CAMPAIGN, + REPORT_LABEL_IDENTITY, + REPORT_LABEL_INDICATOR, + REPORT_LABEL_INTRUSION_SET, + REPORT_LABEL_MALWARE, + REPORT_LABEL_OBSERVED_DATA, + REPORT_LABEL_THREAT_ACTOR, + REPORT_LABEL_TOOL, + REPORT_LABEL_VULNERABILITY, ] +THREAT_ACTOR_LABEL_ACTIVIST = "activist" +THREAT_ACTOR_LABEL_COMPETITOR = "competitor" +THREAT_ACTOR_LABEL_CRIME_SYNDICATE = "crime-syndicate" +THREAT_ACTOR_LABEL_CRIMINAL = "criminal" +THREAT_ACTOR_LABEL_HACKER = "hacker" +THREAT_ACTOR_LABEL_INSIDER_ACCIDENTAL = "insider-accidental" +THREAT_ACTOR_LABEL_INSIDER_DISGRUNTLED = "insider-disgruntled" +THREAT_ACTOR_LABEL_NATION_STATE = "nation-state" +THREAT_ACTOR_LABEL_SENSATIONALIST = "sensationalist" +THREAT_ACTOR_LABEL_SPY = "spy" +THREAT_ACTOR_LABEL_TERRORIST = "terrorist" + + THREAT_ACTOR_LABEL = [ - "activist", - "competitor", - "crime-syndicate", - "criminal", - "hacker", - "insider-accidental", - "insider-disgruntled", - "nation-state", - "sensationalist", - "spy", - "terrorist", + THREAT_ACTOR_LABEL_ACTIVIST, + THREAT_ACTOR_LABEL_COMPETITOR, + THREAT_ACTOR_LABEL_CRIME_SYNDICATE, + THREAT_ACTOR_LABEL_CRIMINAL, + THREAT_ACTOR_LABEL_HACKER, + THREAT_ACTOR_LABEL_INSIDER_ACCIDENTAL, + THREAT_ACTOR_LABEL_INSIDER_DISGRUNTLED, + THREAT_ACTOR_LABEL_NATION_STATE, + THREAT_ACTOR_LABEL_SENSATIONALIST, + THREAT_ACTOR_LABEL_SPY, + THREAT_ACTOR_LABEL_TERRORIST, ] +THREAT_ACTOR_ROLE_AGENT = "agent" +THREAT_ACTOR_ROLE_DIRECTOR = "director" +THREAT_ACTOR_ROLE_INDEPENDENT = "independent" +THREAT_ACTOR_ROLE_INFRASTRUCTURE_ARCHITECT = "infrastructure-architect" +THREAT_ACTOR_ROLE_INFRASTRUCTURE_OPERATOR = "infrastructure-operator" +THREAT_ACTOR_ROLE_MALWARE_AUTHOR = "malware-author" +THREAT_ACTOR_ROLE_SPONSOR = "sponsor" + + THREAT_ACTOR_ROLE = [ - "agent", - "director", - "independent", - "infrastructure-architect", - "infrastructure-operator", - "malware-author", - "sponsor", + THREAT_ACTOR_ROLE_AGENT, + THREAT_ACTOR_ROLE_DIRECTOR, + THREAT_ACTOR_ROLE_INDEPENDENT, + THREAT_ACTOR_ROLE_INFRASTRUCTURE_ARCHITECT, + THREAT_ACTOR_ROLE_INFRASTRUCTURE_OPERATOR, + THREAT_ACTOR_ROLE_MALWARE_AUTHOR, + THREAT_ACTOR_ROLE_SPONSOR, ] +THREAT_ACTOR_SOPHISTICATION_NONE = "none" +THREAT_ACTOR_SOPHISTICATION_MINIMAL = "minimal" +THREAT_ACTOR_SOPHISTICATION_INTERMEDIATE = "intermediate" +THREAT_ACTOR_SOPHISTICATION_ADVANCED = "advanced" +THREAT_ACTOR_SOPHISTICATION_EXPERT = "expert" +THREAT_ACTOR_SOPHISTICATION_INNOVATOR = "innovator" +THREAT_ACTOR_SOPHISTICATION_STRATEGIC = "strategic" + + THREAT_ACTOR_SOPHISTICATION = [ - "none", - "minimal", - "intermediate", - "advanced", - "expert", - "innovator", - "strategic", + THREAT_ACTOR_SOPHISTICATION_NONE, + THREAT_ACTOR_SOPHISTICATION_MINIMAL, + THREAT_ACTOR_SOPHISTICATION_INTERMEDIATE, + THREAT_ACTOR_SOPHISTICATION_ADVANCED, + THREAT_ACTOR_SOPHISTICATION_EXPERT, + THREAT_ACTOR_SOPHISTICATION_INNOVATOR, + THREAT_ACTOR_SOPHISTICATION_STRATEGIC, ] +TOOL_LABEL_DENIAL_OF_SERVICE = "denial-of-service" +TOOL_LABEL_EXPLOITATION = "exploitation" +TOOL_LABEL_INFORMATION_GATHERING = "information-gathering" +TOOL_LABEL_NETWORK_CAPTURE = "network-capture" +TOOL_LABEL_CREDENTIAL_EXPLOITATION = "credential-exploitation" +TOOL_LABEL_REMOTE_ACCESS = "remote-access" +TOOL_LABEL_VULNERABILITY_SCANNING = "vulnerability-scanning" + + TOOL_LABEL = [ - "denial-of-service", - "exploitation", - "information-gathering", - "network-capture", - "credential-exploitation", - "remote-access", - "vulnerability-scanning", + TOOL_LABEL_DENIAL_OF_SERVICE, + TOOL_LABEL_EXPLOITATION, + TOOL_LABEL_INFORMATION_GATHERING, + TOOL_LABEL_NETWORK_CAPTURE, + TOOL_LABEL_CREDENTIAL_EXPLOITATION, + TOOL_LABEL_REMOTE_ACCESS, + TOOL_LABEL_VULNERABILITY_SCANNING, ] diff --git a/stix2/v21/vocab.py b/stix2/v21/vocab.py index c18c320..dc790d7 100644 --- a/stix2/v21/vocab.py +++ b/stix2/v21/vocab.py @@ -2,451 +2,840 @@ STIX 2.1 open vocabularies and enums """ + +ACCOUNT_TYPE_FACEBOOK = "facebook" +ACCOUNT_TYPE_LDAP = "ldap" +ACCOUNT_TYPE_NIS = "nis" +ACCOUNT_TYPE_OPENID = "openid" +ACCOUNT_TYPE_RADIUS = "radius" +ACCOUNT_TYPE_SKYPE = "skype" +ACCOUNT_TYPE_TACACS = "tacacs" +ACCOUNT_TYPE_TWITTER = "twitter" +ACCOUNT_TYPE_UNIX = "unix" +ACCOUNT_TYPE_WINDOWS_LOCAL = "windows-local" +ACCOUNT_TYPE_WINDOWS_DOMAIN = "windows-domain" + + ACCOUNT_TYPE = [ - "facebook", - "ldap", - "nis", - "openid", - "radius", - "skype", - "tacacs", - "twitter", - "unix", - "windows-local", - "windows-domain", + ACCOUNT_TYPE_FACEBOOK, + ACCOUNT_TYPE_LDAP, + ACCOUNT_TYPE_NIS, + ACCOUNT_TYPE_OPENID, + ACCOUNT_TYPE_RADIUS, + ACCOUNT_TYPE_SKYPE, + ACCOUNT_TYPE_TACACS, + ACCOUNT_TYPE_TWITTER, + ACCOUNT_TYPE_UNIX, + ACCOUNT_TYPE_WINDOWS_LOCAL, + ACCOUNT_TYPE_WINDOWS_DOMAIN, ] +ATTACK_MOTIVATION_ACCIDENTAL = "accidental" +ATTACK_MOTIVATION_COERCION = "coercion" +ATTACK_MOTIVATION_DOMINANCE = "dominance" +ATTACK_MOTIVATION_IDEOLOGY = "ideology" +ATTACK_MOTIVATION_NOTORIETY = "notoriety" +ATTACK_MOTIVATION_ORGANIZATIONAL_GAIN = "organizational-gain" +ATTACK_MOTIVATION_PERSONAL_GAIN = "personal-gain" +ATTACK_MOTIVATION_PERSONAL_SATISFACTION = "personal-satisfaction" +ATTACK_MOTIVATION_REVENGE = "revenge" +ATTACK_MOTIVATION_UNPREDICTABLE = "unpredictable" + + ATTACK_MOTIVATION = [ - "accidental", - "coercion", - "dominance", - "ideology", - "notoriety", - "organizational-gain", - "personal-gain", - "personal-satisfaction", - "revenge", - "unpredictable", + ATTACK_MOTIVATION_ACCIDENTAL, + ATTACK_MOTIVATION_COERCION, + ATTACK_MOTIVATION_DOMINANCE, + ATTACK_MOTIVATION_IDEOLOGY, + ATTACK_MOTIVATION_NOTORIETY, + ATTACK_MOTIVATION_ORGANIZATIONAL_GAIN, + ATTACK_MOTIVATION_PERSONAL_GAIN, + ATTACK_MOTIVATION_PERSONAL_SATISFACTION, + ATTACK_MOTIVATION_REVENGE, + ATTACK_MOTIVATION_UNPREDICTABLE, ] +ATTACK_RESOURCE_LEVEL_INDIVIDUAL = "individual" +ATTACK_RESOURCE_LEVEL_CLUB = "club" +ATTACK_RESOURCE_LEVEL_CONTEST = "contest" +ATTACK_RESOURCE_LEVEL_TEAM = "team" +ATTACK_RESOURCE_LEVEL_ORGANIZATION = "organization" +ATTACK_RESOURCE_LEVEL_GOVERNMENT = "government" + + ATTACK_RESOURCE_LEVEL = [ - "individual", - "club", - "contest", - "team", - "organization", - "government", + ATTACK_RESOURCE_LEVEL_INDIVIDUAL, + ATTACK_RESOURCE_LEVEL_CLUB, + ATTACK_RESOURCE_LEVEL_CONTEST, + ATTACK_RESOURCE_LEVEL_TEAM, + ATTACK_RESOURCE_LEVEL_ORGANIZATION, + ATTACK_RESOURCE_LEVEL_GOVERNMENT, ] +ENCRYPTION_ALGORITHM_AES_256_GCM = "AES-256-GCM" +ENCRYPTION_ALGORITHM_CHACHA20_POLY1305 = "ChaCha20-Poly1305" +ENCRYPTION_ALGORITHM_MIME_TYPE_INDICATED = "mime-type-indicated" + + ENCRYPTION_ALGORITHM = [ - "AES-256-GCM", - "ChaCha20-Poly1305", - "mime-type-indicated", + ENCRYPTION_ALGORITHM_AES_256_GCM, + ENCRYPTION_ALGORITHM_CHACHA20_POLY1305, + ENCRYPTION_ALGORITHM_MIME_TYPE_INDICATED, ] +GROUPING_CONTEXT_SUSPICIOUS_ACTIVITY = "suspicious-activity" +GROUPING_CONTEXT_MALWARE_ANALYSIS = "malware-analysis" +GROUPING_CONTEXT_UNSPECIFIED = "unspecified" + + GROUPING_CONTEXT = [ - "suspicious-activity", - "malware-analysis", - "unspecified", + GROUPING_CONTEXT_SUSPICIOUS_ACTIVITY, + GROUPING_CONTEXT_MALWARE_ANALYSIS, + GROUPING_CONTEXT_UNSPECIFIED, ] +HASHING_ALGORITHM_MD5 = "MD5" +HASHING_ALGORITHM_SHA_1 = "SHA-1" +HASHING_ALGORITHM_SHA_256 = "SHA-256" +HASHING_ALGORITHM_SHA_512 = "SHA-512" +HASHING_ALGORITHM_SHA3_256 = "SHA3-256" +HASHING_ALGORITHM_SHA3_512 = "SHA3-512" +HASHING_ALGORITHM_SSDEEP = "SSDEEP" +HASHING_ALGORITHM_TLSH = "TLSH" + + HASHING_ALGORITHM = [ - "MD5", - "SHA-1", - "SHA-256", - "SHA-512", - "SHA3-256", - "SHA3-512", - "SSDEEP", - "TLSH", + HASHING_ALGORITHM_MD5, + HASHING_ALGORITHM_SHA_1, + HASHING_ALGORITHM_SHA_256, + HASHING_ALGORITHM_SHA_512, + HASHING_ALGORITHM_SHA3_256, + HASHING_ALGORITHM_SHA3_512, + HASHING_ALGORITHM_SSDEEP, + HASHING_ALGORITHM_TLSH, ] +IDENTITY_CLASS_INDIVIDUAL = "individual" +IDENTITY_CLASS_GROUP = "group" +IDENTITY_CLASS_SYSTEM = "system" +IDENTITY_CLASS_ORGANIZATION = "organization" +IDENTITY_CLASS_CLASS = "class" +IDENTITY_CLASS_UNKNOWN = "unknown" + + IDENTITY_CLASS = [ - "individual", - "group", - "system", - "organization", - "class", - "unknown", + IDENTITY_CLASS_INDIVIDUAL, + IDENTITY_CLASS_GROUP, + IDENTITY_CLASS_SYSTEM, + IDENTITY_CLASS_ORGANIZATION, + IDENTITY_CLASS_CLASS, + IDENTITY_CLASS_UNKNOWN, ] +IMPLEMENTATION_LANGUAGE_APPLESCRIPT = "applescript" +IMPLEMENTATION_LANGUAGE_BASH = "bash" +IMPLEMENTATION_LANGUAGE_C = "c" +IMPLEMENTATION_LANGUAGE_CPLUSPLUS = "c++" +IMPLEMENTATION_LANGUAGE_CSHARP = "c#" +IMPLEMENTATION_LANGUAGE_GO = "go" +IMPLEMENTATION_LANGUAGE_JAVA = "java" +IMPLEMENTATION_LANGUAGE_JAVASCRIPT = "javascript" +IMPLEMENTATION_LANGUAGE_LUA = "lua" +IMPLEMENTATION_LANGUAGE_OBJECTIVE_C = "objective-c" +IMPLEMENTATION_LANGUAGE_PERL = "perl" +IMPLEMENTATION_LANGUAGE_PHP = "php" +IMPLEMENTATION_LANGUAGE_POWERSHELL = "powershell" +IMPLEMENTATION_LANGUAGE_PYTHON = "python" +IMPLEMENTATION_LANGUAGE_RUBY = "ruby" +IMPLEMENTATION_LANGUAGE_SCALA = "scala" +IMPLEMENTATION_LANGUAGE_SWIFT = "swift" +IMPLEMENTATION_LANGUAGE_TYPESCRIPT = "typescript" +IMPLEMENTATION_LANGUAGE_VISUAL_BASIC = "visual-basic" +IMPLEMENTATION_LANGUAGE_X86_32 = "x86-32" +IMPLEMENTATION_LANGUAGE_X86_64 = "x86-64" + + IMPLEMENTATION_LANGUAGE = [ - "applescript", - "bash", - "c", - "c++", - "c#", - "go", - "java", - "javascript", - "lua", - "objective-c", - "perl", - "php", - "powershell", - "python", - "ruby", - "scala", - "swift", - "typescript", - "visual-basic", - "x86-32", - "x86-64", + IMPLEMENTATION_LANGUAGE_APPLESCRIPT, + IMPLEMENTATION_LANGUAGE_BASH, + IMPLEMENTATION_LANGUAGE_C, + IMPLEMENTATION_LANGUAGE_CPLUSPLUS, + IMPLEMENTATION_LANGUAGE_CSHARP, + IMPLEMENTATION_LANGUAGE_GO, + IMPLEMENTATION_LANGUAGE_JAVA, + IMPLEMENTATION_LANGUAGE_JAVASCRIPT, + IMPLEMENTATION_LANGUAGE_LUA, + IMPLEMENTATION_LANGUAGE_OBJECTIVE_C, + IMPLEMENTATION_LANGUAGE_PERL, + IMPLEMENTATION_LANGUAGE_PHP, + IMPLEMENTATION_LANGUAGE_POWERSHELL, + IMPLEMENTATION_LANGUAGE_PYTHON, + IMPLEMENTATION_LANGUAGE_RUBY, + IMPLEMENTATION_LANGUAGE_SCALA, + IMPLEMENTATION_LANGUAGE_SWIFT, + IMPLEMENTATION_LANGUAGE_TYPESCRIPT, + IMPLEMENTATION_LANGUAGE_VISUAL_BASIC, + IMPLEMENTATION_LANGUAGE_X86_32, + IMPLEMENTATION_LANGUAGE_X86_64, ] +INDICATOR_TYPE_ANOMALOUS_ACTIVITY = "anomalous-activity" +INDICATOR_TYPE_ANONYMIZATION = "anonymization" +INDICATOR_TYPE_BENIGN = "benign" +INDICATOR_TYPE_COMPROMISED = "compromised" +INDICATOR_TYPE_MALICIOUS_ACTIVITY = "malicious-activity" +INDICATOR_TYPE_ATTRIBUTION = "attribution" +INDICATOR_TYPE_UNKNOWN = "unknown" + + INDICATOR_TYPE = [ - "anomalous-activity", - "anonymization", - "benign", - "compromised", - "malicious-activity", - "attribution", - "unknown", + INDICATOR_TYPE_ANOMALOUS_ACTIVITY, + INDICATOR_TYPE_ANONYMIZATION, + INDICATOR_TYPE_BENIGN, + INDICATOR_TYPE_COMPROMISED, + INDICATOR_TYPE_MALICIOUS_ACTIVITY, + INDICATOR_TYPE_ATTRIBUTION, + INDICATOR_TYPE_UNKNOWN, ] +INDUSTRY_SECTOR_AGRICULTURE = "agriculture" +INDUSTRY_SECTOR_AEROSPACE = "aerospace" +INDUSTRY_SECTOR_AUTOMOTIVE = "automotive" +INDUSTRY_SECTOR_CHEMICAL = "chemical" +INDUSTRY_SECTOR_COMMERCIAL = "commercial" +INDUSTRY_SECTOR_COMMUNICATIONS = "communications" +INDUSTRY_SECTOR_CONSTRUCTION = "construction" +INDUSTRY_SECTOR_DEFENSE = "defense" +INDUSTRY_SECTOR_EDUCATION = "education" +INDUSTRY_SECTOR_ENERGY = "energy" +INDUSTRY_SECTOR_ENTERTAINMENT = "entertainment" +INDUSTRY_SECTOR_FINANCIAL_SERVICES = "financial-services" +INDUSTRY_SECTOR_GOVERNMENT = "government" +INDUSTRY_SECTOR_EMERGENCY_SERVICES = "emergency-services" +INDUSTRY_SECTOR_GOVERNMENT_NATIONAL = "government-national" +INDUSTRY_SECTOR_GOVERNMENT_REGIONAL = "government-regional" +INDUSTRY_SECTOR_GOVERNMENT_LOCAL = "government-local" +INDUSTRY_SECTOR_GOVERNMENT_PUBLIC_SERVICES = "government-public-services" +INDUSTRY_SECTOR_HEALTHCARE = "healthcare" +INDUSTRY_SECTOR_HOSPITALITY_LEISURE = "hospitality-leisure" +INDUSTRY_SECTOR_INFRASTRUCTURE = "infrastructure" +INDUSTRY_SECTOR_DAMS = "dams" +INDUSTRY_SECTOR_NUCLEAR = "nuclear" +INDUSTRY_SECTOR_WATER = "water" +INDUSTRY_SECTOR_INSURANCE = "insurance" +INDUSTRY_SECTOR_MANUFACTURING = "manufacturing" +INDUSTRY_SECTOR_MINING = "mining" +INDUSTRY_SECTOR_NON_PROFIT = "non-profit" +INDUSTRY_SECTOR_PHARMACEUTICALS = "pharmaceuticals" +INDUSTRY_SECTOR_RETAIL = "retail" +INDUSTRY_SECTOR_TECHNOLOGY = "technology" +INDUSTRY_SECTOR_TELECOMMUNICATIONS = "telecommunications" +INDUSTRY_SECTOR_TRANSPORTATION = "transportation" +INDUSTRY_SECTOR_UTILITIES = "utilities" + + INDUSTRY_SECTOR = [ - "agriculture", - "aerospace", - "automotive", - "chemical", - "commercial", - "communications", - "construction", - "defense", - "education", - "energy", - "entertainment", - "financial-services", - "government", - "emergency-services", - "government-national", - "government-regional", - "government-local", - "government-public-services", - "healthcare", - "hospitality-leisure", - "infrastructure", - "dams", - "nuclear", - "water", - "insurance", - "manufacturing", - "mining", - "non-profit", - "pharmaceuticals", - "retail", - "technology", - "telecommunications", - "transportation", - "utilities", + INDUSTRY_SECTOR_AGRICULTURE, + INDUSTRY_SECTOR_AEROSPACE, + INDUSTRY_SECTOR_AUTOMOTIVE, + INDUSTRY_SECTOR_CHEMICAL, + INDUSTRY_SECTOR_COMMERCIAL, + INDUSTRY_SECTOR_COMMUNICATIONS, + INDUSTRY_SECTOR_CONSTRUCTION, + INDUSTRY_SECTOR_DEFENSE, + INDUSTRY_SECTOR_EDUCATION, + INDUSTRY_SECTOR_ENERGY, + INDUSTRY_SECTOR_ENTERTAINMENT, + INDUSTRY_SECTOR_FINANCIAL_SERVICES, + INDUSTRY_SECTOR_GOVERNMENT, + INDUSTRY_SECTOR_EMERGENCY_SERVICES, + INDUSTRY_SECTOR_GOVERNMENT_NATIONAL, + INDUSTRY_SECTOR_GOVERNMENT_REGIONAL, + INDUSTRY_SECTOR_GOVERNMENT_LOCAL, + INDUSTRY_SECTOR_GOVERNMENT_PUBLIC_SERVICES, + INDUSTRY_SECTOR_HEALTHCARE, + INDUSTRY_SECTOR_HOSPITALITY_LEISURE, + INDUSTRY_SECTOR_INFRASTRUCTURE, + INDUSTRY_SECTOR_DAMS, + INDUSTRY_SECTOR_NUCLEAR, + INDUSTRY_SECTOR_WATER, + INDUSTRY_SECTOR_INSURANCE, + INDUSTRY_SECTOR_MANUFACTURING, + INDUSTRY_SECTOR_MINING, + INDUSTRY_SECTOR_NON_PROFIT, + INDUSTRY_SECTOR_PHARMACEUTICALS, + INDUSTRY_SECTOR_RETAIL, + INDUSTRY_SECTOR_TECHNOLOGY, + INDUSTRY_SECTOR_TELECOMMUNICATIONS, + INDUSTRY_SECTOR_TRANSPORTATION, + INDUSTRY_SECTOR_UTILITIES, ] +INFRASTRUCTURE_TYPE_AMPLIFICATION = "amplification" +INFRASTRUCTURE_TYPE_ANONYMIZATION = "anonymization" +INFRASTRUCTURE_TYPE_BOTNET = "botnet" +INFRASTRUCTURE_TYPE_COMMAND_AND_CONTROL = "command-and-control" +INFRASTRUCTURE_TYPE_EXFILTRATION = "exfiltration" +INFRASTRUCTURE_TYPE_HOSTING_MALWARE = "hosting-malware" +INFRASTRUCTURE_TYPE_HOSTING_TARGET_LISTS = "hosting-target-lists" +INFRASTRUCTURE_TYPE_PHISHING = "phishing" +INFRASTRUCTURE_TYPE_RECONNAISSANCE = "reconnaissance" +INFRASTRUCTURE_TYPE_STAGING = "staging" +INFRASTRUCTURE_TYPE_UNKNOWN = "unknown" + + INFRASTRUCTURE_TYPE = [ - "amplification", - "anonymization", - "botnet", - "command-and-control", - "exfiltration", - "hosting-malware", - "hosting-target-lists", - "phishing", - "reconnaissance", - "staging", - "unknown", + INFRASTRUCTURE_TYPE_AMPLIFICATION, + INFRASTRUCTURE_TYPE_ANONYMIZATION, + INFRASTRUCTURE_TYPE_BOTNET, + INFRASTRUCTURE_TYPE_COMMAND_AND_CONTROL, + INFRASTRUCTURE_TYPE_EXFILTRATION, + INFRASTRUCTURE_TYPE_HOSTING_MALWARE, + INFRASTRUCTURE_TYPE_HOSTING_TARGET_LISTS, + INFRASTRUCTURE_TYPE_PHISHING, + INFRASTRUCTURE_TYPE_RECONNAISSANCE, + INFRASTRUCTURE_TYPE_STAGING, + INFRASTRUCTURE_TYPE_UNKNOWN, ] +MALWARE_RESULT_MALICIOUS = "malicious" +MALWARE_RESULT_SUSPICIOUS = "suspicious" +MALWARE_RESULT_BENIGN = "benign" +MALWARE_RESULT_UNKNOWN = "unknown" + + MALWARE_RESULT = [ - "malicious", - "suspicious", - "benign", - "unknown", + MALWARE_RESULT_MALICIOUS, + MALWARE_RESULT_SUSPICIOUS, + MALWARE_RESULT_BENIGN, + MALWARE_RESULT_UNKNOWN, ] +MALWARE_CAPABILITIES_ACCESSES_REMOTE_MACHINES = "accesses-remote-machines" +MALWARE_CAPABILITIES_ANTI_DEBUGGING = "anti-debugging" +MALWARE_CAPABILITIES_ANTI_DISASSEMBLY = "anti-disassembly" +MALWARE_CAPABILITIES_ANTI_EMULATION = "anti-emulation" +MALWARE_CAPABILITIES_ANTI_MEMORY_FORENSICS = "anti-memory-forensics" +MALWARE_CAPABILITIES_ANTI_SANDBOX = "anti-sandbox" +MALWARE_CAPABILITIES_ANTI_VM = "anti-vm" +MALWARE_CAPABILITIES_CAPTURES_INPUT_PERIPHERALS = "captures-input-peripherals" +MALWARE_CAPABILITIES_CAPTURES_OUTPUT_PERIPHERALS = "captures-output-peripherals" +MALWARE_CAPABILITIES_CAPTURES_SYSTEM_STATE_DATA = "captures-system-state-data" +MALWARE_CAPABILITIES_CLEANS_TRACES_OF_INFECTION = "cleans-traces-of-infection" +MALWARE_CAPABILITIES_COMMITS_FRAUD = "commits-fraud" +MALWARE_CAPABILITIES_COMMUNICATES_WITH_C2 = "communicates-with-c2" +MALWARE_CAPABILITIES_COMPROMISES_DATA_AVAILABILITY = "compromises-data-availability" +MALWARE_CAPABILITIES_COMPROMISES_DATA_INTEGRITY = "compromises-data-integrity" +MALWARE_CAPABILITIES_COMPROMISES_SYSTEM_AVAILABILITY = "compromises-system-availability" +MALWARE_CAPABILITIES_CONTROLS_LOCAL_MACHINE = "controls-local-machine" +MALWARE_CAPABILITIES_DEGRADES_SECURITY_SOFTWARE = "degrades-security-software" +MALWARE_CAPABILITIES_DEGRADES_SYSTEM_UPDATES = "degrades-system-updates" +MALWARE_CAPABILITIES_DETERMINES_C2_SERVER = "determines-c2-server" +MALWARE_CAPABILITIES_EMAILS_SPAM = "emails-spam" +MALWARE_CAPABILITIES_ESCALATES_PRIVILEGES = "escalates-privileges" +MALWARE_CAPABILITIES_EVADES_AV = "evades-av" +MALWARE_CAPABILITIES_EXFILTRATES_DATA = "exfiltrates-data" +MALWARE_CAPABILITIES_FINGERPRINTS_HOST = "fingerprints-host" +MALWARE_CAPABILITIES_HIDES_ARTIFACTS = "hides-artifacts" +MALWARE_CAPABILITIES_HIDES_EXECUTING_CODE = "hides-executing-code" +MALWARE_CAPABILITIES_INFECTS_FILES = "infects-files" +MALWARE_CAPABILITIES_INFECTS_REMOTE_MACHINES = "infects-remote-machines" +MALWARE_CAPABILITIES_INSTALLS_OTHER_COMPONENTS = "installs-other-components" +MALWARE_CAPABILITIES_PERSISTS_AFTER_SYSTEM_REBOOT = "persists-after-system-reboot" +MALWARE_CAPABILITIES_PREVENTS_ARTIFACT_ACCESS = "prevents-artifact-access" +MALWARE_CAPABILITIES_PREVENTS_ARTIFACT_DELETION = "prevents-artifact-deletion" +MALWARE_CAPABILITIES_PROBES_NETWORK_ENVIRONMENT = "probes-network-environment" +MALWARE_CAPABILITIES_SELF_MODIFIES = "self-modifies" +MALWARE_CAPABILITIES_STEALS_AUTHENTICATION_CREDENTIALS = "steals-authentication-credentials" +MALWARE_CAPABILITIES_VIOLATES_SYSTEM_OPERATIONAL_INTEGRITY = "violates-system-operational-integrity" + + MALWARE_CAPABILITIES = [ - "accesses-remote-machines", - "anti-debugging", - "anti-disassembly", - "anti-emulation", - "anti-memory-forensics", - "anti-sandbox", - "anti-vm", - "captures-input-peripherals", - "captures-output-peripherals", - "captures-system-state-data", - "cleans-traces-of-infection", - "commits-fraud", - "communicates-with-c2", - "compromises-data-availability", - "compromises-data-integrity", - "compromises-system-availability", - "controls-local-machine", - "degrades-security-software", - "degrades-system-updates", - "determines-c2-server", - "emails-spam", - "escalates-privileges", - "evades-av", - "exfiltrates-data", - "fingerprints-host", - "hides-artifacts", - "hides-executing-code", - "infects-files", - "infects-remote-machines", - "installs-other-components", - "persists-after-system-reboot", - "prevents-artifact-access", - "prevents-artifact-deletion", - "probes-network-environment", - "self-modifies", - "steals-authentication-credentials", - "violates-system-operational-integrity", + MALWARE_CAPABILITIES_ACCESSES_REMOTE_MACHINES, + MALWARE_CAPABILITIES_ANTI_DEBUGGING, + MALWARE_CAPABILITIES_ANTI_DISASSEMBLY, + MALWARE_CAPABILITIES_ANTI_EMULATION, + MALWARE_CAPABILITIES_ANTI_MEMORY_FORENSICS, + MALWARE_CAPABILITIES_ANTI_SANDBOX, + MALWARE_CAPABILITIES_ANTI_VM, + MALWARE_CAPABILITIES_CAPTURES_INPUT_PERIPHERALS, + MALWARE_CAPABILITIES_CAPTURES_OUTPUT_PERIPHERALS, + MALWARE_CAPABILITIES_CAPTURES_SYSTEM_STATE_DATA, + MALWARE_CAPABILITIES_CLEANS_TRACES_OF_INFECTION, + MALWARE_CAPABILITIES_COMMITS_FRAUD, + MALWARE_CAPABILITIES_COMMUNICATES_WITH_C2, + MALWARE_CAPABILITIES_COMPROMISES_DATA_AVAILABILITY, + MALWARE_CAPABILITIES_COMPROMISES_DATA_INTEGRITY, + MALWARE_CAPABILITIES_COMPROMISES_SYSTEM_AVAILABILITY, + MALWARE_CAPABILITIES_CONTROLS_LOCAL_MACHINE, + MALWARE_CAPABILITIES_DEGRADES_SECURITY_SOFTWARE, + MALWARE_CAPABILITIES_DEGRADES_SYSTEM_UPDATES, + MALWARE_CAPABILITIES_DETERMINES_C2_SERVER, + MALWARE_CAPABILITIES_EMAILS_SPAM, + MALWARE_CAPABILITIES_ESCALATES_PRIVILEGES, + MALWARE_CAPABILITIES_EVADES_AV, + MALWARE_CAPABILITIES_EXFILTRATES_DATA, + MALWARE_CAPABILITIES_FINGERPRINTS_HOST, + MALWARE_CAPABILITIES_HIDES_ARTIFACTS, + MALWARE_CAPABILITIES_HIDES_EXECUTING_CODE, + MALWARE_CAPABILITIES_INFECTS_FILES, + MALWARE_CAPABILITIES_INFECTS_REMOTE_MACHINES, + MALWARE_CAPABILITIES_INSTALLS_OTHER_COMPONENTS, + MALWARE_CAPABILITIES_PERSISTS_AFTER_SYSTEM_REBOOT, + MALWARE_CAPABILITIES_PREVENTS_ARTIFACT_ACCESS, + MALWARE_CAPABILITIES_PREVENTS_ARTIFACT_DELETION, + MALWARE_CAPABILITIES_PROBES_NETWORK_ENVIRONMENT, + MALWARE_CAPABILITIES_SELF_MODIFIES, + MALWARE_CAPABILITIES_STEALS_AUTHENTICATION_CREDENTIALS, + MALWARE_CAPABILITIES_VIOLATES_SYSTEM_OPERATIONAL_INTEGRITY, ] +MALWARE_TYPE_ADWARE = "adware" +MALWARE_TYPE_BACKDOOR = "backdoor" +MALWARE_TYPE_BOT = "bot" +MALWARE_TYPE_BOOTKIT = "bootkit" +MALWARE_TYPE_DDOS = "ddos" +MALWARE_TYPE_DOWNLOADER = "downloader" +MALWARE_TYPE_DROPPER = "dropper" +MALWARE_TYPE_EXPLOIT_KIT = "exploit-kit" +MALWARE_TYPE_KEYLOGGER = "keylogger" +MALWARE_TYPE_RANSOMWARE = "ransomware" +MALWARE_TYPE_REMOTE_ACCESS_TROJAN = "remote-access-trojan" +MALWARE_TYPE_RESOURCE_EXPLOITATION = "resource-exploitation" +MALWARE_TYPE_ROGUE_SECURITY_SOFTWARE = "rogue-security-software" +MALWARE_TYPE_ROOTKIT = "rootkit" +MALWARE_TYPE_SCREEN_CAPTURE = "screen-capture" +MALWARE_TYPE_SPYWARE = "spyware" +MALWARE_TYPE_TROJAN = "trojan" +MALWARE_TYPE_UNKNOWN = "unknown" +MALWARE_TYPE_VIRUS = "virus" +MALWARE_TYPE_WEBSHELL = "webshell" +MALWARE_TYPE_WIPER = "wiper" +MALWARE_TYPE_WORM = "worm" + + MALWARE_TYPE = [ - "adware", - "backdoor", - "bot", - "bootkit", - "ddos", - "downloader", - "dropper", - "exploit-kit", - "keylogger", - "ransomware", - "remote-access-trojan", - "resource-exploitation", - "rogue-security-software", - "rootkit", - "screen-capture", - "spyware", - "trojan", - "unknown", - "virus", - "webshell", - "wiper", - "worm", + MALWARE_TYPE_ADWARE, + MALWARE_TYPE_BACKDOOR, + MALWARE_TYPE_BOT, + MALWARE_TYPE_BOOTKIT, + MALWARE_TYPE_DDOS, + MALWARE_TYPE_DOWNLOADER, + MALWARE_TYPE_DROPPER, + MALWARE_TYPE_EXPLOIT_KIT, + MALWARE_TYPE_KEYLOGGER, + MALWARE_TYPE_RANSOMWARE, + MALWARE_TYPE_REMOTE_ACCESS_TROJAN, + MALWARE_TYPE_RESOURCE_EXPLOITATION, + MALWARE_TYPE_ROGUE_SECURITY_SOFTWARE, + MALWARE_TYPE_ROOTKIT, + MALWARE_TYPE_SCREEN_CAPTURE, + MALWARE_TYPE_SPYWARE, + MALWARE_TYPE_TROJAN, + MALWARE_TYPE_UNKNOWN, + MALWARE_TYPE_VIRUS, + MALWARE_TYPE_WEBSHELL, + MALWARE_TYPE_WIPER, + MALWARE_TYPE_WORM, ] +NETWORK_SOCKET_ADDRESS_FAMILY_AF_UNSPEC = "AF_UNSPEC" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_INET = "AF_INET" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_IPX = "AF_IPX" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_APPLETALK = "AF_APPLETALK" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_NETBIOS = "AF_NETBIOS" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_INET6 = "AF_INET6" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_IRDA = "AF_IRDA" +NETWORK_SOCKET_ADDRESS_FAMILY_AF_BTH = "AF_BTH" + + NETWORK_SOCKET_ADDRESS_FAMILY = [ - "AF_UNSPEC", - "AF_INET", - "AF_IPX", - "AF_APPLETALK", - "AF_NETBIOS", - "AF_INET6", - "AF_IRDA", - "AF_BTH", + NETWORK_SOCKET_ADDRESS_FAMILY_AF_UNSPEC, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_INET, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_IPX, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_APPLETALK, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_NETBIOS, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_INET6, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_IRDA, + NETWORK_SOCKET_ADDRESS_FAMILY_AF_BTH, ] +NETWORK_SOCKET_TYPE_SOCK_STREAM = "SOCK_STREAM" +NETWORK_SOCKET_TYPE_SOCK_DGRAM = "SOCK_DGRAM" +NETWORK_SOCKET_TYPE_SOCK_RAW = "SOCK_RAW" +NETWORK_SOCKET_TYPE_SOCK_RDM = "SOCK_RDM" +NETWORK_SOCKET_TYPE_SOCK_SEQPACKET = "SOCK_SEQPACKET" + + NETWORK_SOCKET_TYPE = [ - "SOCK_STREAM", - "SOCK_DGRAM", - "SOCK_RAW", - "SOCK_RDM", - "SOCK_SEQPACKET", + NETWORK_SOCKET_TYPE_SOCK_STREAM, + NETWORK_SOCKET_TYPE_SOCK_DGRAM, + NETWORK_SOCKET_TYPE_SOCK_RAW, + NETWORK_SOCKET_TYPE_SOCK_RDM, + NETWORK_SOCKET_TYPE_SOCK_SEQPACKET, ] +OPINION_STRONGLY_DISAGREE = "strongly-disagree" +OPINION_DISAGREE = "disagree" +OPINION_NEUTRAL = "neutral" +OPINION_AGREE = "agree" +OPINION_STRONGLY_AGREE = "strongly-agree" + + OPINION = [ - "strongly-disagree", - "disagree", - "neutral", - "agree", - "strongly-agree", + OPINION_STRONGLY_DISAGREE, + OPINION_DISAGREE, + OPINION_NEUTRAL, + OPINION_AGREE, + OPINION_STRONGLY_AGREE, ] +PATTERN_TYPE_STIX = "stix" +PATTERN_TYPE_PCRE = "pcre" +PATTERN_TYPE_SIGMA = "sigma" +PATTERN_TYPE_SNORT = "snort" +PATTERN_TYPE_SURICATA = "suricata" +PATTERN_TYPE_YARA = "yara" + + PATTERN_TYPE = [ - "stix", - "pcre", - "sigma", - "snort", - "suricata", - "yara", + PATTERN_TYPE_STIX, + PATTERN_TYPE_PCRE, + PATTERN_TYPE_SIGMA, + PATTERN_TYPE_SNORT, + PATTERN_TYPE_SURICATA, + PATTERN_TYPE_YARA, ] +PROCESSOR_ARCHITECTURE_ALPHA = "alpha" +PROCESSOR_ARCHITECTURE_ARM = "arm" +PROCESSOR_ARCHITECTURE_IA_64 = "ia-64" +PROCESSOR_ARCHITECTURE_MIPS = "mips" +PROCESSOR_ARCHITECTURE_POWERPC = "powerpc" +PROCESSOR_ARCHITECTURE_SPARC = "sparc" +PROCESSOR_ARCHITECTURE_X86 = "x86" +PROCESSOR_ARCHITECTURE_X86_64 = "x86-64" + + PROCESSOR_ARCHITECTURE = [ - "alpha", - "arm", - "ia-64", - "mips", - "powerpc", - "sparc", - "x86", - "x86-64", + PROCESSOR_ARCHITECTURE_ALPHA, + PROCESSOR_ARCHITECTURE_ARM, + PROCESSOR_ARCHITECTURE_IA_64, + PROCESSOR_ARCHITECTURE_MIPS, + PROCESSOR_ARCHITECTURE_POWERPC, + PROCESSOR_ARCHITECTURE_SPARC, + PROCESSOR_ARCHITECTURE_X86, + PROCESSOR_ARCHITECTURE_X86_64, ] +REGION_AFRICA = "africa" +REGION_EASTERN_AFRICA = "eastern-africa" +REGION_MIDDLE_AFRICA = "middle-africa" +REGION_NORTHERN_AFRICA = "northern-africa" +REGION_SOUTHERN_AFRICA = "southern-africa" +REGION_WESTERN_AFRICA = "western-africa" +REGION_AMERICAS = "americas" +REGION_LATIN_AMERICA_CARIBBEAN = "latin-america-caribbean" +REGION_SOUTH_AMERICA = "south-america" +REGION_CARIBBEAN = "caribbean" +REGION_CENTRAL_AMERICA = "central-america" +REGION_NORTHERN_AMERICA = "northern-america" +REGION_ASIA = "asia" +REGION_CENTRAL_ASIA = "central-asia" +REGION_EASTERN_ASIA = "eastern-asia" +REGION_SOUTHERN_ASIA = "southern-asia" +REGION_SOUTH_EASTERN_ASIA = "south-eastern-asia" +REGION_WESTERN_ASIA = "western-asia" +REGION_EUROPE = "europe" +REGION_EASTERN_EUROPE = "eastern-europe" +REGION_NORTHERN_EUROPE = "northern-europe" +REGION_SOUTHERN_EUROPE = "southern-europe" +REGION_WESTERN_EUROPE = "western-europe" +REGION_OCEANIA = "oceania" +REGION_ANTARCTICA = "antarctica" +REGION_AUSTRALIA_NEW_ZEALAND = "australia-new-zealand" +REGION_MELANESIA = "melanesia" +REGION_MICRONESIA = "micronesia" +REGION_POLYNESIA = "polynesia" + + REGION = [ - "africa", - "eastern-africa", - "middle-africa", - "northern-africa", - "southern-africa", - "western-africa", - "americas", - "latin-america-caribbean", - "south-america", - "caribbean", - "central-america", - "northern-america", - "asia", - "central-asia", - "eastern-asia", - "southern-asia", - "south-eastern-asia", - "western-asia", - "europe", - "eastern-europe", - "northern-europe", - "southern-europe", - "western-europe", - "oceania", - "antarctica", - "australia-new-zealand", - "melanesia", - "micronesia", - "polynesia", + REGION_AFRICA, + REGION_EASTERN_AFRICA, + REGION_MIDDLE_AFRICA, + REGION_NORTHERN_AFRICA, + REGION_SOUTHERN_AFRICA, + REGION_WESTERN_AFRICA, + REGION_AMERICAS, + REGION_LATIN_AMERICA_CARIBBEAN, + REGION_SOUTH_AMERICA, + REGION_CARIBBEAN, + REGION_CENTRAL_AMERICA, + REGION_NORTHERN_AMERICA, + REGION_ASIA, + REGION_CENTRAL_ASIA, + REGION_EASTERN_ASIA, + REGION_SOUTHERN_ASIA, + REGION_SOUTH_EASTERN_ASIA, + REGION_WESTERN_ASIA, + REGION_EUROPE, + REGION_EASTERN_EUROPE, + REGION_NORTHERN_EUROPE, + REGION_SOUTHERN_EUROPE, + REGION_WESTERN_EUROPE, + REGION_OCEANIA, + REGION_ANTARCTICA, + REGION_AUSTRALIA_NEW_ZEALAND, + REGION_MELANESIA, + REGION_MICRONESIA, + REGION_POLYNESIA, ] +REPORT_TYPE_ATTACK_PATTERN = "attack-pattern" +REPORT_TYPE_CAMPAIGN = "campaign" +REPORT_TYPE_IDENTITY = "identity" +REPORT_TYPE_INDICATOR = "indicator" +REPORT_TYPE_INTRUSION_SET = "intrusion-set" +REPORT_TYPE_MALWARE = "malware" +REPORT_TYPE_OBSERVED_DATA = "observed-data" +REPORT_TYPE_THREAT_ACTOR = "threat-actor" +REPORT_TYPE_THREAT_REPORT = "threat-report" +REPORT_TYPE_TOOL = "tool" +REPORT_TYPE_VULNERABILITY = "vulnerability" + + REPORT_TYPE = [ - "attack-pattern", - "campaign", - "identity", - "indicator", - "intrusion-set", - "malware", - "observed-data", - "threat-actor", - "threat-report", - "tool", - "vulnerability", + REPORT_TYPE_ATTACK_PATTERN, + REPORT_TYPE_CAMPAIGN, + REPORT_TYPE_IDENTITY, + REPORT_TYPE_INDICATOR, + REPORT_TYPE_INTRUSION_SET, + REPORT_TYPE_MALWARE, + REPORT_TYPE_OBSERVED_DATA, + REPORT_TYPE_THREAT_ACTOR, + REPORT_TYPE_THREAT_REPORT, + REPORT_TYPE_TOOL, + REPORT_TYPE_VULNERABILITY, ] +THREAT_ACTOR_TYPE_ACTIVIST = "activist" +THREAT_ACTOR_TYPE_COMPETITOR = "competitor" +THREAT_ACTOR_TYPE_CRIME_SYNDICATE = "crime-syndicate" +THREAT_ACTOR_TYPE_CRIMINAL = "criminal" +THREAT_ACTOR_TYPE_HACKER = "hacker" +THREAT_ACTOR_TYPE_INSIDER_ACCIDENTAL = "insider-accidental" +THREAT_ACTOR_TYPE_INSIDER_DISGRUNTLED = "insider-disgruntled" +THREAT_ACTOR_TYPE_NATION_STATE = "nation-state" +THREAT_ACTOR_TYPE_SENSATIONALIST = "sensationalist" +THREAT_ACTOR_TYPE_SPY = "spy" +THREAT_ACTOR_TYPE_TERRORIST = "terrorist" +THREAT_ACTOR_TYPE_UNKNOWN = "unknown" + + THREAT_ACTOR_TYPE = [ - "activist", - "competitor", - "crime-syndicate", - "criminal", - "hacker", - "insider-accidental", - "insider-disgruntled", - "nation-state", - "sensationalist", - "spy", - "terrorist", - "unknown", + THREAT_ACTOR_TYPE_ACTIVIST, + THREAT_ACTOR_TYPE_COMPETITOR, + THREAT_ACTOR_TYPE_CRIME_SYNDICATE, + THREAT_ACTOR_TYPE_CRIMINAL, + THREAT_ACTOR_TYPE_HACKER, + THREAT_ACTOR_TYPE_INSIDER_ACCIDENTAL, + THREAT_ACTOR_TYPE_INSIDER_DISGRUNTLED, + THREAT_ACTOR_TYPE_NATION_STATE, + THREAT_ACTOR_TYPE_SENSATIONALIST, + THREAT_ACTOR_TYPE_SPY, + THREAT_ACTOR_TYPE_TERRORIST, + THREAT_ACTOR_TYPE_UNKNOWN, ] +THREAT_ACTOR_ROLE_AGENT = "agent" +THREAT_ACTOR_ROLE_DIRECTOR = "director" +THREAT_ACTOR_ROLE_INDEPENDENT = "independent" +THREAT_ACTOR_ROLE_INFRASTRUCTURE_ARCHITECT = "infrastructure-architect" +THREAT_ACTOR_ROLE_INFRASTRUCTURE_OPERATOR = "infrastructure-operator" +THREAT_ACTOR_ROLE_MALWARE_AUTHOR = "malware-author" +THREAT_ACTOR_ROLE_SPONSOR = "sponsor" + + THREAT_ACTOR_ROLE = [ - "agent", - "director", - "independent", - "infrastructure-architect", - "infrastructure-operator", - "malware-author", - "sponsor", + THREAT_ACTOR_ROLE_AGENT, + THREAT_ACTOR_ROLE_DIRECTOR, + THREAT_ACTOR_ROLE_INDEPENDENT, + THREAT_ACTOR_ROLE_INFRASTRUCTURE_ARCHITECT, + THREAT_ACTOR_ROLE_INFRASTRUCTURE_OPERATOR, + THREAT_ACTOR_ROLE_MALWARE_AUTHOR, + THREAT_ACTOR_ROLE_SPONSOR, ] +THREAT_ACTOR_SOPHISTICATION_NONE = "none" +THREAT_ACTOR_SOPHISTICATION_MINIMAL = "minimal" +THREAT_ACTOR_SOPHISTICATION_INTERMEDIATE = "intermediate" +THREAT_ACTOR_SOPHISTICATION_ADVANCED = "advanced" +THREAT_ACTOR_SOPHISTICATION_EXPERT = "expert" +THREAT_ACTOR_SOPHISTICATION_INNOVATOR = "innovator" +THREAT_ACTOR_SOPHISTICATION_STRATEGIC = "strategic" + + THREAT_ACTOR_SOPHISTICATION = [ - "none", - "minimal", - "intermediate", - "advanced", - "expert", - "innovator", - "strategic", + THREAT_ACTOR_SOPHISTICATION_NONE, + THREAT_ACTOR_SOPHISTICATION_MINIMAL, + THREAT_ACTOR_SOPHISTICATION_INTERMEDIATE, + THREAT_ACTOR_SOPHISTICATION_ADVANCED, + THREAT_ACTOR_SOPHISTICATION_EXPERT, + THREAT_ACTOR_SOPHISTICATION_INNOVATOR, + THREAT_ACTOR_SOPHISTICATION_STRATEGIC, ] +TOOL_TYPE_DENIAL_OF_SERVICE = "denial-of-service" +TOOL_TYPE_EXPLOITATION = "exploitation" +TOOL_TYPE_INFORMATION_GATHERING = "information-gathering" +TOOL_TYPE_NETWORK_CAPTURE = "network-capture" +TOOL_TYPE_CREDENTIAL_EXPLOITATION = "credential-exploitation" +TOOL_TYPE_REMOTE_ACCESS = "remote-access" +TOOL_TYPE_VULNERABILITY_SCANNING = "vulnerability-scanning" +TOOL_TYPE_UNKNOWN = "unknown" + + TOOL_TYPE = [ - "denial-of-service", - "exploitation", - "information-gathering", - "network-capture", - "credential-exploitation", - "remote-access", - "vulnerability-scanning", - "unknown", + TOOL_TYPE_DENIAL_OF_SERVICE, + TOOL_TYPE_EXPLOITATION, + TOOL_TYPE_INFORMATION_GATHERING, + TOOL_TYPE_NETWORK_CAPTURE, + TOOL_TYPE_CREDENTIAL_EXPLOITATION, + TOOL_TYPE_REMOTE_ACCESS, + TOOL_TYPE_VULNERABILITY_SCANNING, + TOOL_TYPE_UNKNOWN, ] +WINDOWS_INTEGRITY_LEVEL_LOW = "low" +WINDOWS_INTEGRITY_LEVEL_MEDIUM = "medium" +WINDOWS_INTEGRITY_LEVEL_HIGH = "high" +WINDOWS_INTEGRITY_LEVEL_SYSTEM = "system" + + WINDOWS_INTEGRITY_LEVEL = [ - "low", - "medium", - "high", - "system", + WINDOWS_INTEGRITY_LEVEL_LOW, + WINDOWS_INTEGRITY_LEVEL_MEDIUM, + WINDOWS_INTEGRITY_LEVEL_HIGH, + WINDOWS_INTEGRITY_LEVEL_SYSTEM, ] +WINDOWS_PEBINARY_TYPE_DLL = "dll" +WINDOWS_PEBINARY_TYPE_EXE = "exe" +WINDOWS_PEBINARY_TYPE_SYS = "sys" + + WINDOWS_PEBINARY_TYPE = [ - "dll", - "exe", - "sys", + WINDOWS_PEBINARY_TYPE_DLL, + WINDOWS_PEBINARY_TYPE_EXE, + WINDOWS_PEBINARY_TYPE_SYS, ] +WINDOWS_REGISTRY_DATATYPE_REG_NONE = "REG_NONE" +WINDOWS_REGISTRY_DATATYPE_REG_SZ = "REG_SZ" +WINDOWS_REGISTRY_DATATYPE_REG_EXPAND_SZ = "REG_EXPAND_SZ" +WINDOWS_REGISTRY_DATATYPE_REG_BINARY = "REG_BINARY" +WINDOWS_REGISTRY_DATATYPE_REG_DWORD = "REG_DWORD" +WINDOWS_REGISTRY_DATATYPE_REG_DWORD_BIG_ENDIAN = "REG_DWORD_BIG_ENDIAN" +WINDOWS_REGISTRY_DATATYPE_REG_DWORD_LITTLE_ENDIAN = "REG_DWORD_LITTLE_ENDIAN" +WINDOWS_REGISTRY_DATATYPE_REG_LINK = "REG_LINK" +WINDOWS_REGISTRY_DATATYPE_REG_MULTI_SZ = "REG_MULTI_SZ" +WINDOWS_REGISTRY_DATATYPE_REG_RESOURCE_LIST = "REG_RESOURCE_LIST" +WINDOWS_REGISTRY_DATATYPE_REG_FULL_RESOURCE_DESCRIPTION = "REG_FULL_RESOURCE_DESCRIPTION" +WINDOWS_REGISTRY_DATATYPE_REG_RESOURCE_REQUIREMENTS_LIST = "REG_RESOURCE_REQUIREMENTS_LIST" +WINDOWS_REGISTRY_DATATYPE_REG_QWORD = "REG_QWORD" +WINDOWS_REGISTRY_DATATYPE_REG_INVALID_TYPE = "REG_INVALID_TYPE" + + WINDOWS_REGISTRY_DATATYPE = [ - "REG_NONE", - "REG_SZ", - "REG_EXPAND_SZ", - "REG_BINARY", - "REG_DWORD", - "REG_DWORD_BIG_ENDIAN", - "REG_DWORD_LITTLE_ENDIAN", - "REG_LINK", - "REG_MULTI_SZ", - "REG_RESOURCE_LIST", - "REG_FULL_RESOURCE_DESCRIPTION", - "REG_RESOURCE_REQUIREMENTS_LIST", - "REG_QWORD", - "REG_INVALID_TYPE", + WINDOWS_REGISTRY_DATATYPE_REG_NONE, + WINDOWS_REGISTRY_DATATYPE_REG_SZ, + WINDOWS_REGISTRY_DATATYPE_REG_EXPAND_SZ, + WINDOWS_REGISTRY_DATATYPE_REG_BINARY, + WINDOWS_REGISTRY_DATATYPE_REG_DWORD, + WINDOWS_REGISTRY_DATATYPE_REG_DWORD_BIG_ENDIAN, + WINDOWS_REGISTRY_DATATYPE_REG_DWORD_LITTLE_ENDIAN, + WINDOWS_REGISTRY_DATATYPE_REG_LINK, + WINDOWS_REGISTRY_DATATYPE_REG_MULTI_SZ, + WINDOWS_REGISTRY_DATATYPE_REG_RESOURCE_LIST, + WINDOWS_REGISTRY_DATATYPE_REG_FULL_RESOURCE_DESCRIPTION, + WINDOWS_REGISTRY_DATATYPE_REG_RESOURCE_REQUIREMENTS_LIST, + WINDOWS_REGISTRY_DATATYPE_REG_QWORD, + WINDOWS_REGISTRY_DATATYPE_REG_INVALID_TYPE, ] +WINDOWS_SERVICE_START_TYPE_SERVICE_AUTO_START = "SERVICE_AUTO_START" +WINDOWS_SERVICE_START_TYPE_SERVICE_BOOT_START = "SERVICE_BOOT_START" +WINDOWS_SERVICE_START_TYPE_SERVICE_DEMAND_START = "SERVICE_DEMAND_START" +WINDOWS_SERVICE_START_TYPE_SERVICE_DISABLED = "SERVICE_DISABLED" +WINDOWS_SERVICE_START_TYPE_SERVICE_SYSTEM_ALERT = "SERVICE_SYSTEM_ALERT" + + WINDOWS_SERVICE_START_TYPE = [ - "SERVICE_AUTO_START", - "SERVICE_BOOT_START", - "SERVICE_DEMAND_START", - "SERVICE_DISABLED", - "SERVICE_SYSTEM_ALERT", + WINDOWS_SERVICE_START_TYPE_SERVICE_AUTO_START, + WINDOWS_SERVICE_START_TYPE_SERVICE_BOOT_START, + WINDOWS_SERVICE_START_TYPE_SERVICE_DEMAND_START, + WINDOWS_SERVICE_START_TYPE_SERVICE_DISABLED, + WINDOWS_SERVICE_START_TYPE_SERVICE_SYSTEM_ALERT, ] +WINDOWS_SERVICE_TYPE_SERVICE_KERNEL_DRIVER = "SERVICE_KERNEL_DRIVER" +WINDOWS_SERVICE_TYPE_SERVICE_FILE_SYSTEM_DRIVER = "SERVICE_FILE_SYSTEM_DRIVER" +WINDOWS_SERVICE_TYPE_SERVICE_WIN32_OWN_PROCESS = "SERVICE_WIN32_OWN_PROCESS" +WINDOWS_SERVICE_TYPE_SERVICE_WIN32_SHARE_PROCESS = "SERVICE_WIN32_SHARE_PROCESS" + + WINDOWS_SERVICE_TYPE = [ - "SERVICE_KERNEL_DRIVER", - "SERVICE_FILE_SYSTEM_DRIVER", - "SERVICE_WIN32_OWN_PROCESS", - "SERVICE_WIN32_SHARE_PROCESS", + WINDOWS_SERVICE_TYPE_SERVICE_KERNEL_DRIVER, + WINDOWS_SERVICE_TYPE_SERVICE_FILE_SYSTEM_DRIVER, + WINDOWS_SERVICE_TYPE_SERVICE_WIN32_OWN_PROCESS, + WINDOWS_SERVICE_TYPE_SERVICE_WIN32_SHARE_PROCESS, ] +WINDOWS_SERVICE_STATUS_SERVICE_CONTINUE_PENDING = "SERVICE_CONTINUE_PENDING" +WINDOWS_SERVICE_STATUS_SERVICE_PAUSE_PENDING = "SERVICE_PAUSE_PENDING" +WINDOWS_SERVICE_STATUS_SERVICE_PAUSED = "SERVICE_PAUSED" +WINDOWS_SERVICE_STATUS_SERVICE_RUNNING = "SERVICE_RUNNING" +WINDOWS_SERVICE_STATUS_SERVICE_START_PENDING = "SERVICE_START_PENDING" +WINDOWS_SERVICE_STATUS_SERVICE_STOP_PENDING = "SERVICE_STOP_PENDING" +WINDOWS_SERVICE_STATUS_SERVICE_STOPPED = "SERVICE_STOPPED" + + WINDOWS_SERVICE_STATUS = [ - "SERVICE_CONTINUE_PENDING", - "SERVICE_PAUSE_PENDING", - "SERVICE_PAUSED", - "SERVICE_RUNNING", - "SERVICE_START_PENDING", - "SERVICE_STOP_PENDING", - "SERVICE_STOPPED", + WINDOWS_SERVICE_STATUS_SERVICE_CONTINUE_PENDING, + WINDOWS_SERVICE_STATUS_SERVICE_PAUSE_PENDING, + WINDOWS_SERVICE_STATUS_SERVICE_PAUSED, + WINDOWS_SERVICE_STATUS_SERVICE_RUNNING, + WINDOWS_SERVICE_STATUS_SERVICE_START_PENDING, + WINDOWS_SERVICE_STATUS_SERVICE_STOP_PENDING, + WINDOWS_SERVICE_STATUS_SERVICE_STOPPED, ] From 7209346f0b03b30462df3e39e6330b15ca5c742b Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 11 Jun 2021 13:59:48 -0400 Subject: [PATCH 20/21] Improve error message for invalid reference types --- stix2/properties.py | 20 ++++++++++++++------ stix2/test/v21/test_observed_data.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index b67f627..61c1b43 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -590,18 +590,26 @@ class ReferenceProperty(Property): ) and obj_type not in specifics ) or obj_type in blacklist_exceptions - if not type_ok: - raise ValueError( - "The type-specifying prefix '%s' for this property is not " - "valid" % obj_type, - ) - # We need to figure out whether the referenced object is custom or # not. No good way to do that at present... just check if # unregistered and for the "x-" type prefix, for now? has_custom = not is_object(obj_type, self.spec_version) \ or obj_type.startswith("x-") + if not type_ok: + types = self.specifics.union(self.generics) + types = ", ".join(x.name if isinstance(x, STIXTypeClass) else x for x in types) + if self.auth_type == self._WHITELIST: + msg = "not one of the valid types for this property: %s." % types + else: + msg = "one of the invalid types for this property: %s." % types + if not allow_custom and has_custom: + msg += " A custom object type may be allowed with allow_custom=True." + raise ValueError( + "The type-specifying prefix '%s' for this property is %s" + % (obj_type, msg), + ) + if not allow_custom and has_custom: raise CustomContentError( "reference to custom object type: " + obj_type, diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py index c1cb38e..e9a190a 100644 --- a/stix2/test/v21/test_observed_data.py +++ b/stix2/test/v21/test_observed_data.py @@ -208,7 +208,7 @@ def test_observed_data_example_with_bad_refs(): assert excinfo.value.cls == stix2.v21.Directory assert excinfo.value.prop_name == "contains_refs" - assert "The type-specifying prefix 'monkey' for this property is not valid" in excinfo.value.reason + assert "The type-specifying prefix 'monkey' for this property is not" in excinfo.value.reason def test_observed_data_example_with_non_dictionary(): From 5cce8643041be40171377d01eb6dca4352613b9e Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 11 Jun 2021 14:00:46 -0400 Subject: [PATCH 21/21] Test invalid_types ReferenceProperty with custom --- stix2/test/v21/test_properties.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 60cbeb0..58e84eb 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -190,6 +190,27 @@ def test_reference_property_blacklist_standard_type(): ) +def test_reference_property_blacklist_custom_type(): + ref_prop = ReferenceProperty(invalid_types="my-type", spec_version="2.1") + + result = ref_prop.clean("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + assert result == ("file--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + with pytest.raises(ValueError): + ref_prop.clean("my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + with pytest.raises(CustomContentError): + # This is not the blacklisted type, but it's still custom, and + # customization is disallowed here. + ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", False) + + result = ref_prop.clean("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + assert result == ("not-my-type--8a8e8758-f92c-4058-ba38-f061cd42a0cf", True) + + def test_reference_property_blacklist_generic_type(): ref_prop = ReferenceProperty( invalid_types=["SDO", "SRO"], spec_version="2.1",