From 4aa69fa7c9b6d7123be76dcb559a89e736d907f1 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Mon, 16 Mar 2020 20:25:38 -0400 Subject: [PATCH 01/37] Add support for enforcing STIX 2.1 minimum precision requirement on versioning timestamps. --- setup.py | 1 + stix2/properties.py | 8 +- stix2/test/v21/test_attack_pattern.py | 4 +- stix2/test/v21/test_timestamp_precision.py | 146 ++++++++++++ stix2/test/v21/test_versioning.py | 33 ++- stix2/utils.py | 246 ++++++++++++++++++--- stix2/v21/common.py | 6 +- stix2/v21/sdo.py | 76 +++---- stix2/v21/sro.py | 8 +- 9 files changed, 443 insertions(+), 85 deletions(-) create mode 100644 stix2/test/v21/test_timestamp_precision.py diff --git a/setup.py b/setup.py index 66bf302..b028ffe 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ setup( keywords='stix stix2 json cti cyber threat intelligence', packages=find_packages(exclude=['*.test', '*.test.*']), install_requires=[ + 'enum34 ; python_version<"3.4"', 'python-dateutil', 'pytz', 'requests', diff --git a/stix2/properties.py b/stix2/properties.py index b013b7e..1fd1dd8 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -323,12 +323,16 @@ class BooleanProperty(Property): class TimestampProperty(Property): - def __init__(self, precision=None, **kwargs): + def __init__(self, precision="any", precision_constraint="exact", **kwargs): self.precision = precision + self.precision_constraint = precision_constraint + super(TimestampProperty, self).__init__(**kwargs) def clean(self, value): - return parse_into_datetime(value, self.precision) + return parse_into_datetime( + value, self.precision, self.precision_constraint + ) class DictionaryProperty(Property): diff --git a/stix2/test/v21/test_attack_pattern.py b/stix2/test/v21/test_attack_pattern.py index 165581c..b826f1e 100644 --- a/stix2/test/v21/test_attack_pattern.py +++ b/stix2/test/v21/test_attack_pattern.py @@ -88,8 +88,8 @@ def test_attack_pattern_invalid_labels(): def test_overly_precise_timestamps(): ap = stix2.v21.AttackPattern( id=ATTACK_PATTERN_ID, - created="2016-05-12T08:17:27.0000342Z", - modified="2016-05-12T08:17:27.000287Z", + created="2016-05-12T08:17:27.000000342Z", + modified="2016-05-12T08:17:27.000000287Z", name="Spear Phishing", external_references=[{ "source_name": "capec", diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py new file mode 100644 index 0000000..5256e6c --- /dev/null +++ b/stix2/test/v21/test_timestamp_precision.py @@ -0,0 +1,146 @@ +import datetime +import pytest +from stix2.utils import ( + Precision, PrecisionConstraint, _to_enum, parse_into_datetime, + format_datetime, STIXdatetime +) +import stix2 + + +_DT = datetime.datetime.utcnow() +# intentionally omit microseconds from the following. We add it in as +# needed for each test. +_DT_STR = _DT.strftime("%Y-%m-%dT%H:%M:%S") + + +@pytest.mark.parametrize("value, enum_type, enum_default, enum_expected", [ + ("second", Precision, None, Precision.SECOND), + ("eXaCt", PrecisionConstraint, PrecisionConstraint.MIN, + PrecisionConstraint.EXACT), + (None, Precision, Precision.MILLISECOND, Precision.MILLISECOND), + (Precision.ANY, Precision, None, Precision.ANY) +]) +def test_to_enum(value, enum_type, enum_default, enum_expected): + result = _to_enum(value, enum_type, enum_default) + assert result == enum_expected + + +@pytest.mark.parametrize("value, err_type", [ + ("foo", KeyError), + (1, TypeError), + (PrecisionConstraint.EXACT, TypeError), + (None, TypeError) +]) +def test_to_enum_errors(value, err_type): + with pytest.raises(err_type): + _to_enum(value, Precision) + + +def test_stix_datetime(): + dt = STIXdatetime.utcnow() + assert dt.precision is Precision.ANY + assert dt.precision_constraint is PrecisionConstraint.EXACT + + sdt = STIXdatetime(dt, precision=Precision.SECOND) + assert sdt.precision is Precision.SECOND + assert sdt == dt + + sdt = STIXdatetime( + dt, + precision_constraint=PrecisionConstraint.EXACT + ) + assert sdt.precision_constraint is PrecisionConstraint.EXACT + assert sdt == dt + + +@pytest.mark.parametrize( + "us, precision, precision_constraint, expected_truncated_us", [ + (123456, Precision.ANY, PrecisionConstraint.EXACT, 123456), + (123456, Precision.SECOND, PrecisionConstraint.EXACT, 0), + (123456, Precision.SECOND, PrecisionConstraint.MIN, 123456), + (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, 123000), + (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, 123456), + (1234, Precision.MILLISECOND, PrecisionConstraint.EXACT, 1000), + (123, Precision.MILLISECOND, PrecisionConstraint.EXACT, 0) +]) +def test_parse_datetime( + us, precision, precision_constraint, expected_truncated_us +): + + # complete the datetime string with microseconds + dt_us_str = "{}.{:06d}Z".format(_DT_STR, us) + + sdt = parse_into_datetime( + dt_us_str, + precision=precision, + precision_constraint=precision_constraint + ) + + assert sdt.precision is precision + assert sdt.precision_constraint is precision_constraint + assert sdt.microsecond == expected_truncated_us + + +@pytest.mark.parametrize( + "us, precision, precision_constraint, expected_us_str", [ + (123456, Precision.ANY, PrecisionConstraint.EXACT, ".123456"), + (123456, Precision.SECOND, PrecisionConstraint.EXACT, ""), + (123456, Precision.SECOND, PrecisionConstraint.MIN, ".123456"), + (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".123"), + (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, ".123456"), + (0, Precision.SECOND, PrecisionConstraint.MIN, ""), + (0, Precision.MILLISECOND, PrecisionConstraint.MIN, ".000"), + (0, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".000"), + (1000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".001"), + (10000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".010"), + (100000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".100"), + (1000, Precision.ANY, PrecisionConstraint.EXACT, ".001"), + (10000, Precision.ANY, PrecisionConstraint.EXACT, ".01"), + (100000, Precision.ANY, PrecisionConstraint.EXACT, ".1"), + (1001, Precision.MILLISECOND, PrecisionConstraint.MIN, ".001001"), + (10010, Precision.MILLISECOND, PrecisionConstraint.MIN, ".01001"), + (100100, Precision.MILLISECOND, PrecisionConstraint.MIN, ".1001"), +]) +def test_format_datetime(us, precision, precision_constraint, expected_us_str): + + dt = _DT.replace(microsecond=us) + expected_dt_str = "{}{}Z".format(_DT_STR, expected_us_str) + + sdt = STIXdatetime( + dt, + precision=precision, + precision_constraint=precision_constraint + ) + s = format_datetime(sdt) + assert s == expected_dt_str + + +def test_sdo_extra_precision(): + # add extra precision for "modified", ensure it's not lost + identity_dict = { + "type": "identity", + "id": "identity--4a457eeb-6639-4aa3-be81-5930a3000c39", + "created": "2015-12-21T19:59:11.000Z", + "modified" :"2015-12-21T19:59:11.0001Z", + "name" :"John Smith", + "identity_class" :"individual", + "spec_version": "2.1" + } + + identity_obj = stix2.parse(identity_dict) + assert identity_obj.modified.microsecond == 100 + assert identity_obj.modified.precision is Precision.MILLISECOND + assert identity_obj.modified.precision_constraint is PrecisionConstraint.MIN + + identity_str = identity_obj.serialize(pretty=True) + + # ensure precision is retained in JSON + assert identity_str == """{ + "type": "identity", + "spec_version": "2.1", + "id": "identity--4a457eeb-6639-4aa3-be81-5930a3000c39", + "created": "2015-12-21T19:59:11.000Z", + "modified": "2015-12-21T19:59:11.0001Z", + "name": "John Smith", + "identity_class": "individual" +}""" diff --git a/stix2/test/v21/test_versioning.py b/stix2/test/v21/test_versioning.py index c46183c..a1a21c8 100644 --- a/stix2/test/v21/test_versioning.py +++ b/stix2/test/v21/test_versioning.py @@ -1,6 +1,8 @@ +import datetime import pytest import stix2 +import stix2.utils from .constants import CAMPAIGN_MORE_KWARGS @@ -236,8 +238,7 @@ def test_remove_custom_stix_property(): mal_nc = stix2.utils.remove_custom_stix(mal) assert "x_custom" not in mal_nc - assert (stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < - stix2.utils.parse_into_datetime(mal_nc["modified"], precision="millisecond")) + assert mal["modified"] < mal_nc["modified"] def test_remove_custom_stix_object(): @@ -264,3 +265,31 @@ def test_remove_custom_stix_no_custom(): assert len(campaign_v1.keys()) == len(campaign_v2.keys()) assert campaign_v1.id == campaign_v2.id assert campaign_v1.description == campaign_v2.description + + +@pytest.mark.parametrize("old, candidate_new, expected_new, use_stix21", [ + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.0001Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.9999Z", "1999-08-15T00:19:08.000Z", False), + + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999999Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.999999Z", "1999-08-15T00:19:07.999999Z", True), +]) +def test_fudge_modified(old, candidate_new, expected_new, use_stix21): + old_dt = datetime.datetime.strptime(old, "%Y-%m-%dT%H:%M:%S.%fZ") + candidate_new_dt = datetime.datetime.strptime( + candidate_new, "%Y-%m-%dT%H:%M:%S.%fZ" + ) + expected_new_dt = datetime.datetime.strptime( + expected_new, "%Y-%m-%dT%H:%M:%S.%fZ" + ) + + fudged = stix2.utils._fudge_modified(old_dt, candidate_new_dt, use_stix21) + assert fudged == expected_new_dt diff --git a/stix2/utils.py b/stix2/utils.py index 7b3b6cf..7e93259 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -6,7 +6,9 @@ except ImportError: from collections import Mapping import copy import datetime as dt +import enum import json +import six from dateutil import parser import pytz @@ -29,9 +31,79 @@ TYPE_REGEX = r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$' SCO21_EXT_REGEX = r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-ext$' +class Precision(enum.Enum): + """ + Timestamp format precisions. + """ + # auto() wasn't introduced until Python 3.6. + ANY = 1 + SECOND = 2 + MILLISECOND = 3 + + +class PrecisionConstraint(enum.Enum): + """ + Timestamp precision constraints. These affect how the Precision + values are applied when formatting a timestamp. + + These constraints don't really make sense with the ANY precision, so they + have no effect in that case. + """ + EXACT = 1 # format must have exactly the given precision + MIN = 2 # format must have at least the given precision + # no need for a MAX constraint yet + + +def _to_enum(value, enum_type, enum_default=None): + """ + Detect and convert strings to enums and None to a default enum. This + allows use of strings and None in APIs, while enforcing the enum type: if + you use a string, it must name a valid enum value. This implementation is + case-insensitive. + + :param value: A value to be interpreted as an enum (string, Enum instance, + or None). If an Enum instance, it must be an instance of enum_type. + :param enum_type: The enum type which strings will be interpreted against + :param enum_default: The default enum to use if value is None. Must be + an instance of enum_type, or None. If None, you are disallowing a + default and requiring that value be non-None. + :return: An instance of enum_type + :raises TypeError: If value was neither an instance of enum_type, None, nor + a string + :raises KeyError: If value was a string which couldn't be interpreted as an + enum value from enum_type + """ + assert enum_default is None or isinstance(enum_default, enum_type) + + if not isinstance(value, enum_type): + if value is None and enum_default is not None: + value = enum_default + elif isinstance(value, six.string_types): + value = enum_type[value.upper()] + else: + raise TypeError("Not a valid {}: {}".format( + enum_type.__name__, value + )) + + return value + + class STIXdatetime(dt.datetime): + """ + Bundle a datetime with some format-related metadata, so that JSON + serialization has the info it needs to produce compliant timestamps. + """ + def __new__(cls, *args, **kwargs): - precision = kwargs.pop('precision', None) + precision = _to_enum( + kwargs.pop("precision", Precision.ANY), + Precision + ) + precision_constraint = _to_enum( + kwargs.pop("precision_constraint", PrecisionConstraint.EXACT), + PrecisionConstraint + ) + if isinstance(args[0], dt.datetime): # Allow passing in a datetime object dttm = args[0] args = ( @@ -41,6 +113,7 @@ class STIXdatetime(dt.datetime): # self will be an instance of STIXdatetime, not dt.datetime self = dt.datetime.__new__(cls, *args, **kwargs) self.precision = precision + self.precision_constraint = precision_constraint return self def __repr__(self): @@ -90,7 +163,7 @@ def format_datetime(dttm): 2. Convert to UTC 3. Format in ISO format 4. Ensure correct precision - a. Add subsecond value if non-zero and precision not defined + a. Add subsecond value if warranted, according to precision settings 5. Add "Z" """ @@ -101,20 +174,74 @@ def format_datetime(dttm): else: zoned = dttm.astimezone(pytz.utc) ts = zoned.strftime('%Y-%m-%dT%H:%M:%S') - ms = zoned.strftime('%f') - precision = getattr(dttm, 'precision', None) - if precision == 'second': - pass # Already precise to the second - elif precision == 'millisecond': - ts = ts + '.' + ms[:3] - elif zoned.microsecond > 0: - ts = ts + '.' + ms.rstrip('0') - return ts + 'Z' + precision = getattr(dttm, 'precision', Precision.ANY) + precision_constraint = getattr( + dttm, 'precision_constraint', PrecisionConstraint.EXACT + ) + + frac_seconds_str = "" + if precision == Precision.ANY: + # No need to truncate; ignore constraint + if zoned.microsecond: + frac_seconds_str = "{:06d}".format(zoned.microsecond).rstrip("0") + + elif precision == Precision.SECOND: + if precision_constraint == PrecisionConstraint.MIN: + # second precision, or better. Winds up being the same as ANY: + # just use all our digits + if zoned.microsecond: + frac_seconds_str = "{:06d}".format(zoned.microsecond)\ + .rstrip("0") + # exact: ignore microseconds entirely + + else: + # precision == millisecond + if precision_constraint == PrecisionConstraint.EXACT: + # can't rstrip() here or we may lose precision + frac_seconds_str = "{:06d}".format(zoned.microsecond)[:3] + + else: + # millisecond precision, or better. So we can rstrip() zeros, but + # only to a length of at least 3 digits (ljust() adds zeros back, + # if it stripped too far.) + frac_seconds_str = "{:06d}"\ + .format(zoned.microsecond)\ + .rstrip("0")\ + .ljust(3, "0") + + ts = "{}{}{}Z".format( + ts, + "." if frac_seconds_str else "", + frac_seconds_str + ) + + return ts -def parse_into_datetime(value, precision=None): - """Parse a value into a valid STIX timestamp object. +def parse_into_datetime( + value, precision=Precision.ANY, + precision_constraint=PrecisionConstraint.EXACT +): """ + Parse a value into a valid STIX timestamp object. Also, optionally adjust + precision of fractional seconds. This allows alignment with JSON + serialization requirements, and helps ensure we're not using extra + precision which would be lost upon JSON serialization. The precision + info will be embedded in the returned object, so that JSON serialization + will format it correctly. + + :param value: A datetime.datetime or datetime.date instance, or a string + :param precision: A precision value: either an instance of the Precision + enum, or a string naming one of the enum values (case-insensitive) + :param precision_constraint: A precision constraint value: either an + instance of the PrecisionConstraint enum, or a string naming one of + the enum values (case-insensitive) + :return: A STIXdatetime instance, which is a datetime but also carries the + precision info necessary to properly JSON-serialize it. + """ + precision = _to_enum(precision, Precision) + precision_constraint = _to_enum(precision_constraint, PrecisionConstraint) + if isinstance(value, dt.date): if hasattr(value, 'hour'): ts = value @@ -138,20 +265,23 @@ def parse_into_datetime(value, precision=None): ts = pytz.utc.localize(parsed) # Ensure correct precision - if not precision: - return STIXdatetime(ts, precision=precision) - ms = ts.microsecond - if precision == 'second': - ts = ts.replace(microsecond=0) - elif precision == 'millisecond': - ms_len = len(str(ms)) - if ms_len > 3: - # Truncate to millisecond precision - factor = 10 ** (ms_len - 3) - ts = ts.replace(microsecond=(ts.microsecond // factor) * factor) - else: + if precision == Precision.SECOND: + if precision_constraint == PrecisionConstraint.EXACT: ts = ts.replace(microsecond=0) - return STIXdatetime(ts, precision=precision) + # else, no need to modify fractional seconds + + elif precision == Precision.MILLISECOND: + if precision_constraint == PrecisionConstraint.EXACT: + us = (ts.microsecond // 1000) * 1000 + ts = ts.replace(microsecond=us) + # else: at least millisecond precision: the constraint will affect JSON + # formatting, but there's nothing we need to do here. + + # else, precision == Precision.ANY: nothing for us to do. + + return STIXdatetime( + ts, precision=precision, precision_constraint=precision_constraint + ) def _get_dict(data): @@ -256,6 +386,39 @@ def find_property_index(obj, search_key, search_value): return idx +def _fudge_modified(old_modified, new_modified, use_stix21): + """ + Ensures a new modified timestamp is newer than the old. When they are + too close together, new_modified must be pushed further ahead to ensure + it is distinct and later, after JSON serialization (which may mean it's + actually being pushed a little ways into the future). JSON serialization + can remove precision, which can cause distinct timestamps to accidentally + become equal, if we're not careful. + + :param old_modified: A previous "modified" timestamp, as a datetime object + :param new_modified: A candidate new "modified" timestamp, as a datetime + object + :param use_stix21: Whether to use STIX 2.1+ versioning timestamp precision + rules (boolean). This is important so that we are aware of how + timestamp precision will be truncated, so we know how close together + the timestamps can be, and how far ahead to potentially push the new + one. + :return: A suitable new "modified" timestamp. This may be different from + what was passed in, if it had to be pushed ahead. + """ + if use_stix21: + # 2.1+: we can use full precision + if new_modified <= old_modified: + new_modified = old_modified + dt.timedelta(microseconds=1) + else: + # 2.0: we must use millisecond precision + one_ms = dt.timedelta(milliseconds=1) + if new_modified - old_modified < one_ms: + new_modified = old_modified + one_ms + + return new_modified + + def new_version(data, **kwargs): """Create a new version of a STIX object, by modifying properties and updating the ``modified`` property. @@ -283,12 +446,32 @@ def new_version(data, **kwargs): if unchangable_properties: raise UnmodifiablePropertyError(unchangable_properties) + # Different versioning precision rules in STIX 2.0 vs 2.1, so we need + # to know which rules to apply. + is_21 = "spec_version" in data + precision_constraint = "min" if is_21 else "exact" + cls = type(data) if 'modified' not in kwargs: - kwargs['modified'] = get_timestamp() + old_modified = parse_into_datetime( + data["modified"], precision="millisecond", + precision_constraint=precision_constraint + ) + + new_modified = get_timestamp() + new_modified = _fudge_modified(old_modified, new_modified, is_21) + + kwargs['modified'] = new_modified + elif 'modified' in data: - old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond') - new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') + old_modified_property = parse_into_datetime( + data.get('modified'), precision='millisecond', + precision_constraint=precision_constraint + ) + new_modified_property = parse_into_datetime( + kwargs['modified'], precision='millisecond', + precision_constraint=precision_constraint + ) if new_modified_property <= old_modified_property: raise InvalidValueError( cls, 'modified', @@ -378,11 +561,6 @@ def remove_custom_stix(stix_obj): new_obj = new_version(stix_obj, **(dict(props))) - while parse_into_datetime(new_obj['modified']) == parse_into_datetime(stix_obj['modified']): - # Prevents bug when fast computation allows multiple STIX object - # versions to be created in single unit of time - new_obj = new_version(stix_obj, **(dict(props))) - return new_obj else: diff --git a/stix2/v21/common.py b/stix2/v21/common.py index ac8daf1..86be923 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -91,8 +91,8 @@ class LanguageContent(_STIXBase): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('object_ref', ReferenceProperty(valid_types=["SCO", "SDO", "SRO"], spec_version='2.1', required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. ('object_modified', TimestampProperty(precision='millisecond')), @@ -162,7 +162,7 @@ class MarkingDefinition(_STIXBase, _MarkingsMixin): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type)), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('external_references', ListProperty(ExternalReference)), ('object_marking_refs', ListProperty(ReferenceProperty(valid_types='marking-definition', spec_version='2.1'))), ('granular_markings', ListProperty(GranularMarking)), diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index f2dc0ea..f9c01f1 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -33,8 +33,8 @@ class AttackPattern(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('aliases', ListProperty(StringProperty)), @@ -61,8 +61,8 @@ class Campaign(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('aliases', ListProperty(StringProperty)), @@ -101,8 +101,8 @@ class CourseOfAction(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -126,8 +126,8 @@ class Grouping(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -155,8 +155,8 @@ class Identity(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('roles', ListProperty(StringProperty)), @@ -185,8 +185,8 @@ class Indicator(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), ('indicator_types', ListProperty(StringProperty)), @@ -245,8 +245,8 @@ class Infrastructure(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), ('confidence', IntegerProperty()), @@ -286,8 +286,8 @@ class IntrusionSet(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('aliases', ListProperty(StringProperty)), @@ -329,8 +329,8 @@ class Location(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), ('latitude', FloatProperty(min=-90.0, max=90.0)), @@ -437,8 +437,8 @@ class Malware(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty()), ('description', StringProperty()), ('malware_types', ListProperty(StringProperty)), @@ -489,8 +489,8 @@ class MalwareAnalysis(STIXDomainObject): ('type', TypeProperty(_type)), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), ('revoked', BooleanProperty(default=lambda: False)), ('labels', ListProperty(StringProperty)), @@ -535,8 +535,8 @@ class Note(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('abstract', StringProperty()), ('content', StringProperty(required=True)), ('authors', ListProperty(StringProperty)), @@ -563,8 +563,8 @@ class ObservedData(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('first_observed', TimestampProperty(required=True)), ('last_observed', TimestampProperty(required=True)), ('number_observed', IntegerProperty(min=1, max=999999999, required=True)), @@ -619,8 +619,8 @@ class Opinion(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('explanation', StringProperty()), ('authors', ListProperty(StringProperty)), ( @@ -657,8 +657,8 @@ class Report(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('report_types', ListProperty(StringProperty)), @@ -686,8 +686,8 @@ class ThreatActor(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('threat_actor_types', ListProperty(StringProperty)), @@ -733,8 +733,8 @@ class Tool(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('tool_types', ListProperty(StringProperty)), @@ -763,8 +763,8 @@ class Vulnerability(STIXDomainObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('name', StringProperty(required=True)), ('description', StringProperty()), ('revoked', BooleanProperty(default=lambda: False)), @@ -813,8 +813,8 @@ def CustomObject(type='x-custom-type', properties=None): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ], [x for x in properties if not x[0].startswith('x_')], [ diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 059bb66..8f2de0c 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -25,8 +25,8 @@ class Relationship(STIXRelationshipObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('relationship_type', StringProperty(required=True)), ('description', StringProperty()), ('source_ref', ReferenceProperty(invalid_types=_invalid_source_target_types, spec_version='2.1', required=True)), @@ -80,8 +80,8 @@ class Sighting(STIXRelationshipObject): ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond', precision_constraint='min')), ('description', StringProperty()), ('first_seen', TimestampProperty()), ('last_seen', TimestampProperty()), From a9ac7ce838da9bb79be55eaa442399d5caa002e2 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 17 Mar 2020 18:26:57 -0400 Subject: [PATCH 02/37] pre-commit hook changes, e.g. trailing commas, import sorting, flake8 style. --- stix2/properties.py | 2 +- stix2/test/v21/test_timestamp_precision.py | 63 ++++++++++++---------- stix2/test/v21/test_versioning.py | 35 ++++++------ stix2/utils.py | 22 ++++---- 4 files changed, 67 insertions(+), 55 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 1fd1dd8..0bb1f0e 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -331,7 +331,7 @@ class TimestampProperty(Property): def clean(self, value): return parse_into_datetime( - value, self.precision, self.precision_constraint + value, self.precision, self.precision_constraint, ) diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index 5256e6c..fa8bee3 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -1,11 +1,12 @@ import datetime -import pytest -from stix2.utils import ( - Precision, PrecisionConstraint, _to_enum, parse_into_datetime, - format_datetime, STIXdatetime -) -import stix2 +import pytest + +import stix2 +from stix2.utils import ( + Precision, PrecisionConstraint, STIXdatetime, _to_enum, format_datetime, + parse_into_datetime, +) _DT = datetime.datetime.utcnow() # intentionally omit microseconds from the following. We add it in as @@ -13,24 +14,30 @@ _DT = datetime.datetime.utcnow() _DT_STR = _DT.strftime("%Y-%m-%dT%H:%M:%S") -@pytest.mark.parametrize("value, enum_type, enum_default, enum_expected", [ - ("second", Precision, None, Precision.SECOND), - ("eXaCt", PrecisionConstraint, PrecisionConstraint.MIN, - PrecisionConstraint.EXACT), - (None, Precision, Precision.MILLISECOND, Precision.MILLISECOND), - (Precision.ANY, Precision, None, Precision.ANY) -]) +@pytest.mark.parametrize( + "value, enum_type, enum_default, enum_expected", [ + ("second", Precision, None, Precision.SECOND), + ( + "eXaCt", PrecisionConstraint, PrecisionConstraint.MIN, + PrecisionConstraint.EXACT + ), + (None, Precision, Precision.MILLISECOND, Precision.MILLISECOND), + (Precision.ANY, Precision, None, Precision.ANY), + ], +) def test_to_enum(value, enum_type, enum_default, enum_expected): result = _to_enum(value, enum_type, enum_default) assert result == enum_expected -@pytest.mark.parametrize("value, err_type", [ - ("foo", KeyError), - (1, TypeError), - (PrecisionConstraint.EXACT, TypeError), - (None, TypeError) -]) +@pytest.mark.parametrize( + "value, err_type", [ + ("foo", KeyError), + (1, TypeError), + (PrecisionConstraint.EXACT, TypeError), + (None, TypeError), + ], +) def test_to_enum_errors(value, err_type): with pytest.raises(err_type): _to_enum(value, Precision) @@ -47,7 +54,7 @@ def test_stix_datetime(): sdt = STIXdatetime( dt, - precision_constraint=PrecisionConstraint.EXACT + precision_constraint=PrecisionConstraint.EXACT, ) assert sdt.precision_constraint is PrecisionConstraint.EXACT assert sdt == dt @@ -61,10 +68,11 @@ def test_stix_datetime(): (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, 123000), (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, 123456), (1234, Precision.MILLISECOND, PrecisionConstraint.EXACT, 1000), - (123, Precision.MILLISECOND, PrecisionConstraint.EXACT, 0) -]) + (123, Precision.MILLISECOND, PrecisionConstraint.EXACT, 0), +], +) def test_parse_datetime( - us, precision, precision_constraint, expected_truncated_us + us, precision, precision_constraint, expected_truncated_us, ): # complete the datetime string with microseconds @@ -73,7 +81,7 @@ def test_parse_datetime( sdt = parse_into_datetime( dt_us_str, precision=precision, - precision_constraint=precision_constraint + precision_constraint=precision_constraint, ) assert sdt.precision is precision @@ -100,7 +108,8 @@ def test_parse_datetime( (1001, Precision.MILLISECOND, PrecisionConstraint.MIN, ".001001"), (10010, Precision.MILLISECOND, PrecisionConstraint.MIN, ".01001"), (100100, Precision.MILLISECOND, PrecisionConstraint.MIN, ".1001"), -]) +], +) def test_format_datetime(us, precision, precision_constraint, expected_us_str): dt = _DT.replace(microsecond=us) @@ -109,7 +118,7 @@ def test_format_datetime(us, precision, precision_constraint, expected_us_str): sdt = STIXdatetime( dt, precision=precision, - precision_constraint=precision_constraint + precision_constraint=precision_constraint, ) s = format_datetime(sdt) assert s == expected_dt_str @@ -124,7 +133,7 @@ def test_sdo_extra_precision(): "modified" :"2015-12-21T19:59:11.0001Z", "name" :"John Smith", "identity_class" :"individual", - "spec_version": "2.1" + "spec_version": "2.1", } identity_obj = stix2.parse(identity_dict) diff --git a/stix2/test/v21/test_versioning.py b/stix2/test/v21/test_versioning.py index a1a21c8..bee0c07 100644 --- a/stix2/test/v21/test_versioning.py +++ b/stix2/test/v21/test_versioning.py @@ -1,4 +1,5 @@ import datetime + import pytest import stix2 @@ -267,28 +268,30 @@ def test_remove_custom_stix_no_custom(): assert campaign_v1.description == campaign_v2.description -@pytest.mark.parametrize("old, candidate_new, expected_new, use_stix21", [ - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", False), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", False), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.001Z", False), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999Z", "1999-08-15T00:19:07.001Z", False), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.0001Z", "1999-08-15T00:19:07.001Z", False), - ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.9999Z", "1999-08-15T00:19:08.000Z", False), +@pytest.mark.parametrize( + "old, candidate_new, expected_new, use_stix21", [ + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.0001Z", "1999-08-15T00:19:07.001Z", False), + ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.9999Z", "1999-08-15T00:19:08.000Z", False), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", True), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", True), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.000001Z", True), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999999Z", "1999-08-15T00:19:07.000001Z", True), - ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", "1999-08-15T00:19:07.000001Z", True), - ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.999999Z", "1999-08-15T00:19:07.999999Z", True), -]) + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.001Z", "1999-08-15T00:19:07.001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.000Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:06.999999Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.000Z", "1999-08-15T00:19:07.000001Z", "1999-08-15T00:19:07.000001Z", True), + ("1999-08-15T00:19:07.999Z", "1999-08-15T00:19:07.999999Z", "1999-08-15T00:19:07.999999Z", True), + ], +) def test_fudge_modified(old, candidate_new, expected_new, use_stix21): old_dt = datetime.datetime.strptime(old, "%Y-%m-%dT%H:%M:%S.%fZ") candidate_new_dt = datetime.datetime.strptime( - candidate_new, "%Y-%m-%dT%H:%M:%S.%fZ" + candidate_new, "%Y-%m-%dT%H:%M:%S.%fZ", ) expected_new_dt = datetime.datetime.strptime( - expected_new, "%Y-%m-%dT%H:%M:%S.%fZ" + expected_new, "%Y-%m-%dT%H:%M:%S.%fZ", ) fudged = stix2.utils._fudge_modified(old_dt, candidate_new_dt, use_stix21) diff --git a/stix2/utils.py b/stix2/utils.py index 7e93259..9b98d88 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -8,10 +8,10 @@ import copy import datetime as dt import enum import json -import six from dateutil import parser import pytz +import six import stix2.base @@ -82,7 +82,7 @@ def _to_enum(value, enum_type, enum_default=None): value = enum_type[value.upper()] else: raise TypeError("Not a valid {}: {}".format( - enum_type.__name__, value + enum_type.__name__, value, )) return value @@ -97,11 +97,11 @@ class STIXdatetime(dt.datetime): def __new__(cls, *args, **kwargs): precision = _to_enum( kwargs.pop("precision", Precision.ANY), - Precision + Precision, ) precision_constraint = _to_enum( kwargs.pop("precision_constraint", PrecisionConstraint.EXACT), - PrecisionConstraint + PrecisionConstraint, ) if isinstance(args[0], dt.datetime): # Allow passing in a datetime object @@ -176,7 +176,7 @@ def format_datetime(dttm): ts = zoned.strftime('%Y-%m-%dT%H:%M:%S') precision = getattr(dttm, 'precision', Precision.ANY) precision_constraint = getattr( - dttm, 'precision_constraint', PrecisionConstraint.EXACT + dttm, 'precision_constraint', PrecisionConstraint.EXACT, ) frac_seconds_str = "" @@ -212,7 +212,7 @@ def format_datetime(dttm): ts = "{}{}{}Z".format( ts, "." if frac_seconds_str else "", - frac_seconds_str + frac_seconds_str, ) return ts @@ -220,7 +220,7 @@ def format_datetime(dttm): def parse_into_datetime( value, precision=Precision.ANY, - precision_constraint=PrecisionConstraint.EXACT + precision_constraint=PrecisionConstraint.EXACT, ): """ Parse a value into a valid STIX timestamp object. Also, optionally adjust @@ -280,7 +280,7 @@ def parse_into_datetime( # else, precision == Precision.ANY: nothing for us to do. return STIXdatetime( - ts, precision=precision, precision_constraint=precision_constraint + ts, precision=precision, precision_constraint=precision_constraint, ) @@ -455,7 +455,7 @@ def new_version(data, **kwargs): if 'modified' not in kwargs: old_modified = parse_into_datetime( data["modified"], precision="millisecond", - precision_constraint=precision_constraint + precision_constraint=precision_constraint, ) new_modified = get_timestamp() @@ -466,11 +466,11 @@ def new_version(data, **kwargs): elif 'modified' in data: old_modified_property = parse_into_datetime( data.get('modified'), precision='millisecond', - precision_constraint=precision_constraint + precision_constraint=precision_constraint, ) new_modified_property = parse_into_datetime( kwargs['modified'], precision='millisecond', - precision_constraint=precision_constraint + precision_constraint=precision_constraint, ) if new_modified_property <= old_modified_property: raise InvalidValueError( From cf9aef59c293446bc1de77e6345a121c9d09367c Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 17 Mar 2020 18:28:38 -0400 Subject: [PATCH 03/37] More flake8 style fixes --- stix2/test/v21/test_timestamp_precision.py | 58 +++++++++++----------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index fa8bee3..154ebea 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -62,14 +62,14 @@ def test_stix_datetime(): @pytest.mark.parametrize( "us, precision, precision_constraint, expected_truncated_us", [ - (123456, Precision.ANY, PrecisionConstraint.EXACT, 123456), - (123456, Precision.SECOND, PrecisionConstraint.EXACT, 0), - (123456, Precision.SECOND, PrecisionConstraint.MIN, 123456), - (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, 123000), - (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, 123456), - (1234, Precision.MILLISECOND, PrecisionConstraint.EXACT, 1000), - (123, Precision.MILLISECOND, PrecisionConstraint.EXACT, 0), -], + (123456, Precision.ANY, PrecisionConstraint.EXACT, 123456), + (123456, Precision.SECOND, PrecisionConstraint.EXACT, 0), + (123456, Precision.SECOND, PrecisionConstraint.MIN, 123456), + (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, 123000), + (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, 123456), + (1234, Precision.MILLISECOND, PrecisionConstraint.EXACT, 1000), + (123, Precision.MILLISECOND, PrecisionConstraint.EXACT, 0), + ], ) def test_parse_datetime( us, precision, precision_constraint, expected_truncated_us, @@ -91,24 +91,24 @@ def test_parse_datetime( @pytest.mark.parametrize( "us, precision, precision_constraint, expected_us_str", [ - (123456, Precision.ANY, PrecisionConstraint.EXACT, ".123456"), - (123456, Precision.SECOND, PrecisionConstraint.EXACT, ""), - (123456, Precision.SECOND, PrecisionConstraint.MIN, ".123456"), - (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".123"), - (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, ".123456"), - (0, Precision.SECOND, PrecisionConstraint.MIN, ""), - (0, Precision.MILLISECOND, PrecisionConstraint.MIN, ".000"), - (0, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".000"), - (1000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".001"), - (10000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".010"), - (100000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".100"), - (1000, Precision.ANY, PrecisionConstraint.EXACT, ".001"), - (10000, Precision.ANY, PrecisionConstraint.EXACT, ".01"), - (100000, Precision.ANY, PrecisionConstraint.EXACT, ".1"), - (1001, Precision.MILLISECOND, PrecisionConstraint.MIN, ".001001"), - (10010, Precision.MILLISECOND, PrecisionConstraint.MIN, ".01001"), - (100100, Precision.MILLISECOND, PrecisionConstraint.MIN, ".1001"), -], + (123456, Precision.ANY, PrecisionConstraint.EXACT, ".123456"), + (123456, Precision.SECOND, PrecisionConstraint.EXACT, ""), + (123456, Precision.SECOND, PrecisionConstraint.MIN, ".123456"), + (123456, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".123"), + (123456, Precision.MILLISECOND, PrecisionConstraint.MIN, ".123456"), + (0, Precision.SECOND, PrecisionConstraint.MIN, ""), + (0, Precision.MILLISECOND, PrecisionConstraint.MIN, ".000"), + (0, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".000"), + (1000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".001"), + (10000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".010"), + (100000, Precision.MILLISECOND, PrecisionConstraint.EXACT, ".100"), + (1000, Precision.ANY, PrecisionConstraint.EXACT, ".001"), + (10000, Precision.ANY, PrecisionConstraint.EXACT, ".01"), + (100000, Precision.ANY, PrecisionConstraint.EXACT, ".1"), + (1001, Precision.MILLISECOND, PrecisionConstraint.MIN, ".001001"), + (10010, Precision.MILLISECOND, PrecisionConstraint.MIN, ".01001"), + (100100, Precision.MILLISECOND, PrecisionConstraint.MIN, ".1001"), + ], ) def test_format_datetime(us, precision, precision_constraint, expected_us_str): @@ -130,9 +130,9 @@ def test_sdo_extra_precision(): "type": "identity", "id": "identity--4a457eeb-6639-4aa3-be81-5930a3000c39", "created": "2015-12-21T19:59:11.000Z", - "modified" :"2015-12-21T19:59:11.0001Z", - "name" :"John Smith", - "identity_class" :"individual", + "modified": "2015-12-21T19:59:11.0001Z", + "name": "John Smith", + "identity_class": "individual", "spec_version": "2.1", } From f99665f2ba87c4687bbf04bac550773d7a5a4cfb Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 17 Mar 2020 19:45:39 -0400 Subject: [PATCH 04/37] One more comma, because python 3.8's add-trailing-comma pre-commit hook doesn't add all the commas Travis's hook script expects... --- stix2/test/v21/test_timestamp_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index 154ebea..32a7319 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -19,7 +19,7 @@ _DT_STR = _DT.strftime("%Y-%m-%dT%H:%M:%S") ("second", Precision, None, Precision.SECOND), ( "eXaCt", PrecisionConstraint, PrecisionConstraint.MIN, - PrecisionConstraint.EXACT + PrecisionConstraint.EXACT, ), (None, Precision, Precision.MILLISECOND, Precision.MILLISECOND), (Precision.ANY, Precision, None, Precision.ANY), From 6f43814918d9a16ef4cfb199b480618715d21cad Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 17 Mar 2020 20:21:09 -0400 Subject: [PATCH 05/37] Add xfail mark to a unit test which trips a Python 3.6 bug. https://bugs.python.org/issue32404 --- stix2/test/v21/test_timestamp_precision.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index 32a7319..be07f1b 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -1,6 +1,7 @@ import datetime import pytest +import sys import stix2 from stix2.utils import ( @@ -43,11 +44,19 @@ def test_to_enum_errors(value, err_type): _to_enum(value, Precision) -def test_stix_datetime(): +@pytest.mark.xfail( + sys.version_info[:2] == (3, 6), strict=True, + reason="https://bugs.python.org/issue32404", +) +def test_stix_datetime_now(): dt = STIXdatetime.utcnow() assert dt.precision is Precision.ANY assert dt.precision_constraint is PrecisionConstraint.EXACT + +def test_stix_datetime(): + dt = datetime.datetime.utcnow() + sdt = STIXdatetime(dt, precision=Precision.SECOND) assert sdt.precision is Precision.SECOND assert sdt == dt From 1741cc9f6bb3766b95a9c2e24212d4a43fd2f488 Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Tue, 17 Mar 2020 20:26:21 -0400 Subject: [PATCH 06/37] Fix import sort order for the import sorter precommit hook --- stix2/test/v21/test_timestamp_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_timestamp_precision.py b/stix2/test/v21/test_timestamp_precision.py index be07f1b..8cb9735 100644 --- a/stix2/test/v21/test_timestamp_precision.py +++ b/stix2/test/v21/test_timestamp_precision.py @@ -1,7 +1,7 @@ import datetime +import sys import pytest -import sys import stix2 from stix2.utils import ( From 9699c78ad82ccd20932af6adb91b005de86ffbae Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 19 Mar 2020 10:40:35 -0400 Subject: [PATCH 07/37] issue-365 --- stix2/base.py | 13 ++++- stix2/core.py | 9 ++-- stix2/custom.py | 85 +++++++++++++++++++++++++++------ stix2/test/v21/test_custom.py | 90 +++++++++++++++++++++++++++++++++++ stix2/utils.py | 7 ++- 5 files changed, 184 insertions(+), 20 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 4248075..75bb0ab 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -2,6 +2,7 @@ import copy import datetime as dt +import re import uuid import simplejson as json @@ -17,7 +18,7 @@ from .exceptions import ( from .markings.utils import validate from .utils import NOW, find_property_index, format_datetime, get_timestamp from .utils import new_version as _new_version -from .utils import revoke as _revoke +from .utils import revoke as _revoke, PREFIX_21_REGEX try: from collections.abc import Mapping @@ -76,6 +77,11 @@ def get_required_properties(properties): class _STIXBase(Mapping): """Base class for STIX object types""" + def get_class_version(self): + module_name = self.__class__.__module__ + module_parts = module_name.split(".") + return module_parts[1] + def object_properties(self): props = set(self._properties.keys()) custom_props = list(set(self._inner.keys()) - props) @@ -163,6 +169,11 @@ class _STIXBase(Mapping): raise ExtraPropertiesError(cls, extra_kwargs) if custom_props: self._allow_custom = True + if self.get_class_version() == "v21": + for prop_name, prop_value in custom_props.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise InvalidValueError(self.__class__, prop_name, + reason="Property names must begin with an alpha character.") # Remove any keyword arguments whose value is None or [] (i.e. empty list) setting_kwargs = {} diff --git a/stix2/core.py b/stix2/core.py index b03e3d7..cf97344 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -10,7 +10,7 @@ import stix2 from .base import _Observable, _STIXBase from .exceptions import ParseError from .markings import _MarkingsMixin -from .utils import SCO21_EXT_REGEX, TYPE_REGEX, _get_dict +from .utils import _get_dict, SCO21_EXT_REGEX, TYPE_REGEX STIX2_OBJ_MAPS = {} @@ -210,6 +210,7 @@ def _register_object(new_type, version=None): None, use latest version. """ + if version: v = 'v' + version.replace('.', '') else: @@ -248,6 +249,7 @@ def _register_observable(new_observable, version=None): None, use latest version. """ + if version: v = 'v' + version.replace('.', '') else: @@ -289,8 +291,9 @@ def _register_observable_extension( if not re.match(SCO21_EXT_REGEX, ext_type): raise ValueError( "Invalid extension type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, hyphen (-), and end " - "with '-ext'." % ext_type, + "characters a-z (lowercase ASCII), 0-9, hyphen (-), " + "must begin with an a-z character" + "and end with '-ext'." % ext_type, ) if len(ext_type) < 3 or len(ext_type) > 250: diff --git a/stix2/custom.py b/stix2/custom.py index f3c89cf..e73a16a 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -8,18 +8,28 @@ from .core import ( STIXDomainObject, _register_marking, _register_object, _register_observable, _register_observable_extension, ) -from .utils import TYPE_REGEX, get_class_hierarchy_names +from .utils import get_class_hierarchy_names, SCO21_TYPE_REGEX, TYPE_REGEX, PREFIX_21_REGEX def _custom_object_builder(cls, type, properties, version): class _CustomObject(cls, STIXDomainObject): - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type, - ) - elif len(type) < 3 or len(type) > 250: + if version == "2.0": + if not re.match(TYPE_REGEX, type): + raise ValueError( + "Invalid type type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type, + ) + else: # 2.1+ + if not re.match(SCO21_TYPE_REGEX, type): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type, + ) + + if len(type) < 3 or len(type) > 250: raise ValueError( "Invalid type name '%s': must be between 3 and 250 characters." % type, ) @@ -27,6 +37,11 @@ def _custom_object_builder(cls, type, properties, version): if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + if version == "2.1": + for prop_name, prop in properties: + if not re.match(r'^[a-z]', prop_name): + raise ValueError("Property name %s must begin with an alpha character" % prop_name) + _type = type _properties = OrderedDict(properties) @@ -41,9 +56,34 @@ def _custom_object_builder(cls, type, properties, version): def _custom_marking_builder(cls, type, properties, version): class _CustomMarking(cls, _STIXBase): + if version == "2.0": + if not re.match(TYPE_REGEX, type): + raise ValueError( + "Invalid type type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type, + ) + else: # 2.1+ + if not re.match(SCO21_TYPE_REGEX, type): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type, + ) + + if len(type) < 3 or len(type) > 250: + raise ValueError( + "Invalid type name '%s': must be between 3 and 250 characters." % type, + ) + if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + if version == "2.1": + for prop_name, prop in properties: + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name %s must begin with an alpha character." % prop_name) + _type = type _properties = OrderedDict(properties) @@ -61,12 +101,22 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= class _CustomObservable(cls, _Observable): - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid observable type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type, - ) - elif len(type) < 3 or len(type) > 250: + if version == "2.0": + if not re.match(TYPE_REGEX, type): + raise ValueError( + "Invalid type type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type, + ) + else: # 2.1+ + if not re.match(SCO21_TYPE_REGEX, type): + raise ValueError( + "Invalid observable type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type, + ) + + if len(type) < 3 or len(type) > 250: raise ValueError("Invalid observable type name '%s': must be between 3 and 250 characters." % type) if not properties or not isinstance(properties, list): @@ -89,7 +139,9 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= else: # If using STIX2.1 (or newer...), check properties ending in "_ref/s" are ReferenceProperties for prop_name, prop in properties: - if prop_name.endswith('_ref') and ('ReferenceProperty' not in get_class_hierarchy_names(prop)): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name %s must begin with an alpha character." % prop_name) + elif prop_name.endswith('_ref') and ('ReferenceProperty' not in get_class_hierarchy_names(prop)): raise ValueError( "'%s' is named like a reference property but " "is not a ReferenceProperty." % prop_name, @@ -130,6 +182,11 @@ def _custom_extension_builder(cls, observable, type, properties, version): class _CustomExtension(cls, _Extension): + if version == "2.1": + for prop_name, prop_value in prop_dict.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name %s must begin with an alpha character." % prop_name) + _type = type _properties = prop_dict diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 1e6f629..33cffd5 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -43,6 +43,19 @@ def test_identity_custom_property(): ) assert "Unexpected properties for Identity" in str(excinfo.value) + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Identity( + id=IDENTITY_ID, + created="2015-12-21T19:59:11Z", + modified="2015-12-21T19:59:11Z", + name="John Smith", + identity_class="individual", + custom_properties={ + "7foo": "bar", + }, + ) + assert "Property names must begin with an alpha character." in str(excinfo.value) + identity = stix2.v21.Identity( id=IDENTITY_ID, created="2015-12-21T19:59:11Z", @@ -184,6 +197,18 @@ def test_custom_property_in_observed_data(): assert '"x_foo": "bar"' in str(observed_data) +def test_invalid_custom_property_in_observed_data(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.File( + custom_properties={"8foo": 1}, + allow_custom=True, + name='test', + x_foo='bar', + ) + + assert "Property names must begin with an alpha character." in str(excinfo.value) + + def test_custom_property_object_in_observable_extension(): ntfs = stix2.v21.NTFSExt( allow_custom=True, @@ -293,6 +318,38 @@ def test_custom_marking_no_init_2(): assert no2.property1 == 'something' +def test_custom_marking_invalid_type_name(): + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomMarking( + 'x', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj(object): + pass # pragma: no cover + assert "Invalid type name 'x': " in str(excinfo.value) + + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomMarking( + 'x_new_marking', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(object): + pass # pragma: no cover + assert "Invalid type name 'x_new_marking':" in str(excinfo.value) + + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomMarking( + '7x-new-marking', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj3(object): + pass # pragma: no cover + assert "Invalid type name '7x-new-marking':" in str(excinfo.value) + + @stix2.v21.CustomObject( 'x-new-type', [ ('property1', stix2.properties.StringProperty(required=True)), @@ -374,6 +431,17 @@ def test_custom_object_invalid_type_name(): pass # pragma: no cover assert "Invalid type name 'x_new_object':" in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomObject( + '7x-new-object', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj3(object): + pass # pragma: no cover + assert "Invalid type name '7x-new-object':" in str(excinfo.value) + + def test_parse_custom_object_type(): nt_string = """{ @@ -500,6 +568,17 @@ def test_custom_observable_object_invalid_type_name(): pass # pragma: no cover assert "Invalid observable type name 'x_new_obs':" in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomObservable( + '7x-new-obs', [ + ('property1', stix2.properties.StringProperty()), + ], + ) + class NewObs3(object): + pass # pragma: no cover + assert "Invalid observable type name '7x-new-obs':" in str(excinfo.value) + + def test_custom_observable_object_invalid_ref_property(): with pytest.raises(ValueError) as excinfo: @@ -874,6 +953,17 @@ def test_custom_extension_invalid_type_name(): pass # pragma: no cover assert "Invalid extension type name 'x_new_ext':" in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomExtension( + stix2.v21.File, '7x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }, + ) + class Bla2Extension(): + pass # pragma: no cover + assert "Invalid extension type name '7x-new-ext':" in str(excinfo.value) + + def test_custom_extension_no_properties(): with pytest.raises(ValueError): diff --git a/stix2/utils.py b/stix2/utils.py index 7b3b6cf..4d75f8e 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -7,6 +7,7 @@ except ImportError: import copy import datetime as dt import json +import re from dateutil import parser import pytz @@ -25,8 +26,10 @@ NOW = object() # STIX object properties that cannot be modified STIX_UNMOD_PROPERTIES = ['created', 'created_by_ref', 'id', 'type'] -TYPE_REGEX = r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$' -SCO21_EXT_REGEX = r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-ext$' +TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') +SCO21_TYPE_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') +SCO21_EXT_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') +PREFIX_21_REGEX = re.compile(r'^[a-z].*') class STIXdatetime(dt.datetime): From 844ec2c3bf8ffb24033e116676c5bb1ca330667b Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 19 Mar 2020 14:16:48 -0400 Subject: [PATCH 08/37] more on issue 365 --- stix2/custom.py | 29 +++++++++++++++++++++-------- stix2/test/v21/test_custom.py | 21 ++++++++++++++++----- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index e73a16a..736d459 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -17,7 +17,7 @@ def _custom_object_builder(cls, type, properties, version): if version == "2.0": if not re.match(TYPE_REGEX, type): raise ValueError( - "Invalid type type name '%s': must only contain the " + "Invalid type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type, ) @@ -39,7 +39,7 @@ def _custom_object_builder(cls, type, properties, version): if version == "2.1": for prop_name, prop in properties: - if not re.match(r'^[a-z]', prop_name): + if not re.match(PREFIX_21_REGEX, prop_name): raise ValueError("Property name %s must begin with an alpha character" % prop_name) _type = type @@ -54,33 +54,46 @@ def _custom_object_builder(cls, type, properties, version): def _custom_marking_builder(cls, type, properties, version): + + try: + prop_dict = OrderedDict(properties) + except TypeError as e: + six.raise_from( + ValueError( + "Marking properties must be dict-like, e.g. a list " + "containing tuples. For example, " + "[('property1', IntegerProperty())]", + ), + e, + ) + class _CustomMarking(cls, _STIXBase): if version == "2.0": if not re.match(TYPE_REGEX, type): raise ValueError( - "Invalid type type name '%s': must only contain the " + "Invalid marking type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type, ) else: # 2.1+ if not re.match(SCO21_TYPE_REGEX, type): raise ValueError( - "Invalid type name '%s': must only contain the " + "Invalid marking type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " "and must begin with an a-z character" % type, ) if len(type) < 3 or len(type) > 250: raise ValueError( - "Invalid type name '%s': must be between 3 and 250 characters." % type, + "Invalid marking type name '%s': must be between 3 and 250 characters." % type, ) if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - if version == "2.1": - for prop_name, prop in properties: + if version == "2.1 ": + for prop_name, prop_value in prop_dict.items(): if not re.match(PREFIX_21_REGEX, prop_name): raise ValueError("Property name %s must begin with an alpha character." % prop_name) @@ -104,7 +117,7 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= if version == "2.0": if not re.match(TYPE_REGEX, type): raise ValueError( - "Invalid type type name '%s': must only contain the " + "Invalid observable type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % type, ) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 33cffd5..8b1224e 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -292,6 +292,19 @@ def test_identity_custom_property_edit_markings(): identity2.clear_markings('x_foo') +def test_invalid_custom_property_in_marking(): + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomMarking( + 'x-new-obj', [ + ('9property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj(): + pass + + assert "Property names must begin with an alpha character." in str(excinfo.value) + + def test_custom_marking_no_init_1(): @stix2.v21.CustomMarking( 'x-new-obj', [ @@ -327,7 +340,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj(object): pass # pragma: no cover - assert "Invalid type name 'x': " in str(excinfo.value) + assert "Invalid marking type name 'x': " in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomMarking( @@ -337,7 +350,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj2(object): pass # pragma: no cover - assert "Invalid type name 'x_new_marking':" in str(excinfo.value) + assert "Invalid marking type name 'x_new_marking':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomMarking( @@ -347,7 +360,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj3(object): pass # pragma: no cover - assert "Invalid type name '7x-new-marking':" in str(excinfo.value) + assert "Invalid marking type name '7x-new-marking':" in str(excinfo.value) @stix2.v21.CustomObject( @@ -442,7 +455,6 @@ def test_custom_object_invalid_type_name(): assert "Invalid type name '7x-new-object':" in str(excinfo.value) - def test_parse_custom_object_type(): nt_string = """{ "type": "x-new-type", @@ -579,7 +591,6 @@ def test_custom_observable_object_invalid_type_name(): assert "Invalid observable type name '7x-new-obs':" in str(excinfo.value) - def test_custom_observable_object_invalid_ref_property(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomObservable( From f60e4170fdc8a75262d78a80f411929657489414 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 19 Mar 2020 16:11:52 -0400 Subject: [PATCH 09/37] finish 365 --- stix2/base.py | 2 +- stix2/core.py | 40 +++++++++++++++++++++++-- stix2/custom.py | 55 ++++------------------------------- stix2/test/v21/test_custom.py | 39 +++++++++++++++++++++++-- stix2/utils.py | 4 +-- 5 files changed, 82 insertions(+), 58 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 75bb0ab..eabff70 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -173,7 +173,7 @@ class _STIXBase(Mapping): for prop_name, prop_value in custom_props.items(): if not re.match(PREFIX_21_REGEX, prop_name): raise InvalidValueError(self.__class__, prop_name, - reason="Property names must begin with an alpha character.") + reason="Property name '%s' must begin with an alpha character." % prop_name) # Remove any keyword arguments whose value is None or [] (i.e. empty list) setting_kwargs = {} diff --git a/stix2/core.py b/stix2/core.py index cf97344..2c388f7 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -10,7 +10,7 @@ import stix2 from .base import _Observable, _STIXBase from .exceptions import ParseError from .markings import _MarkingsMixin -from .utils import _get_dict, SCO21_EXT_REGEX, TYPE_REGEX +from .utils import _get_dict, TYPE_REGEX, PREFIX_21_REGEX, TYPE_21_REGEX STIX2_OBJ_MAPS = {} @@ -230,6 +230,36 @@ def _register_marking(new_marking, version=None): None, use latest version. """ + + type = new_marking._type + + if version == "2.0": + if not re.match(TYPE_REGEX, type): + raise ValueError( + "Invalid marking type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type, + ) + else: # 2.1+ + if not re.match(TYPE_21_REGEX, type): + raise ValueError( + "Invalid marking type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type, + ) + + if len(type) < 3 or len(type) > 250: + raise ValueError( + "Invalid marking type name '%s': must be between 3 and 250 characters." % type, + ) + + properties = new_marking._properties + + if version == "2.1": + for prop_name, prop_value in properties.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) + if version: v = 'v' + version.replace('.', '') else: @@ -276,6 +306,7 @@ def _register_observable_extension( obs_class = observable if isinstance(observable, type) else \ type(observable) ext_type = new_extension._type + properties = new_extension._properties if not issubclass(obs_class, _Observable): raise ValueError("'observable' must be a valid Observable class!") @@ -288,7 +319,7 @@ def _register_observable_extension( ext_type, ) else: # 2.1+ - if not re.match(SCO21_EXT_REGEX, ext_type): + if not re.match(TYPE_21_REGEX, ext_type): raise ValueError( "Invalid extension type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, hyphen (-), " @@ -308,6 +339,11 @@ def _register_observable_extension( ext_type, ) + if version == "2.1": + for prop_name, prop_value in properties.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) + v = 'v' + version.replace('.', '') try: diff --git a/stix2/custom.py b/stix2/custom.py index 736d459..1b26998 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -8,7 +8,7 @@ from .core import ( STIXDomainObject, _register_marking, _register_object, _register_observable, _register_observable_extension, ) -from .utils import get_class_hierarchy_names, SCO21_TYPE_REGEX, TYPE_REGEX, PREFIX_21_REGEX +from .utils import get_class_hierarchy_names, TYPE_21_REGEX, TYPE_REGEX, PREFIX_21_REGEX def _custom_object_builder(cls, type, properties, version): @@ -22,7 +22,7 @@ def _custom_object_builder(cls, type, properties, version): type, ) else: # 2.1+ - if not re.match(SCO21_TYPE_REGEX, type): + if not re.match(TYPE_21_REGEX, type): raise ValueError( "Invalid type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " @@ -40,7 +40,7 @@ def _custom_object_builder(cls, type, properties, version): if version == "2.1": for prop_name, prop in properties: if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name %s must begin with an alpha character" % prop_name) + raise ValueError("Property name '%s' must begin with an alpha character" % prop_name) _type = type _properties = OrderedDict(properties) @@ -55,48 +55,8 @@ def _custom_object_builder(cls, type, properties, version): def _custom_marking_builder(cls, type, properties, version): - try: - prop_dict = OrderedDict(properties) - except TypeError as e: - six.raise_from( - ValueError( - "Marking properties must be dict-like, e.g. a list " - "containing tuples. For example, " - "[('property1', IntegerProperty())]", - ), - e, - ) - class _CustomMarking(cls, _STIXBase): - if version == "2.0": - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid marking type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - type, - ) - else: # 2.1+ - if not re.match(SCO21_TYPE_REGEX, type): - raise ValueError( - "Invalid marking type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " - "and must begin with an a-z character" % type, - ) - - if len(type) < 3 or len(type) > 250: - raise ValueError( - "Invalid marking type name '%s': must be between 3 and 250 characters." % type, - ) - - if not properties or not isinstance(properties, list): - raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - - if version == "2.1 ": - for prop_name, prop_value in prop_dict.items(): - if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name %s must begin with an alpha character." % prop_name) - _type = type _properties = OrderedDict(properties) @@ -122,7 +82,7 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= type, ) else: # 2.1+ - if not re.match(SCO21_TYPE_REGEX, type): + if not re.match(TYPE_21_REGEX, type): raise ValueError( "Invalid observable type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " @@ -153,7 +113,7 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= # If using STIX2.1 (or newer...), check properties ending in "_ref/s" are ReferenceProperties for prop_name, prop in properties: if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name %s must begin with an alpha character." % prop_name) + raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) elif prop_name.endswith('_ref') and ('ReferenceProperty' not in get_class_hierarchy_names(prop)): raise ValueError( "'%s' is named like a reference property but " @@ -195,11 +155,6 @@ def _custom_extension_builder(cls, observable, type, properties, version): class _CustomExtension(cls, _Extension): - if version == "2.1": - for prop_name, prop_value in prop_dict.items(): - if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name %s must begin with an alpha character." % prop_name) - _type = type _properties = prop_dict diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 8b1224e..c81d914 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -9,6 +9,8 @@ import stix2.v21 from ...exceptions import InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID +# Custom Properties in SDOs + IDENTITY_CUSTOM_PROP = stix2.v21.Identity( name="John Smith", identity_class="individual", @@ -54,7 +56,7 @@ def test_identity_custom_property(): "7foo": "bar", }, ) - assert "Property names must begin with an alpha character." in str(excinfo.value) + assert "must begin with an alpha character." in str(excinfo.value) identity = stix2.v21.Identity( id=IDENTITY_ID, @@ -178,6 +180,7 @@ def test_custom_properties_dict_in_bundled_object(): assert bundle.objects[0].x_foo == "bar" assert '"x_foo": "bar"' in str(bundle) +# Custom properties in SCOs def test_custom_property_in_observed_data(): artifact = stix2.v21.File( @@ -206,7 +209,7 @@ def test_invalid_custom_property_in_observed_data(): x_foo='bar', ) - assert "Property names must begin with an alpha character." in str(excinfo.value) + assert "must begin with an alpha character." in str(excinfo.value) def test_custom_property_object_in_observable_extension(): @@ -270,6 +273,7 @@ def test_identity_custom_property_revoke(): identity = IDENTITY_CUSTOM_PROP.revoke() assert identity.x_foo == "bar" +# Custom markings def test_identity_custom_property_edit_markings(): marking_obj = stix2.v21.MarkingDefinition( @@ -302,7 +306,7 @@ def test_invalid_custom_property_in_marking(): class NewObj(): pass - assert "Property names must begin with an alpha character." in str(excinfo.value) + assert "must begin with an alpha character." in str(excinfo.value) def test_custom_marking_no_init_1(): @@ -362,6 +366,7 @@ def test_custom_marking_invalid_type_name(): pass # pragma: no cover assert "Invalid marking type name '7x-new-marking':" in str(excinfo.value) +# Custom Objects @stix2.v21.CustomObject( 'x-new-type', [ @@ -492,6 +497,7 @@ def test_parse_unregistered_custom_object_type_w_allow_custom(): custom_obj = stix2.parse(nt_string, version="2.1", allow_custom=True) assert custom_obj["type"] == "x-foobar-observable" +# Custom SCOs @stix2.v21.CustomObservable( 'x-new-observable', [ @@ -559,6 +565,18 @@ def test_custom_observable_object_no_init_2(): assert no2.property1 == 'something' +def test_invalid_custom_property_in_custom_observable_object(): + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomObservable( + 'x-new-sco', [ + ('5property1', stix2.properties.StringProperty()), + ], + ) + class NewObs(object): + pass # pragma: no cover + assert "must begin with an alpha character." in str(excinfo.value) + + def test_custom_observable_object_invalid_type_name(): with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomObservable( @@ -826,6 +844,7 @@ def test_custom_observable_object_no_id_contrib_props(): assert uuid_obj.variant == uuid.RFC_4122 assert uuid_obj.version == 4 +# Custom Extensions @stix2.v21.CustomExtension( stix2.v21.DomainName, 'x-new-ext', [ @@ -1023,6 +1042,20 @@ def test_custom_extension_no_init_2(): assert ne2.property1 == "foobar" +def test_invalid_custom_property_in_extension(): + with pytest.raises(ValueError) as excinfo: + @stix2.v21.CustomExtension( + stix2.v21.DomainName, 'x-new3-ext', [ + ('6property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewExt(): + pass + + assert "must begin with an alpha character." in str(excinfo.value) + + + def test_parse_observable_with_custom_extension(): input_str = """{ "type": "domain-name", diff --git a/stix2/utils.py b/stix2/utils.py index 4d75f8e..abec9a9 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -27,8 +27,8 @@ NOW = object() STIX_UNMOD_PROPERTIES = ['created', 'created_by_ref', 'id', 'type'] TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') -SCO21_TYPE_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') -SCO21_EXT_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') +TYPE_21_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') +EXT_21_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') PREFIX_21_REGEX = re.compile(r'^[a-z].*') From fe919049b8b59a5e5a5a734cd573d2e383125f18 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 19 Mar 2020 16:43:37 -0400 Subject: [PATCH 10/37] fix marking test --- stix2/custom.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stix2/custom.py b/stix2/custom.py index 1b26998..8a08f0e 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -57,6 +57,9 @@ def _custom_marking_builder(cls, type, properties, version): class _CustomMarking(cls, _STIXBase): + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + _type = type _properties = OrderedDict(properties) From 6e4151aeebba03c112e6062c253c135d77364688 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 19 Mar 2020 16:49:46 -0400 Subject: [PATCH 11/37] flaky --- stix2/base.py | 12 ++++++++---- stix2/core.py | 2 +- stix2/custom.py | 4 +++- stix2/test/v21/test_custom.py | 7 +++++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index eabff70..8bffa3a 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -16,9 +16,11 @@ from .exceptions import ( MissingPropertiesError, MutuallyExclusivePropertiesError, ) from .markings.utils import validate -from .utils import NOW, find_property_index, format_datetime, get_timestamp +from .utils import ( + NOW, PREFIX_21_REGEX, find_property_index, format_datetime, get_timestamp, +) from .utils import new_version as _new_version -from .utils import revoke as _revoke, PREFIX_21_REGEX +from .utils import revoke as _revoke try: from collections.abc import Mapping @@ -172,8 +174,10 @@ class _STIXBase(Mapping): if self.get_class_version() == "v21": for prop_name, prop_value in custom_props.items(): if not re.match(PREFIX_21_REGEX, prop_name): - raise InvalidValueError(self.__class__, prop_name, - reason="Property name '%s' must begin with an alpha character." % prop_name) + raise InvalidValueError( + self.__class__, prop_name, + reason="Property name '%s' must begin with an alpha character." % prop_name, + ) # Remove any keyword arguments whose value is None or [] (i.e. empty list) setting_kwargs = {} diff --git a/stix2/core.py b/stix2/core.py index 2c388f7..bd2b524 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -10,7 +10,7 @@ import stix2 from .base import _Observable, _STIXBase from .exceptions import ParseError from .markings import _MarkingsMixin -from .utils import _get_dict, TYPE_REGEX, PREFIX_21_REGEX, TYPE_21_REGEX +from .utils import PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, _get_dict STIX2_OBJ_MAPS = {} diff --git a/stix2/custom.py b/stix2/custom.py index 8a08f0e..bd2fe1d 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -8,7 +8,9 @@ from .core import ( STIXDomainObject, _register_marking, _register_object, _register_observable, _register_observable_extension, ) -from .utils import get_class_hierarchy_names, TYPE_21_REGEX, TYPE_REGEX, PREFIX_21_REGEX +from .utils import ( + PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, get_class_hierarchy_names, +) def _custom_object_builder(cls, type, properties, version): diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index c81d914..168c7fe 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -182,6 +182,7 @@ def test_custom_properties_dict_in_bundled_object(): # Custom properties in SCOs + def test_custom_property_in_observed_data(): artifact = stix2.v21.File( allow_custom=True, @@ -275,6 +276,7 @@ def test_identity_custom_property_revoke(): # Custom markings + def test_identity_custom_property_edit_markings(): marking_obj = stix2.v21.MarkingDefinition( id=MARKING_DEFINITION_ID, @@ -368,6 +370,7 @@ def test_custom_marking_invalid_type_name(): # Custom Objects + @stix2.v21.CustomObject( 'x-new-type', [ ('property1', stix2.properties.StringProperty(required=True)), @@ -499,6 +502,7 @@ def test_parse_unregistered_custom_object_type_w_allow_custom(): # Custom SCOs + @stix2.v21.CustomObservable( 'x-new-observable', [ ('property1', stix2.properties.StringProperty(required=True)), @@ -846,6 +850,7 @@ def test_custom_observable_object_no_id_contrib_props(): # Custom Extensions + @stix2.v21.CustomExtension( stix2.v21.DomainName, 'x-new-ext', [ ('property1', stix2.properties.StringProperty(required=True)), @@ -994,7 +999,6 @@ def test_custom_extension_invalid_type_name(): assert "Invalid extension type name '7x-new-ext':" in str(excinfo.value) - def test_custom_extension_no_properties(): with pytest.raises(ValueError): @stix2.v21.CustomExtension(stix2.v21.DomainName, 'x-new2-ext', None) @@ -1055,7 +1059,6 @@ def test_invalid_custom_property_in_extension(): assert "must begin with an alpha character." in str(excinfo.value) - def test_parse_observable_with_custom_extension(): input_str = """{ "type": "domain-name", From 2c4e47de562a5b8b7df93267819e2d09e8276ac4 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 20 Mar 2020 11:56:09 -0400 Subject: [PATCH 12/37] remove leading - from type name re --- stix2/core.py | 4 ++-- stix2/test/v21/test_custom.py | 15 ++++++++++++++- stix2/utils.py | 4 ++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index bd2b524..6f05bf9 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -10,7 +10,7 @@ import stix2 from .base import _Observable, _STIXBase from .exceptions import ParseError from .markings import _MarkingsMixin -from .utils import PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, _get_dict +from .utils import PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, EXT_21_REGEX, _get_dict STIX2_OBJ_MAPS = {} @@ -319,7 +319,7 @@ def _register_observable_extension( ext_type, ) else: # 2.1+ - if not re.match(TYPE_21_REGEX, ext_type): + if not re.match(EXT_21_REGEX, ext_type): raise ValueError( "Invalid extension type name '%s': must only contain the " "characters a-z (lowercase ASCII), 0-9, hyphen (-), " diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 168c7fe..2bdca05 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -58,6 +58,19 @@ def test_identity_custom_property(): ) assert "must begin with an alpha character." in str(excinfo.value) + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.v21.Identity( + id=IDENTITY_ID, + created="2015-12-21T19:59:11Z", + modified="2015-12-21T19:59:11Z", + name="John Smith", + identity_class="individual", + custom_properties={ + "_foo": "bar", + }, + ) + assert "must begin with an alpha character." in str(excinfo.value) + identity = stix2.v21.Identity( id=IDENTITY_ID, created="2015-12-21T19:59:11Z", @@ -210,7 +223,7 @@ def test_invalid_custom_property_in_observed_data(): x_foo='bar', ) - assert "must begin with an alpha character." in str(excinfo.value) + assert "must begin with an alpha character." in str(excinfo.value) def test_custom_property_object_in_observable_extension(): diff --git a/stix2/utils.py b/stix2/utils.py index abec9a9..f2bac52 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -27,8 +27,8 @@ NOW = object() STIX_UNMOD_PROPERTIES = ['created', 'created_by_ref', 'id', 'type'] TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') -TYPE_21_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') -EXT_21_REGEX = re.compile(r'^\-?([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') +TYPE_21_REGEX = re.compile(r'([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') +EXT_21_REGEX = re.compile(r'([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') PREFIX_21_REGEX = re.compile(r'^[a-z].*') From 9e5e998c3d83c0706eb7f7b8097cbc78f4772e25 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 20 Mar 2020 12:49:20 -0400 Subject: [PATCH 13/37] don't allow leading '_' on custom properties, whenever allow_custom is true --- stix2/base.py | 10 +++++++-- stix2/test/v21/test_custom.py | 38 +++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 8bffa3a..e018059 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -169,10 +169,16 @@ class _STIXBase(Mapping): extra_kwargs = list(set(kwargs) - set(self._properties)) if extra_kwargs: raise ExtraPropertiesError(cls, extra_kwargs) - if custom_props: + else: + # because allow_custom is true, any extra kwargs are custom + extra_kwargs = list(set(kwargs) - set(self._properties)) + + if custom_props or extra_kwargs: self._allow_custom = True if self.get_class_version() == "v21": - for prop_name, prop_value in custom_props.items(): + all_custom_prop_names = extra_kwargs + all_custom_prop_names.extend(list(custom_props.keys())) + for prop_name in all_custom_prop_names: if not re.match(PREFIX_21_REGEX, prop_name): raise InvalidValueError( self.__class__, prop_name, diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 2bdca05..ef57a9b 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -20,6 +20,18 @@ IDENTITY_CUSTOM_PROP = stix2.v21.Identity( def test_identity_custom_property(): + identity = stix2.v21.Identity( + id=IDENTITY_ID, + created="2015-12-21T19:59:11Z", + modified="2015-12-21T19:59:11Z", + name="John Smith", + identity_class="individual", + custom_properties={ + "foo": "bar", + }, + ) + assert identity.foo == "bar" + with pytest.raises(ValueError) as excinfo: stix2.v21.Identity( id=IDENTITY_ID, @@ -45,6 +57,8 @@ def test_identity_custom_property(): ) assert "Unexpected properties for Identity" in str(excinfo.value) + # leading numeric character is illegal in 2.1 + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Identity( id=IDENTITY_ID, @@ -58,6 +72,8 @@ def test_identity_custom_property(): ) assert "must begin with an alpha character." in str(excinfo.value) + # leading "_" is illegal in 2.1 + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: stix2.v21.Identity( id=IDENTITY_ID, @@ -71,17 +87,17 @@ def test_identity_custom_property(): ) assert "must begin with an alpha character." in str(excinfo.value) - identity = stix2.v21.Identity( - id=IDENTITY_ID, - created="2015-12-21T19:59:11Z", - modified="2015-12-21T19:59:11Z", - name="John Smith", - identity_class="individual", - custom_properties={ - "foo": "bar", - }, - ) - assert identity.foo == "bar" + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + identity = stix2.v21.Identity( + id=IDENTITY_ID, + created="2015-12-21T19:59:11Z", + modified="2015-12-21T19:59:11Z", + name="John Smith", + identity_class="individual", + _x_foo="bar", + allow_custom=True, + ) + assert "must begin with an alpha character." in str(excinfo.value) def test_identity_custom_property_invalid(): From d8a9fc23065336bad2e75de0a154889162e1525e Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 20 Mar 2020 13:15:42 -0400 Subject: [PATCH 14/37] flaky --- stix2/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stix2/core.py b/stix2/core.py index 6f05bf9..055a4a6 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -10,7 +10,9 @@ import stix2 from .base import _Observable, _STIXBase from .exceptions import ParseError from .markings import _MarkingsMixin -from .utils import PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, EXT_21_REGEX, _get_dict +from .utils import ( + EXT_21_REGEX, PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, _get_dict, +) STIX2_OBJ_MAPS = {} From 2dea4caf00e5ad3706cd26f67734285a64dc7a81 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 20 Mar 2020 14:24:16 -0400 Subject: [PATCH 15/37] fix re so they begin with ^ --- stix2/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index f2bac52..da04a3e 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -27,8 +27,8 @@ NOW = object() STIX_UNMOD_PROPERTIES = ['created', 'created_by_ref', 'id', 'type'] TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') -TYPE_21_REGEX = re.compile(r'([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') -EXT_21_REGEX = re.compile(r'([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') +TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') +EXT_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') PREFIX_21_REGEX = re.compile(r'^[a-z].*') From 1260c7b45eaa351442015212f92904ee83047be3 Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 20 Mar 2020 16:49:20 -0400 Subject: [PATCH 16/37] Fix existing tests and add new tests. Fixes #363 --- stix2/core.py | 6 +- stix2/test/v20/test_core.py | 49 --------------- stix2/test/v20/test_custom.py | 69 ++++++++++++++++++++- stix2/test/v20/test_datastore_filesystem.py | 6 +- stix2/test/v20/test_datastore_memory.py | 2 +- stix2/test/v20/test_properties.py | 2 +- stix2/test/v21/test_core.py | 47 -------------- stix2/test/v21/test_custom.py | 67 +++++++++++++++++++- stix2/test/v21/test_datastore_filesystem.py | 4 +- stix2/test/v21/test_datastore_memory.py | 2 +- stix2/test/v21/test_properties.py | 2 +- 11 files changed, 149 insertions(+), 107 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index 0d1fee5..c9282d2 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -8,7 +8,7 @@ import re import stix2 from .base import _STIXBase -from .exceptions import ParseError +from .exceptions import DuplicateObjectRegistrationError, ParseError from .markings import _MarkingsMixin from .utils import _get_dict @@ -217,6 +217,8 @@ def _register_object(new_type, version=None): v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') OBJ_MAP = STIX2_OBJ_MAPS[v]['objects'] + if new_type._type in OBJ_MAP.keys(): + raise DuplicateObjectRegistrationError("An object with type '%s' already exists and cannot be created again." % new_type._type) OBJ_MAP[new_type._type] = new_type @@ -255,6 +257,8 @@ def _register_observable(new_observable, version=None): v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') OBJ_MAP_OBSERVABLE = STIX2_OBJ_MAPS[v]['observables'] + if new_observable._type in OBJ_MAP_OBSERVABLE.keys(): + raise DuplicateObjectRegistrationError("An observable with type '%s' already exists and cannot be created again." % new_observable._type) OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable diff --git a/stix2/test/v20/test_core.py b/stix2/test/v20/test_core.py index d2efa22..2808ba5 100644 --- a/stix2/test/v20/test_core.py +++ b/stix2/test/v20/test_core.py @@ -68,17 +68,6 @@ def test_parse_observable_with_no_version(): assert v in str(obs_obj.__class__) -def test_register_object_with_version(): - bundle = core.dict_to_stix2(BUNDLE, version='2.0') - core._register_object(bundle.objects[0].__class__, version='2.0') - v = 'v20' - - assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] - # spec_version is not in STIX 2.0, and is required in 2.1, so this - # suffices as a test for a STIX 2.0 object. - assert "spec_version" not in bundle.objects[0] - - def test_register_marking_with_version(): core._register_marking(stix2.v20.TLP_WHITE.__class__, version='2.0') v = 'v20' @@ -97,44 +86,6 @@ def test_register_marking_with_no_version(): assert v in str(stix2.v20.TLP_WHITE.__class__) -def test_register_observable_with_version(): - observed_data = stix2.v20.ObservedData( - id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - created_by_ref=IDENTITY_ID, - created="2016-04-06T19:58:16.000Z", - modified="2016-04-06T19:58:16.000Z", - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=50, - objects={ - "0": { - "name": "foo.exe", - "type": "file", - "extensions": { - "ntfs-ext": { - "alternate_data_streams": [ - { - "name": "second.stream", - "size": 25536, - }, - ], - }, - }, - }, - "1": { - "type": "directory", - "path": "/usr/home", - "contains_refs": ["0"], - }, - }, - ) - core._register_observable(observed_data.objects['0'].__class__, version='2.0') - v = 'v20' - - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] - assert v in str(observed_data.objects['0'].__class__) - - def test_register_observable_extension_with_version(): observed_data = stix2.v20.ObservedData( id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index ce1aac3..8bc6ddd 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -1,6 +1,7 @@ import pytest import stix2 +from stix2 import core import stix2.v20 from ...exceptions import InvalidValueError @@ -449,7 +450,7 @@ def test_custom_observable_raises_exception(): def test_custom_observable_object_no_init_1(): @stix2.v20.CustomObservable( - 'x-new-observable', [ + 'x-new-observable-1', [ ('property1', stix2.properties.StringProperty()), ], ) @@ -1014,3 +1015,69 @@ def test_custom_object_nested_dictionary(data): ) assert data == str(example) + + +@stix2.v20.CustomObject( + 'x-new-type-2', [ + ('property1', stix2.properties.StringProperty()), + ('property2', stix2.properties.IntegerProperty()), + ], +) +class NewType2(object): + pass + + +def test_register_custom_object_with_version(): + custom_obj_1 = { + "type": "x-new-type-2", + "id": "x-new-type-2--00000000-0000-4000-8000-000000000007", + } + + cust_obj_1 = core.dict_to_stix2(custom_obj_1, version='2.0') + v = 'v20' + + assert cust_obj_1.type in core.STIX2_OBJ_MAPS[v]['objects'] + # spec_version is not in STIX 2.0, and is required in 2.1, so this + # suffices as a test for a STIX 2.0 object. + assert "spec_version" not in cust_obj_1 + + +def test_register_duplicate_object_with_version(): + with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + @stix2.v20.CustomObject( + 'x-new-type-2', [ + ('property1', stix2.properties.StringProperty()), + ('property2', stix2.properties.IntegerProperty()), + ], + ) + class NewType2(object): + pass + assert "An object with type 'x-new-type-2' already exists and cannot be created again." in str(excinfo.value) + + +@stix2.v20.CustomObservable( + 'x-new-observable-2', [ + ('property1', stix2.properties.StringProperty()), + ], +) +class NewObservable2(object): + pass + + +def test_register_observable_with_version(): + custom_obs = NewObservable2(property1="Test Observable") + v = 'v20' + + assert custom_obs.type in core.STIX2_OBJ_MAPS[v]['observables'] + + +def test_register_duplicate_observable_with_version(): + with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + @stix2.v20.CustomObservable( + 'x-new-observable-2', [ + ('property1', stix2.properties.StringProperty()), + ], + ) + class NewObservable2(object): + pass + assert "An observable with type 'x-new-observable-2' already exists and cannot be created again." in str(excinfo.value) diff --git a/stix2/test/v20/test_datastore_filesystem.py b/stix2/test/v20/test_datastore_filesystem.py index 317f927..b7e9aca 100644 --- a/stix2/test/v20/test_datastore_filesystem.py +++ b/stix2/test/v20/test_datastore_filesystem.py @@ -501,6 +501,7 @@ def test_filesystem_store_query_single_filter(fs_store): def test_filesystem_store_empty_query(fs_store): results = fs_store.query() # returns all + print (results) assert len(results) == 30 assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] @@ -515,6 +516,7 @@ def test_filesystem_store_query_multiple_filters(fs_store): def test_filesystem_store_query_dont_include_type_folder(fs_store): results = fs_store.query(stix2.Filter("type", "!=", "tool")) + print (results) assert len(results) == 28 @@ -635,7 +637,7 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): def test_filesystem_custom_object(fs_store): @stix2.v20.CustomObject( - 'x-new-obj', [ + 'x-new-obj-2', [ ('property1', stix2.properties.StringProperty(required=True)), ], ) @@ -650,7 +652,7 @@ def test_filesystem_custom_object(fs_store): assert newobj_r["property1"] == 'something' # remove dir - shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) + shutil.rmtree(os.path.join(FS_PATH, "x-new-obj-2"), True) def test_relationships(rel_fs_store): diff --git a/stix2/test/v20/test_datastore_memory.py b/stix2/test/v20/test_datastore_memory.py index 7852746..28d8e52 100644 --- a/stix2/test/v20/test_datastore_memory.py +++ b/stix2/test/v20/test_datastore_memory.py @@ -329,7 +329,7 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store): def test_memory_store_custom_object(mem_store): @CustomObject( - 'x-new-obj', [ + 'x-new-obj-3', [ ('property1', properties.StringProperty(required=True)), ], ) diff --git a/stix2/test/v20/test_properties.py b/stix2/test/v20/test_properties.py index f71d829..1d1474a 100644 --- a/stix2/test/v20/test_properties.py +++ b/stix2/test/v20/test_properties.py @@ -392,7 +392,7 @@ def test_dictionary_property_invalid(d): def test_property_list_of_dictionary(): @stix2.v20.CustomObject( - 'x-new-obj', [ + 'x-new-obj-4', [ ('property1', ListProperty(DictionaryProperty(spec_version="2.0"), required=True)), ], ) diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index 2018395..f9f9de6 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -73,15 +73,6 @@ def test_parse_observable_with_no_version(): assert v in str(obs_obj.__class__) -def test_register_object_with_version(): - bundle = core.dict_to_stix2(BUNDLE, version='2.1') - core._register_object(bundle.objects[0].__class__) - v = 'v21' - - assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] - assert bundle.objects[0].spec_version == "2.1" - - def test_register_marking_with_version(): core._register_marking(stix2.v21.TLP_WHITE.__class__, version='2.1') v = 'v21' @@ -100,44 +91,6 @@ def test_register_marking_with_no_version(): assert v in str(stix2.v21.TLP_WHITE.__class__) -def test_register_observable_with_default_version(): - observed_data = stix2.v21.ObservedData( - id=OBSERVED_DATA_ID, - created_by_ref=IDENTITY_ID, - created="2016-04-06T19:58:16.000Z", - modified="2016-04-06T19:58:16.000Z", - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=50, - objects={ - "0": { - "name": "foo.exe", - "type": "file", - "extensions": { - "ntfs-ext": { - "alternate_data_streams": [ - { - "name": "second.stream", - "size": 25536, - }, - ], - }, - }, - }, - "1": { - "type": "directory", - "path": "/usr/home", - "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], - }, - }, - ) - core._register_observable(observed_data.objects['0'].__class__) - v = 'v21' - - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] - assert v in str(observed_data.objects['0'].__class__) - - def test_register_observable_extension_with_default_version(): observed_data = stix2.v21.ObservedData( id=OBSERVED_DATA_ID, diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index b46288d..9bd0afa 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -455,7 +455,7 @@ def test_custom_observable_raises_exception(): def test_custom_observable_object_no_init_1(): @stix2.v21.CustomObservable( - 'x-new-observable', [ + 'x-new-observable-2', [ ('property1', stix2.properties.StringProperty()), ], ) @@ -1068,3 +1068,68 @@ def test_custom_object_nested_dictionary(data): ) assert data == str(example) + + +@stix2.v21.CustomObject( + 'x-new-type-2', [ + ('property1', stix2.properties.StringProperty()), + ('property2', stix2.properties.IntegerProperty()), + ], +) +class NewType3(object): + pass + + +def test_register_custom_object_with_version(): + custom_obj_1 = { + "type": "x-new-type-2", + "id": "x-new-type-2--00000000-0000-4000-8000-000000000007", + "spec_version": "2.1", + } + + cust_obj_1 = stix2.core.dict_to_stix2(custom_obj_1, version='2.1') + v = 'v21' + + assert cust_obj_1.type in stix2.core.STIX2_OBJ_MAPS[v]['objects'] + assert cust_obj_1.spec_version == "2.1" + + +def test_register_duplicate_object_with_version(): + with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + @stix2.v21.CustomObject( + 'x-new-type-2', [ + ('property1', stix2.properties.StringProperty()), + ('property2', stix2.properties.IntegerProperty()), + ], + ) + class NewType2(object): + pass + assert "An object with type 'x-new-type-2' already exists and cannot be created again." in str(excinfo.value) + + +@stix2.v21.CustomObservable( + 'x-new-observable-3', [ + ('property1', stix2.properties.StringProperty()), + ], +) +class NewObservable3(object): + pass + + +def test_register_observable_with_version(): + custom_obs = NewObservable3(property1="Test Observable") + v = 'v21' + + assert custom_obs.type in stix2.core.STIX2_OBJ_MAPS[v]['observables'] + + +def test_register_duplicate_observable_with_version(): + with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + @stix2.v21.CustomObservable( + 'x-new-observable-2', [ + ('property1', stix2.properties.StringProperty()), + ], + ) + class NewObservable2(object): + pass + assert "An observable with type 'x-new-observable-2' already exists and cannot be created again." in str(excinfo.value) diff --git a/stix2/test/v21/test_datastore_filesystem.py b/stix2/test/v21/test_datastore_filesystem.py index 3eb8aaa..123fd7a 100644 --- a/stix2/test/v21/test_datastore_filesystem.py +++ b/stix2/test/v21/test_datastore_filesystem.py @@ -656,7 +656,7 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): def test_filesystem_custom_object(fs_store): @stix2.v21.CustomObject( - 'x-new-obj', [ + 'x-new-obj-2', [ ('property1', stix2.properties.StringProperty(required=True)), ], ) @@ -671,7 +671,7 @@ def test_filesystem_custom_object(fs_store): assert newobj_r["property1"] == 'something' # remove dir - shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) + shutil.rmtree(os.path.join(FS_PATH, "x-new-obj-2"), True) def test_relationships(rel_fs_store): diff --git a/stix2/test/v21/test_datastore_memory.py b/stix2/test/v21/test_datastore_memory.py index e07943c..60f577e 100644 --- a/stix2/test/v21/test_datastore_memory.py +++ b/stix2/test/v21/test_datastore_memory.py @@ -344,7 +344,7 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store): def test_memory_store_custom_object(mem_store): @CustomObject( - 'x-new-obj', [ + 'x-new-obj-3', [ ('property1', properties.StringProperty(required=True)), ], ) diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 50bce17..31dd941 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -404,7 +404,7 @@ def test_dictionary_property_invalid(d): def test_property_list_of_dictionary(): @stix2.v21.CustomObject( - 'x-new-obj', [ + 'x-new-obj-4', [ ('property1', ListProperty(DictionaryProperty(spec_version='2.1'), required=True)), ], ) From b06bc1afc128a3f08c2208c229abac3dd2ebe8cc Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 20 Mar 2020 17:32:18 -0400 Subject: [PATCH 17/37] Fix import issues --- stix2/core.py | 2 +- stix2/exceptions.py | 7 +++++++ stix2/test/v20/test_custom.py | 6 +++--- stix2/test/v21/test_custom.py | 6 +++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index bb583ba..ffe77f2 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -8,7 +8,7 @@ import re import stix2 from .base import _Observable, _STIXBase -from .exceptions import ParseError +from .exceptions import DuplicateObjectRegistrationError, ParseError from .markings import _MarkingsMixin from .utils import SCO21_EXT_REGEX, TYPE_REGEX, _get_dict diff --git a/stix2/exceptions.py b/stix2/exceptions.py index d2ec3fc..f65f48b 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -233,3 +233,10 @@ class STIXDeprecationWarning(DeprecationWarning): Represents usage of a deprecated component of a STIX specification. """ pass + + +class DuplicateObjectRegistrationError(STIXError): + """An object (or observable) with the same type as an existing object (or observable) is being registered""" + + def __init__(self, msg): + super(DuplicateObjectRegistrationError, self).__init__(msg) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index f3fa59a..19cb28a 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -4,7 +4,7 @@ import stix2 from stix2 import core import stix2.v20 -from ...exceptions import InvalidValueError +from ...exceptions import DuplicateObjectRegistrationError, InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v20.Identity( @@ -1040,7 +1040,7 @@ def test_register_custom_object_with_version(): def test_register_duplicate_object_with_version(): - with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateObjectRegistrationError) as excinfo: @stix2.v20.CustomObject( 'x-new-type-2', [ ('property1', stix2.properties.StringProperty()), @@ -1069,7 +1069,7 @@ def test_register_observable_with_version(): def test_register_duplicate_observable_with_version(): - with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateObjectRegistrationError) as excinfo: @stix2.v20.CustomObservable( 'x-new-observable-2', [ ('property1', stix2.properties.StringProperty()), diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index e2b7f22..92cacf3 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -6,7 +6,7 @@ import stix2 import stix2.base import stix2.v21 -from ...exceptions import InvalidValueError +from ...exceptions import DuplicateObjectRegistrationError, InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v21.Identity( @@ -1092,7 +1092,7 @@ def test_register_custom_object_with_version(): def test_register_duplicate_object_with_version(): - with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateObjectRegistrationError) as excinfo: @stix2.v21.CustomObject( 'x-new-type-2', [ ('property1', stix2.properties.StringProperty()), @@ -1121,7 +1121,7 @@ def test_register_observable_with_version(): def test_register_duplicate_observable_with_version(): - with pytest.raises(stix2.exceptions.DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateObjectRegistrationError) as excinfo: @stix2.v21.CustomObservable( 'x-new-observable-2', [ ('property1', stix2.properties.StringProperty()), From 1a1ad90388fad847963b2af4db14e2e92a8af71f Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 20 Mar 2020 17:37:15 -0400 Subject: [PATCH 18/37] Fixes #363 --- stix2/test/v20/test_datastore_filesystem.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stix2/test/v20/test_datastore_filesystem.py b/stix2/test/v20/test_datastore_filesystem.py index b7e9aca..25207dc 100644 --- a/stix2/test/v20/test_datastore_filesystem.py +++ b/stix2/test/v20/test_datastore_filesystem.py @@ -501,7 +501,6 @@ def test_filesystem_store_query_single_filter(fs_store): def test_filesystem_store_empty_query(fs_store): results = fs_store.query() # returns all - print (results) assert len(results) == 30 assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] @@ -516,7 +515,6 @@ def test_filesystem_store_query_multiple_filters(fs_store): def test_filesystem_store_query_dont_include_type_folder(fs_store): results = fs_store.query(stix2.Filter("type", "!=", "tool")) - print (results) assert len(results) == 28 From e31634c32b0fee2d4344fe454efe1215773e26a5 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Sat, 21 Mar 2020 22:22:36 -0400 Subject: [PATCH 19/37] Rework spec version detection for _STIXBase objs --- stix2/base.py | 7 +------ stix2/custom.py | 30 +++++++++++++++--------------- stix2/v20/base.py | 24 ++++++++++++++++++++++++ stix2/v20/bundle.py | 4 ++-- stix2/v20/common.py | 16 ++++++++-------- stix2/v20/observables.py | 22 +++++++++++----------- stix2/v20/sdo.py | 4 ++-- stix2/v20/sro.py | 2 +- stix2/v21/base.py | 24 ++++++++++++++++++++++++ stix2/v21/bundle.py | 4 ++-- stix2/v21/common.py | 18 +++++++++--------- stix2/v21/observables.py | 22 +++++++++++----------- stix2/v21/sdo.py | 4 ++-- stix2/v21/sro.py | 2 +- 14 files changed, 113 insertions(+), 70 deletions(-) create mode 100644 stix2/v20/base.py create mode 100644 stix2/v21/base.py diff --git a/stix2/base.py b/stix2/base.py index e018059..5af3607 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -79,11 +79,6 @@ def get_required_properties(properties): class _STIXBase(Mapping): """Base class for STIX object types""" - def get_class_version(self): - module_name = self.__class__.__module__ - module_parts = module_name.split(".") - return module_parts[1] - def object_properties(self): props = set(self._properties.keys()) custom_props = list(set(self._inner.keys()) - props) @@ -175,7 +170,7 @@ class _STIXBase(Mapping): if custom_props or extra_kwargs: self._allow_custom = True - if self.get_class_version() == "v21": + if self._spec_version == "2.1": all_custom_prop_names = extra_kwargs all_custom_prop_names.extend(list(custom_props.keys())) for prop_name in all_custom_prop_names: diff --git a/stix2/custom.py b/stix2/custom.py index bd2fe1d..687618d 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -3,18 +3,18 @@ import re import six -from .base import _cls_init, _Extension, _Observable, _STIXBase +from .base import _cls_init from .core import ( - STIXDomainObject, _register_marking, _register_object, - _register_observable, _register_observable_extension, + _register_marking, _register_object, _register_observable, + _register_observable_extension, ) from .utils import ( PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, get_class_hierarchy_names, ) -def _custom_object_builder(cls, type, properties, version): - class _CustomObject(cls, STIXDomainObject): +def _custom_object_builder(cls, type, properties, version, base_class): + class _CustomObject(cls, base_class): if version == "2.0": if not re.match(TYPE_REGEX, type): @@ -48,16 +48,16 @@ def _custom_object_builder(cls, type, properties, version): _properties = OrderedDict(properties) def __init__(self, **kwargs): - _STIXBase.__init__(self, **kwargs) + base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) _register_object(_CustomObject, version=version) return _CustomObject -def _custom_marking_builder(cls, type, properties, version): +def _custom_marking_builder(cls, type, properties, version, base_class): - class _CustomMarking(cls, _STIXBase): + class _CustomMarking(cls, base_class): if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") @@ -66,18 +66,18 @@ def _custom_marking_builder(cls, type, properties, version): _properties = OrderedDict(properties) def __init__(self, **kwargs): - _STIXBase.__init__(self, **kwargs) + base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) _register_marking(_CustomMarking, version=version) return _CustomMarking -def _custom_observable_builder(cls, type, properties, version, id_contrib_props=None): +def _custom_observable_builder(cls, type, properties, version, base_class, id_contrib_props=None): if id_contrib_props is None: id_contrib_props = [] - class _CustomObservable(cls, _Observable): + class _CustomObservable(cls, base_class): if version == "2.0": if not re.match(TYPE_REGEX, type): @@ -137,14 +137,14 @@ def _custom_observable_builder(cls, type, properties, version, id_contrib_props= _id_contributing_properties = id_contrib_props def __init__(self, **kwargs): - _Observable.__init__(self, **kwargs) + base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) _register_observable(_CustomObservable, version=version) return _CustomObservable -def _custom_extension_builder(cls, observable, type, properties, version): +def _custom_extension_builder(cls, observable, type, properties, version, base_class): try: prop_dict = OrderedDict(properties) @@ -158,13 +158,13 @@ def _custom_extension_builder(cls, observable, type, properties, version): e, ) - class _CustomExtension(cls, _Extension): + class _CustomExtension(cls, base_class): _type = type _properties = prop_dict def __init__(self, **kwargs): - _Extension.__init__(self, **kwargs) + base_class.__init__(self, **kwargs) _cls_init(cls, self, kwargs) _register_observable_extension(observable, _CustomExtension, version=version) diff --git a/stix2/v20/base.py b/stix2/v20/base.py new file mode 100644 index 0000000..e32069a --- /dev/null +++ b/stix2/v20/base.py @@ -0,0 +1,24 @@ +"""Base classes for STIX 2.0 type definitions.""" + +from ..base import _Extension, _Observable, _STIXBase +from ..core import STIXDomainObject, STIXRelationshipObject + + +class _STIXBase20(_STIXBase): + _spec_version = "2.0" + + +class _Observable(_Observable, _STIXBase20): + pass + + +class _Extension(_Extension, _STIXBase20): + pass + + +class STIXDomainObject(STIXDomainObject, _STIXBase20): + pass + + +class STIXRelationshipObject(STIXRelationshipObject, _STIXBase20): + pass diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index ffd70ea..b811a79 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -2,13 +2,13 @@ from collections import OrderedDict -from ..base import _STIXBase from ..properties import ( IDProperty, ListProperty, STIXObjectProperty, StringProperty, TypeProperty, ) +from .base import _STIXBase20 -class Bundle(_STIXBase): +class Bundle(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ diff --git a/stix2/v20/common.py b/stix2/v20/common.py index a1ffa60..55f2c3e 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -5,7 +5,6 @@ import copy import six -from ..base import _STIXBase from ..custom import _custom_marking_builder from ..markings import _MarkingsMixin from ..markings.utils import check_tlp_marking @@ -14,6 +13,7 @@ from ..properties import ( SelectorProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW, _get_dict +from .base import _STIXBase20 def _should_set_millisecond(cr, marking_type): @@ -31,7 +31,7 @@ def _should_set_millisecond(cr, marking_type): return False -class ExternalReference(_STIXBase): +class ExternalReference(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -49,7 +49,7 @@ class ExternalReference(_STIXBase): self._check_at_least_one_property(['description', 'external_id', 'url']) -class KillChainPhase(_STIXBase): +class KillChainPhase(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -60,7 +60,7 @@ class KillChainPhase(_STIXBase): ]) -class GranularMarking(_STIXBase): +class GranularMarking(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -71,7 +71,7 @@ class GranularMarking(_STIXBase): ]) -class TLPMarking(_STIXBase): +class TLPMarking(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -83,7 +83,7 @@ class TLPMarking(_STIXBase): ]) -class StatementMarking(_STIXBase): +class StatementMarking(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -113,7 +113,7 @@ class MarkingProperty(Property): raise ValueError("must be a Statement, TLP Marking or a registered marking.") -class MarkingDefinition(_STIXBase, _MarkingsMixin): +class MarkingDefinition(_STIXBase20, _MarkingsMixin): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -182,7 +182,7 @@ def CustomMarking(type='x-custom-marking', properties=None): """ def wrapper(cls): - return _custom_marking_builder(cls, type, properties, '2.0') + return _custom_marking_builder(cls, type, properties, '2.0', _STIXBase20) return wrapper diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 81e2c48..eb9bfde 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -1,14 +1,13 @@ """STIX 2.0 Cyber Observable Objects. Embedded observable object types, such as Email MIME Component, which is -embedded in Email Message objects, inherit from ``_STIXBase`` instead of -Observable and do not have a ``_type`` attribute. +embedded in Email Message objects, inherit from ``_STIXBase20`` instead of +_Observable and do not have a ``_type`` attribute. """ from collections import OrderedDict import itertools -from ..base import _Extension, _Observable, _STIXBase from ..custom import _custom_extension_builder, _custom_observable_builder from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError from ..properties import ( @@ -17,6 +16,7 @@ from ..properties import ( HashesProperty, HexProperty, IntegerProperty, ListProperty, ObjectReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) +from .base import _Extension, _Observable, _STIXBase20 class Artifact(_Observable): @@ -103,7 +103,7 @@ class EmailAddress(_Observable): ]) -class EmailMIMEComponent(_STIXBase): +class EmailMIMEComponent(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -166,7 +166,7 @@ class ArchiveExt(_Extension): ]) -class AlternateDataStream(_STIXBase): +class AlternateDataStream(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -220,7 +220,7 @@ class RasterImageExt(_Extension): ]) -class WindowsPEOptionalHeaderType(_STIXBase): +class WindowsPEOptionalHeaderType(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -264,7 +264,7 @@ class WindowsPEOptionalHeaderType(_STIXBase): self._check_at_least_one_property() -class WindowsPESection(_STIXBase): +class WindowsPESection(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -677,7 +677,7 @@ class UserAccount(_Observable): ]) -class WindowsRegistryValueType(_STIXBase): +class WindowsRegistryValueType(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -724,7 +724,7 @@ class WindowsRegistryKey(_Observable): ]) -class X509V3ExtenstionsType(_STIXBase): +class X509V3ExtenstionsType(_STIXBase20): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ # noqa @@ -795,7 +795,7 @@ def CustomObservable(type='x-custom-observable', properties=None): properties, [('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=type))], ])) - return _custom_observable_builder(cls, type, _properties, '2.0') + return _custom_observable_builder(cls, type, _properties, '2.0', _Observable) return wrapper @@ -803,5 +803,5 @@ def CustomExtension(observable=None, type='x-custom-observable-ext', properties= """Decorator for custom extensions to STIX Cyber Observables. """ def wrapper(cls): - return _custom_extension_builder(cls, observable, type, properties, '2.0') + return _custom_extension_builder(cls, observable, type, properties, '2.0', _Extension) return wrapper diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index 430bfc0..b083e42 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -5,7 +5,6 @@ import itertools from stix2patterns.validator import run_validator -from ..core import STIXDomainObject from ..custom import _custom_object_builder from ..exceptions import InvalidValueError from ..properties import ( @@ -14,6 +13,7 @@ from ..properties import ( TimestampProperty, TypeProperty, ) from ..utils import NOW +from .base import STIXDomainObject from .common import ExternalReference, GranularMarking, KillChainPhase @@ -374,5 +374,5 @@ def CustomObject(type='x-custom-type', properties=None): ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) - return _custom_object_builder(cls, type, _properties, '2.0') + return _custom_object_builder(cls, type, _properties, '2.0', STIXDomainObject) return wrapper diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index b85eb68..0a07969 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -2,12 +2,12 @@ from collections import OrderedDict -from ..core import STIXRelationshipObject from ..properties import ( BooleanProperty, IDProperty, IntegerProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW +from .base import STIXRelationshipObject from .common import ExternalReference, GranularMarking diff --git a/stix2/v21/base.py b/stix2/v21/base.py new file mode 100644 index 0000000..bfe1d5e --- /dev/null +++ b/stix2/v21/base.py @@ -0,0 +1,24 @@ +"""Base classes for STIX 2.1 type definitions.""" + +from ..base import _Extension, _Observable, _STIXBase +from ..core import STIXDomainObject, STIXRelationshipObject + + +class _STIXBase21(_STIXBase): + _spec_version = "2.1" + + +class _Observable(_Observable, _STIXBase21): + pass + + +class _Extension(_Extension, _STIXBase21): + pass + + +class STIXDomainObject(STIXDomainObject, _STIXBase21): + pass + + +class STIXRelationshipObject(STIXRelationshipObject, _STIXBase21): + pass diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index 168771f..ca60fe0 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -2,13 +2,13 @@ from collections import OrderedDict -from ..base import _STIXBase from ..properties import ( IDProperty, ListProperty, STIXObjectProperty, TypeProperty, ) +from .base import _STIXBase21 -class Bundle(_STIXBase): +class Bundle(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. diff --git a/stix2/v21/common.py b/stix2/v21/common.py index ac8daf1..4254767 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -2,7 +2,6 @@ from collections import OrderedDict -from ..base import _STIXBase from ..custom import _custom_marking_builder from ..exceptions import InvalidValueError from ..markings import _MarkingsMixin @@ -13,9 +12,10 @@ from ..properties import ( SelectorProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW, _get_dict +from .base import _STIXBase21 -class ExternalReference(_STIXBase): +class ExternalReference(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -50,7 +50,7 @@ class ExternalReference(_STIXBase): ) -class KillChainPhase(_STIXBase): +class KillChainPhase(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -62,7 +62,7 @@ class KillChainPhase(_STIXBase): ]) -class GranularMarking(_STIXBase): +class GranularMarking(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -79,7 +79,7 @@ class GranularMarking(_STIXBase): self._check_at_least_one_property(['lang', 'marking_ref']) -class LanguageContent(_STIXBase): +class LanguageContent(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -107,7 +107,7 @@ class LanguageContent(_STIXBase): ]) -class TLPMarking(_STIXBase): +class TLPMarking(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -119,7 +119,7 @@ class TLPMarking(_STIXBase): ]) -class StatementMarking(_STIXBase): +class StatementMarking(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -150,7 +150,7 @@ class MarkingProperty(Property): raise ValueError("must be a Statement, TLP Marking or a registered marking.") -class MarkingDefinition(_STIXBase, _MarkingsMixin): +class MarkingDefinition(_STIXBase21, _MarkingsMixin): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -215,7 +215,7 @@ def CustomMarking(type='x-custom-marking', properties=None): """ def wrapper(cls): - return _custom_marking_builder(cls, type, properties, '2.1') + return _custom_marking_builder(cls, type, properties, '2.1', _STIXBase21) return wrapper diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index 8c9a2a1..e71e445 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -1,14 +1,13 @@ """STIX 2.1 Cyber Observable Objects. Embedded observable object types, such as Email MIME Component, which is -embedded in Email Message objects, inherit from ``_STIXBase`` instead of -Observable and do not have a ``_type`` attribute. +embedded in Email Message objects, inherit from ``_STIXBase21`` instead of +_Observable and do not have a ``_type`` attribute. """ from collections import OrderedDict import itertools -from ..base import _Extension, _Observable, _STIXBase from ..custom import _custom_extension_builder, _custom_observable_builder from ..exceptions import AtLeastOnePropertyError, DependentPropertiesError from ..properties import ( @@ -18,6 +17,7 @@ from ..properties import ( ObjectReferenceProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) +from .base import _Extension, _Observable, _STIXBase21 from .common import GranularMarking @@ -142,7 +142,7 @@ class EmailAddress(_Observable): _id_contributing_properties = ["value"] -class EmailMIMEComponent(_STIXBase): +class EmailMIMEComponent(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -214,7 +214,7 @@ class ArchiveExt(_Extension): ]) -class AlternateDataStream(_STIXBase): +class AlternateDataStream(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -271,7 +271,7 @@ class RasterImageExt(_Extension): ]) -class WindowsPEOptionalHeaderType(_STIXBase): +class WindowsPEOptionalHeaderType(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -316,7 +316,7 @@ class WindowsPEOptionalHeaderType(_STIXBase): self._check_at_least_one_property() -class WindowsPESection(_STIXBase): +class WindowsPESection(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -841,7 +841,7 @@ class UserAccount(_Observable): _id_contributing_properties = ["account_type", "user_id", "account_login"] -class WindowsRegistryValueType(_STIXBase): +class WindowsRegistryValueType(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -896,7 +896,7 @@ class WindowsRegistryKey(_Observable): _id_contributing_properties = ["key", "values"] -class X509V3ExtenstionsType(_STIXBase): +class X509V3ExtenstionsType(_STIXBase21): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -988,7 +988,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro properties, [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))], ])) - return _custom_observable_builder(cls, type, _properties, '2.1', id_contrib_props) + return _custom_observable_builder(cls, type, _properties, '2.1', _Observable, id_contrib_props) return wrapper @@ -996,5 +996,5 @@ def CustomExtension(observable=None, type='x-custom-observable-ext', properties= """Decorator for custom extensions to STIX Cyber Observables. """ def wrapper(cls): - return _custom_extension_builder(cls, observable, type, properties, '2.1') + return _custom_extension_builder(cls, observable, type, properties, '2.1', _Extension) return wrapper diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index f2dc0ea..42c58e4 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -7,7 +7,6 @@ import warnings from six.moves.urllib.parse import quote_plus from stix2patterns.validator import run_validator -from ..core import STIXDomainObject from ..custom import _custom_object_builder from ..exceptions import ( InvalidValueError, PropertyPresenceError, STIXDeprecationWarning, @@ -18,6 +17,7 @@ from ..properties import ( StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW +from .base import STIXDomainObject from .common import ExternalReference, GranularMarking, KillChainPhase @@ -828,6 +828,6 @@ def CustomObject(type='x-custom-type', properties=None): ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) - return _custom_object_builder(cls, type, _properties, '2.1') + return _custom_object_builder(cls, type, _properties, '2.1', STIXDomainObject) return wrapper diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 059bb66..91b8b80 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -2,12 +2,12 @@ from collections import OrderedDict -from ..core import STIXRelationshipObject from ..properties import ( BooleanProperty, IDProperty, IntegerProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW +from .base import STIXRelationshipObject from .common import ExternalReference, GranularMarking From 01ba190525639791389a1698fab3e478f233d84b Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Mar 2020 02:40:42 -0400 Subject: [PATCH 20/37] Reorganize bases, use isinstance to check version Renamed STIXDomainObject -> _DomainObject. Renamed STIXRelationshipObject -> _RelationshipObject. --- stix2/base.py | 24 ++++++++++++++++-------- stix2/core.py | 11 +---------- stix2/properties.py | 2 +- stix2/v20/__init__.py | 3 +++ stix2/v20/base.py | 11 ++++++----- stix2/v20/sdo.py | 28 ++++++++++++++-------------- stix2/v20/sro.py | 6 +++--- stix2/v21/__init__.py | 3 +++ stix2/v21/base.py | 11 ++++++----- stix2/v21/sdo.py | 42 +++++++++++++++++++++--------------------- stix2/v21/sro.py | 6 +++--- 11 files changed, 77 insertions(+), 70 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 5af3607..a1229ca 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -8,6 +8,7 @@ import uuid import simplejson as json import six +import stix2 from stix2.canonicalization.Canonicalize import canonicalize from .exceptions import ( @@ -15,6 +16,7 @@ from .exceptions import ( ImmutableError, InvalidObjRefError, InvalidValueError, MissingPropertiesError, MutuallyExclusivePropertiesError, ) +from .markings import _MarkingsMixin from .markings.utils import validate from .utils import ( NOW, PREFIX_21_REGEX, find_property_index, format_datetime, get_timestamp, @@ -160,17 +162,15 @@ class _STIXBase(Mapping): custom_props = kwargs.pop('custom_properties', {}) if custom_props and not isinstance(custom_props, dict): raise ValueError("'custom_properties' must be a dictionary") - if not self._allow_custom: - extra_kwargs = list(set(kwargs) - set(self._properties)) - if extra_kwargs: - raise ExtraPropertiesError(cls, extra_kwargs) - else: - # because allow_custom is true, any extra kwargs are custom - extra_kwargs = list(set(kwargs) - set(self._properties)) + extra_kwargs = list(set(kwargs) - set(self._properties)) + if extra_kwargs and not self._allow_custom: + raise ExtraPropertiesError(cls, extra_kwargs) + + # because allow_custom is true, any extra kwargs are custom if custom_props or extra_kwargs: self._allow_custom = True - if self._spec_version == "2.1": + if isinstance(self, stix2.v21._STIXBase21): all_custom_prop_names = extra_kwargs all_custom_prop_names.extend(list(custom_props.keys())) for prop_name in all_custom_prop_names: @@ -321,6 +321,14 @@ class _STIXBase(Mapping): return json.dumps(self, cls=STIXJSONEncoder, **kwargs) +class _DomainObject(_STIXBase, _MarkingsMixin): + pass + + +class _RelationshipObject(_STIXBase, _MarkingsMixin): + pass + + class _Observable(_STIXBase): def __init__(self, **kwargs): diff --git a/stix2/core.py b/stix2/core.py index 055a4a6..7dedc15 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -7,9 +7,8 @@ import re import stix2 -from .base import _Observable, _STIXBase +from .base import _Observable from .exceptions import ParseError -from .markings import _MarkingsMixin from .utils import ( EXT_21_REGEX, PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, _get_dict, ) @@ -17,14 +16,6 @@ from .utils import ( STIX2_OBJ_MAPS = {} -class STIXDomainObject(_STIXBase, _MarkingsMixin): - pass - - -class STIXRelationshipObject(_STIXBase, _MarkingsMixin): - pass - - def parse(data, allow_custom=False, version=None): """Convert a string, dict or file-like object into a STIX object. diff --git a/stix2/properties.py b/stix2/properties.py index b013b7e..e02f96b 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -639,7 +639,7 @@ class STIXObjectProperty(Property): def clean(self, value): # Any STIX Object (SDO, SRO, or Marking Definition) can be added to # a bundle with no further checks. - if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition') + if any(x in ('_DomainObject', '_RelationshipObject', 'MarkingDefinition') for x in get_class_hierarchy_names(value)): # A simple "is this a spec version 2.1+ object" test. For now, # limit 2.0 bundles to 2.0 objects. It's not possible yet to diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py index 4d0a98f..b77d46c 100644 --- a/stix2/v20/__init__.py +++ b/stix2/v20/__init__.py @@ -14,6 +14,9 @@ # flake8: noqa +from .base import ( + _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase20, +) from .bundle import Bundle from .common import ( TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, diff --git a/stix2/v20/base.py b/stix2/v20/base.py index e32069a..b5437ca 100644 --- a/stix2/v20/base.py +++ b/stix2/v20/base.py @@ -1,11 +1,12 @@ """Base classes for STIX 2.0 type definitions.""" -from ..base import _Extension, _Observable, _STIXBase -from ..core import STIXDomainObject, STIXRelationshipObject +from ..base import ( + _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase, +) class _STIXBase20(_STIXBase): - _spec_version = "2.0" + pass class _Observable(_Observable, _STIXBase20): @@ -16,9 +17,9 @@ class _Extension(_Extension, _STIXBase20): pass -class STIXDomainObject(STIXDomainObject, _STIXBase20): +class _DomainObject(_DomainObject, _STIXBase20): pass -class STIXRelationshipObject(STIXRelationshipObject, _STIXBase20): +class _RelationshipObject(_RelationshipObject, _STIXBase20): pass diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index b083e42..cb5ba04 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -13,11 +13,11 @@ from ..properties import ( TimestampProperty, TypeProperty, ) from ..utils import NOW -from .base import STIXDomainObject +from .base import _DomainObject from .common import ExternalReference, GranularMarking, KillChainPhase -class AttackPattern(STIXDomainObject): +class AttackPattern(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -40,7 +40,7 @@ class AttackPattern(STIXDomainObject): ]) -class Campaign(STIXDomainObject): +class Campaign(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -66,7 +66,7 @@ class Campaign(STIXDomainObject): ]) -class CourseOfAction(STIXDomainObject): +class CourseOfAction(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -88,7 +88,7 @@ class CourseOfAction(STIXDomainObject): ]) -class Identity(STIXDomainObject): +class Identity(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -113,7 +113,7 @@ class Identity(STIXDomainObject): ]) -class Indicator(STIXDomainObject): +class Indicator(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -144,7 +144,7 @@ class Indicator(STIXDomainObject): raise InvalidValueError(self.__class__, 'pattern', str(errors[0])) -class IntrusionSet(STIXDomainObject): +class IntrusionSet(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -173,7 +173,7 @@ class IntrusionSet(STIXDomainObject): ]) -class Malware(STIXDomainObject): +class Malware(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -196,7 +196,7 @@ class Malware(STIXDomainObject): ]) -class ObservedData(STIXDomainObject): +class ObservedData(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -226,7 +226,7 @@ class ObservedData(STIXDomainObject): super(ObservedData, self).__init__(*args, **kwargs) -class Report(STIXDomainObject): +class Report(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -250,7 +250,7 @@ class Report(STIXDomainObject): ]) -class ThreatActor(STIXDomainObject): +class ThreatActor(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -280,7 +280,7 @@ class ThreatActor(STIXDomainObject): ]) -class Tool(STIXDomainObject): +class Tool(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -304,7 +304,7 @@ class Tool(STIXDomainObject): ]) -class Vulnerability(STIXDomainObject): +class Vulnerability(_DomainObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -374,5 +374,5 @@ def CustomObject(type='x-custom-type', properties=None): ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) - return _custom_object_builder(cls, type, _properties, '2.0', STIXDomainObject) + return _custom_object_builder(cls, type, _properties, '2.0', _DomainObject) return wrapper diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index 0a07969..b7d1ad3 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -7,11 +7,11 @@ from ..properties import ( ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW -from .base import STIXRelationshipObject +from .base import _RelationshipObject from .common import ExternalReference, GranularMarking -class Relationship(STIXRelationshipObject): +class Relationship(_RelationshipObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ @@ -52,7 +52,7 @@ class Relationship(STIXRelationshipObject): super(Relationship, self).__init__(**kwargs) -class Sighting(STIXRelationshipObject): +class Sighting(_RelationshipObject): """For more detailed information on this object's properties, see `the STIX 2.0 specification `__. """ diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index b2451d2..eea61dd 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -14,6 +14,9 @@ # flake8: noqa +from .base import ( + _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase21, +) from .bundle import Bundle from .common import ( TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, diff --git a/stix2/v21/base.py b/stix2/v21/base.py index bfe1d5e..8b5a5f1 100644 --- a/stix2/v21/base.py +++ b/stix2/v21/base.py @@ -1,11 +1,12 @@ """Base classes for STIX 2.1 type definitions.""" -from ..base import _Extension, _Observable, _STIXBase -from ..core import STIXDomainObject, STIXRelationshipObject +from ..base import ( + _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase, +) class _STIXBase21(_STIXBase): - _spec_version = "2.1" + pass class _Observable(_Observable, _STIXBase21): @@ -16,9 +17,9 @@ class _Extension(_Extension, _STIXBase21): pass -class STIXDomainObject(STIXDomainObject, _STIXBase21): +class _DomainObject(_DomainObject, _STIXBase21): pass -class STIXRelationshipObject(STIXRelationshipObject, _STIXBase21): +class _RelationshipObject(_RelationshipObject, _STIXBase21): pass diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 42c58e4..1e9633e 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -17,11 +17,11 @@ from ..properties import ( StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW -from .base import STIXDomainObject +from .base import _DomainObject from .common import ExternalReference, GranularMarking, KillChainPhase -class AttackPattern(STIXDomainObject): +class AttackPattern(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -49,7 +49,7 @@ class AttackPattern(STIXDomainObject): ]) -class Campaign(STIXDomainObject): +class Campaign(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -89,7 +89,7 @@ class Campaign(STIXDomainObject): raise ValueError(msg.format(self)) -class CourseOfAction(STIXDomainObject): +class CourseOfAction(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -115,7 +115,7 @@ class CourseOfAction(STIXDomainObject): ]) -class Grouping(STIXDomainObject): +class Grouping(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -143,7 +143,7 @@ class Grouping(STIXDomainObject): ]) -class Identity(STIXDomainObject): +class Identity(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -173,7 +173,7 @@ class Identity(STIXDomainObject): ]) -class Indicator(STIXDomainObject): +class Indicator(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -210,7 +210,7 @@ class Indicator(STIXDomainObject): if kwargs.get('pattern') and kwargs.get('pattern_type') == 'stix' and not kwargs.get('pattern_version'): kwargs['pattern_version'] = '2.1' - super(STIXDomainObject, self).__init__(*args, **kwargs) + super(_DomainObject, self).__init__(*args, **kwargs) def _check_object_constraints(self): super(Indicator, self)._check_object_constraints() @@ -233,7 +233,7 @@ class Indicator(STIXDomainObject): raise InvalidValueError(self.__class__, 'pattern', str(errors[0])) -class Infrastructure(STIXDomainObject): +class Infrastructure(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -274,7 +274,7 @@ class Infrastructure(STIXDomainObject): raise ValueError(msg.format(self)) -class IntrusionSet(STIXDomainObject): +class IntrusionSet(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -317,7 +317,7 @@ class IntrusionSet(STIXDomainObject): raise ValueError(msg.format(self)) -class Location(STIXDomainObject): +class Location(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -425,7 +425,7 @@ class Location(STIXDomainObject): return final_url -class Malware(STIXDomainObject): +class Malware(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -478,7 +478,7 @@ class Malware(STIXDomainObject): ) -class MalwareAnalysis(STIXDomainObject): +class MalwareAnalysis(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -523,7 +523,7 @@ class MalwareAnalysis(STIXDomainObject): self._check_at_least_one_property(["result", "analysis_sco_refs"]) -class Note(STIXDomainObject): +class Note(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -551,7 +551,7 @@ class Note(STIXDomainObject): ]) -class ObservedData(STIXDomainObject): +class ObservedData(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -607,7 +607,7 @@ class ObservedData(STIXDomainObject): ) -class Opinion(STIXDomainObject): +class Opinion(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -645,7 +645,7 @@ class Opinion(STIXDomainObject): ]) -class Report(STIXDomainObject): +class Report(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -674,7 +674,7 @@ class Report(STIXDomainObject): ]) -class ThreatActor(STIXDomainObject): +class ThreatActor(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -721,7 +721,7 @@ class ThreatActor(STIXDomainObject): raise ValueError(msg.format(self)) -class Tool(STIXDomainObject): +class Tool(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -751,7 +751,7 @@ class Tool(STIXDomainObject): ]) -class Vulnerability(STIXDomainObject): +class Vulnerability(_DomainObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -828,6 +828,6 @@ def CustomObject(type='x-custom-type', properties=None): ], sorted([x for x in properties if x[0].startswith('x_')], key=lambda x: x[0]), ])) - return _custom_object_builder(cls, type, _properties, '2.1', STIXDomainObject) + return _custom_object_builder(cls, type, _properties, '2.1', _DomainObject) return wrapper diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 91b8b80..3ffdd77 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -7,11 +7,11 @@ from ..properties import ( ReferenceProperty, StringProperty, TimestampProperty, TypeProperty, ) from ..utils import NOW -from .base import STIXRelationshipObject +from .base import _RelationshipObject from .common import ExternalReference, GranularMarking -class Relationship(STIXRelationshipObject): +class Relationship(_RelationshipObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. @@ -68,7 +68,7 @@ class Relationship(STIXRelationshipObject): raise ValueError(msg.format(self)) -class Sighting(STIXRelationshipObject): +class Sighting(_RelationshipObject): # TODO: Add link """For more detailed information on this object's properties, see `the STIX 2.1 specification `__. From 50df6f14744697fdbdbad04bebf8516e4c09aea3 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Mar 2020 05:53:39 -0400 Subject: [PATCH 21/37] Rename core.py -> parsing.py --- stix2/__init__.py | 2 +- stix2/custom.py | 2 +- stix2/datastore/filesystem.py | 2 +- stix2/datastore/memory.py | 2 +- stix2/datastore/taxii.py | 2 +- stix2/environment.py | 2 +- stix2/{core.py => parsing.py} | 0 stix2/properties.py | 2 +- stix2/test/test_spec_version_detect.py | 2 +- stix2/test/v20/test_core.py | 32 +++++++++++++------------- stix2/test/v20/test_custom.py | 2 +- stix2/test/v21/test_core.py | 32 +++++++++++++------------- stix2/test/v21/test_custom.py | 2 +- 13 files changed, 42 insertions(+), 42 deletions(-) rename stix2/{core.py => parsing.py} (100%) diff --git a/stix2/__init__.py b/stix2/__init__.py index c9384a0..f3e02b4 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -23,7 +23,6 @@ DEFAULT_VERSION = '2.0' # Default version will always be the latest STIX 2.X version from .confidence import scales -from .core import _collect_stix2_mappings, parse, parse_observable from .datastore import CompositeDataSource from .datastore.filesystem import ( FileSystemSink, FileSystemSource, FileSystemStore, @@ -38,6 +37,7 @@ from .markings import ( add_markings, clear_markings, get_markings, is_marked, remove_markings, set_markings, ) +from .parsing import _collect_stix2_mappings, parse, parse_observable from .patterns import ( AndBooleanExpression, AndObservationExpression, BasicObjectPathComponent, BinaryConstant, BooleanConstant, EqualityComparisonExpression, diff --git a/stix2/custom.py b/stix2/custom.py index 687618d..6d1abc5 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -4,7 +4,7 @@ import re import six from .base import _cls_init -from .core import ( +from .parsing import ( _register_marking, _register_object, _register_observable, _register_observable_extension, ) diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index d5acc24..5a5844a 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -10,11 +10,11 @@ import six from stix2 import v20, v21 from stix2.base import _STIXBase -from stix2.core import parse from stix2.datastore import ( DataSink, DataSource, DataSourceError, DataStoreMixin, ) from stix2.datastore.filters import Filter, FilterSet, apply_common_filters +from stix2.parsing import parse from stix2.utils import format_datetime, get_type_from_id diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index 52da168..f71b763 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -7,9 +7,9 @@ import os from stix2 import v20, v21 from stix2.base import _STIXBase -from stix2.core import parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import FilterSet, apply_common_filters +from stix2.parsing import parse def _add(store, stix_data, allow_custom=True, version=None): diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 41c968f..58dcdbc 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -4,11 +4,11 @@ from requests.exceptions import HTTPError from stix2 import v20, v21 from stix2.base import _STIXBase -from stix2.core import parse from stix2.datastore import ( DataSink, DataSource, DataSourceError, DataStoreMixin, ) from stix2.datastore.filters import Filter, FilterSet, apply_common_filters +from stix2.parsing import parse from stix2.utils import deduplicate try: diff --git a/stix2/environment.py b/stix2/environment.py index ada5f33..6e3c76a 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -4,8 +4,8 @@ import copy import logging import time -from .core import parse as _parse from .datastore import CompositeDataSource, DataStoreMixin +from .parsing import parse as _parse from .utils import STIXdatetime, parse_into_datetime logger = logging.getLogger(__name__) diff --git a/stix2/core.py b/stix2/parsing.py similarity index 100% rename from stix2/core.py rename to stix2/parsing.py diff --git a/stix2/properties.py b/stix2/properties.py index e02f96b..be66533 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -12,11 +12,11 @@ from six import string_types, text_type import stix2 from .base import _STIXBase -from .core import STIX2_OBJ_MAPS, parse, parse_observable from .exceptions import ( CustomContentError, DictionaryKeyError, MissingPropertiesError, MutuallyExclusivePropertiesError, ) +from .parsing import STIX2_OBJ_MAPS, parse, parse_observable from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime try: diff --git a/stix2/test/test_spec_version_detect.py b/stix2/test/test_spec_version_detect.py index d2a9da8..7039024 100644 --- a/stix2/test/test_spec_version_detect.py +++ b/stix2/test/test_spec_version_detect.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import pytest -from stix2.core import _detect_spec_version +from stix2.parsing import _detect_spec_version @pytest.mark.parametrize( diff --git a/stix2/test/v20/test_core.py b/stix2/test/v20/test_core.py index d2efa22..ca701fb 100644 --- a/stix2/test/v20/test_core.py +++ b/stix2/test/v20/test_core.py @@ -1,7 +1,7 @@ import pytest import stix2 -from stix2 import core, exceptions +from stix2 import exceptions, parsing from .constants import IDENTITY_ID @@ -46,14 +46,14 @@ BUNDLE = { def test_dict_to_stix2_bundle_with_version(): with pytest.raises(exceptions.ExtraPropertiesError) as excinfo: - core.dict_to_stix2(BUNDLE, version='2.1') + parsing.dict_to_stix2(BUNDLE, version='2.1') assert str(excinfo.value) == "Unexpected properties for Bundle: (spec_version)." def test_parse_observable_with_version(): observable = {"type": "file", "name": "foo.exe"} - obs_obj = core.parse_observable(observable, version='2.0') + obs_obj = parsing.parse_observable(observable, version='2.0') v = 'v20' assert v in str(obs_obj.__class__) @@ -62,38 +62,38 @@ def test_parse_observable_with_version(): @pytest.mark.xfail(reason="The default version is no longer 2.0", condition=stix2.DEFAULT_VERSION != "2.0") def test_parse_observable_with_no_version(): observable = {"type": "file", "name": "foo.exe"} - obs_obj = core.parse_observable(observable) + obs_obj = parsing.parse_observable(observable) v = 'v20' assert v in str(obs_obj.__class__) def test_register_object_with_version(): - bundle = core.dict_to_stix2(BUNDLE, version='2.0') - core._register_object(bundle.objects[0].__class__, version='2.0') + bundle = parsing.dict_to_stix2(BUNDLE, version='2.0') + parsing._register_object(bundle.objects[0].__class__, version='2.0') v = 'v20' - assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] + assert bundle.objects[0].type in parsing.STIX2_OBJ_MAPS[v]['objects'] # spec_version is not in STIX 2.0, and is required in 2.1, so this # suffices as a test for a STIX 2.0 object. assert "spec_version" not in bundle.objects[0] def test_register_marking_with_version(): - core._register_marking(stix2.v20.TLP_WHITE.__class__, version='2.0') + parsing._register_marking(stix2.v20.TLP_WHITE.__class__, version='2.0') v = 'v20' - assert stix2.v20.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] + assert stix2.v20.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v20.TLP_WHITE.__class__) @pytest.mark.xfail(reason="The default version is no longer 2.0", condition=stix2.DEFAULT_VERSION != "2.0") def test_register_marking_with_no_version(): # Uses default version (2.0 in this case) - core._register_marking(stix2.v20.TLP_WHITE.__class__) + parsing._register_marking(stix2.v20.TLP_WHITE.__class__) v = 'v20' - assert stix2.v20.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] + assert stix2.v20.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v20.TLP_WHITE.__class__) @@ -128,10 +128,10 @@ def test_register_observable_with_version(): }, }, ) - core._register_observable(observed_data.objects['0'].__class__, version='2.0') + parsing._register_observable(observed_data.objects['0'].__class__, version='2.0') v = 'v20' - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] + assert observed_data.objects['0'].type in parsing.STIX2_OBJ_MAPS[v]['observables'] assert v in str(observed_data.objects['0'].__class__) @@ -166,11 +166,11 @@ def test_register_observable_extension_with_version(): }, }, ) - core._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__, version='2.0') + parsing._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__, version='2.0') v = 'v20' - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] + assert observed_data.objects['0'].type in parsing.STIX2_OBJ_MAPS[v]['observables'] assert v in str(observed_data.objects['0'].__class__) - assert observed_data.objects['0'].extensions['ntfs-ext']._type in core.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] + assert observed_data.objects['0'].extensions['ntfs-ext']._type in parsing.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] assert v in str(observed_data.objects['0'].extensions['ntfs-ext'].__class__) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index b986777..e24393c 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -967,7 +967,7 @@ def test_register_custom_object(): class CustomObject2(object): _type = 'awesome-object' - stix2.core._register_object(CustomObject2, version="2.0") + stix2.parsing._register_object(CustomObject2, version="2.0") # Note that we will always check against newest OBJ_MAP. assert (CustomObject2._type, CustomObject2) in stix2.v20.OBJ_MAP.items() diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index 2018395..48f0055 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -1,7 +1,7 @@ import pytest import stix2 -from stix2 import core, exceptions +from stix2 import exceptions, parsing from .constants import IDENTITY_ID, OBSERVED_DATA_ID @@ -50,7 +50,7 @@ BUNDLE = { def test_dict_to_stix2_bundle_with_version(): with pytest.raises(exceptions.InvalidValueError) as excinfo: - core.dict_to_stix2(BUNDLE, version='2.0') + parsing.dict_to_stix2(BUNDLE, version='2.0') msg = "Invalid value for Bundle 'objects': Spec version 2.0 bundles don't yet support containing objects of a different spec version." assert str(excinfo.value) == msg @@ -58,7 +58,7 @@ def test_dict_to_stix2_bundle_with_version(): def test_parse_observable_with_version(): observable = {"type": "file", "name": "foo.exe"} - obs_obj = core.parse_observable(observable, version='2.1') + obs_obj = parsing.parse_observable(observable, version='2.1') v = 'v21' assert v in str(obs_obj.__class__) @@ -67,36 +67,36 @@ def test_parse_observable_with_version(): @pytest.mark.xfail(reason="The default version is not 2.1", condition=stix2.DEFAULT_VERSION != "2.1") def test_parse_observable_with_no_version(): observable = {"type": "file", "name": "foo.exe"} - obs_obj = core.parse_observable(observable) + obs_obj = parsing.parse_observable(observable) v = 'v21' assert v in str(obs_obj.__class__) def test_register_object_with_version(): - bundle = core.dict_to_stix2(BUNDLE, version='2.1') - core._register_object(bundle.objects[0].__class__) + bundle = parsing.dict_to_stix2(BUNDLE, version='2.1') + parsing._register_object(bundle.objects[0].__class__) v = 'v21' - assert bundle.objects[0].type in core.STIX2_OBJ_MAPS[v]['objects'] + assert bundle.objects[0].type in parsing.STIX2_OBJ_MAPS[v]['objects'] assert bundle.objects[0].spec_version == "2.1" def test_register_marking_with_version(): - core._register_marking(stix2.v21.TLP_WHITE.__class__, version='2.1') + parsing._register_marking(stix2.v21.TLP_WHITE.__class__, version='2.1') v = 'v21' - assert stix2.v21.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] + assert stix2.v21.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v21.TLP_WHITE.__class__) @pytest.mark.xfail(reason="The default version is not 2.1", condition=stix2.DEFAULT_VERSION != "2.1") def test_register_marking_with_no_version(): # Uses default version (2.0 in this case) - core._register_marking(stix2.v21.TLP_WHITE.__class__) + parsing._register_marking(stix2.v21.TLP_WHITE.__class__) v = 'v21' - assert stix2.v21.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] + assert stix2.v21.TLP_WHITE.definition._type in parsing.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v21.TLP_WHITE.__class__) @@ -131,10 +131,10 @@ def test_register_observable_with_default_version(): }, }, ) - core._register_observable(observed_data.objects['0'].__class__) + parsing._register_observable(observed_data.objects['0'].__class__) v = 'v21' - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] + assert observed_data.objects['0'].type in parsing.STIX2_OBJ_MAPS[v]['observables'] assert v in str(observed_data.objects['0'].__class__) @@ -169,11 +169,11 @@ def test_register_observable_extension_with_default_version(): }, }, ) - core._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__) + parsing._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__) v = 'v21' - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] + assert observed_data.objects['0'].type in parsing.STIX2_OBJ_MAPS[v]['observables'] assert v in str(observed_data.objects['0'].__class__) - assert observed_data.objects['0'].extensions['ntfs-ext']._type in core.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] + assert observed_data.objects['0'].extensions['ntfs-ext']._type in parsing.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] assert v in str(observed_data.objects['0'].extensions['ntfs-ext'].__class__) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index ef57a9b..a9969b8 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1186,7 +1186,7 @@ def test_register_custom_object(): class CustomObject2(object): _type = 'awesome-object' - stix2.core._register_object(CustomObject2, version="2.1") + stix2.parsing._register_object(CustomObject2, version="2.1") # Note that we will always check against newest OBJ_MAP. assert (CustomObject2._type, CustomObject2) in stix2.v21.OBJ_MAP.items() From b4700e6d00d1c46e91bede4b6e1cf128334d8369 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Mar 2020 06:33:29 -0400 Subject: [PATCH 22/37] Fix import errors And pin medallion version for testing. --- stix2/utils.py | 6 ++---- tox.ini | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index da04a3e..69489d9 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -12,7 +12,7 @@ import re from dateutil import parser import pytz -import stix2.base +import stix2 from .exceptions import ( InvalidValueError, RevokeError, UnmodifiablePropertyError, @@ -233,14 +233,12 @@ def find_property_index(obj, search_key, search_value): Returns: int: An index; -1 if the key and value aren't found """ - from .base import _STIXBase - # Special-case keys which are numbers-as-strings, e.g. for cyber-observable # mappings. Use the int value of the key as the index. if search_key.isdigit(): return int(search_key) - if isinstance(obj, _STIXBase): + if isinstance(obj, stix2.base._STIXBase): if search_key in obj and obj[search_key] == search_value: idx = _find(obj.object_properties(), search_key) else: diff --git a/tox.ini b/tox.ini index d34aac1..38ecb61 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ deps = fuzzywuzzy haversine python-Levenshtein - medallion + medallion<2.0.0 commands = python -m pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning From 202111acdf51da5177c9981ea8f0d169a33a9da5 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 27 Mar 2020 11:22:00 -0400 Subject: [PATCH 23/37] more pattern tests --- stix2/pattern_visitor.py | 90 +++++++++++------ stix2/test/v21/test_pattern_expressions.py | 108 ++++++++++++++++++++- 2 files changed, 164 insertions(+), 34 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 6ac3e98..51e309f 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -2,11 +2,16 @@ import importlib import inspect from stix2patterns.exceptions import ParseException -from stix2patterns.grammars.STIXPatternParser import ( - STIXPatternParser, TerminalNode, -) +from stix2patterns.grammars.STIXPatternParser import TerminalNode + +from stix2patterns.v20.grammars.STIXPatternParser import STIXPatternParser as STIXPatternParser20 +from stix2patterns.v21.grammars.STIXPatternParser import STIXPatternParser as STIXPatternParser21 + +from stix2patterns.v20.grammars.STIXPatternVisitor import STIXPatternVisitor as STIXPatternVisitor20 +from stix2patterns.v21.grammars.STIXPatternVisitor import STIXPatternVisitor as STIXPatternVisitor21 from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor -from stix2patterns.v20.pattern import Pattern +from stix2patterns.v20.pattern import Pattern as Pattern20 +from stix2patterns.v21.pattern import Pattern as Pattern21 from .patterns import * from .patterns import _BooleanExpression @@ -32,23 +37,12 @@ def remove_terminal_nodes(parse_tree_nodes): return values -# This class defines a complete generic visitor for a parse tree produced by STIXPatternParser. -class STIXPatternVisitorForSTIX2(STIXPatternVisitor): + +class STIXPatternVisitorForSTIX2(): classes = {} - def __init__(self, module_suffix, module_name): - if module_suffix and module_name: - self.module_suffix = module_suffix - if not STIXPatternVisitorForSTIX2.classes: - module = importlib.import_module(module_name) - for k, c in inspect.getmembers(module, inspect.isclass): - STIXPatternVisitorForSTIX2.classes[k] = c - else: - self.module_suffix = None - super(STIXPatternVisitor, self).__init__() - def get_class(self, class_name): if class_name in STIXPatternVisitorForSTIX2.classes: return STIXPatternVisitorForSTIX2.classes[class_name] @@ -147,7 +141,7 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def visitPropTestEqual(self, ctx): children = self.visitChildren(ctx) operator = children[1].symbol.type - negated = operator != STIXPatternParser.EQ + negated = operator != self.parser_class.EQ return self.instantiate( "EqualityComparisonExpression", children[0], children[3 if len(children) > 3 else 2], negated, @@ -157,22 +151,22 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): def visitPropTestOrder(self, ctx): children = self.visitChildren(ctx) operator = children[1].symbol.type - if operator == STIXPatternParser.GT: + if operator == self.parser_class.GT: return self.instantiate( "GreaterThanComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False, ) - elif operator == STIXPatternParser.LT: + elif operator == self.parser_class.LT: return self.instantiate( "LessThanComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False, ) - elif operator == STIXPatternParser.GE: + elif operator == self.parser_class.GE: return self.instantiate( "GreaterThanEqualComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False, ) - elif operator == STIXPatternParser.LE: + elif operator == self.parser_class.LE: return self.instantiate( "LessThanEqualComparisonExpression", children[0], children[3 if len(children) > 3 else 2], False, @@ -294,22 +288,22 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): return children[0] def visitTerminal(self, node): - if node.symbol.type == STIXPatternParser.IntPosLiteral or node.symbol.type == STIXPatternParser.IntNegLiteral: + if node.symbol.type == self.parser_class.IntPosLiteral or node.symbol.type == self.parser_class.IntNegLiteral: return IntegerConstant(node.getText()) - elif node.symbol.type == STIXPatternParser.FloatPosLiteral or node.symbol.type == STIXPatternParser.FloatNegLiteral: + elif node.symbol.type == self.parser_class.FloatPosLiteral or node.symbol.type == self.parser_class.FloatNegLiteral: return FloatConstant(node.getText()) - elif node.symbol.type == STIXPatternParser.HexLiteral: + elif node.symbol.type == self.parser_class.HexLiteral: return HexConstant(node.getText(), from_parse_tree=True) - elif node.symbol.type == STIXPatternParser.BinaryLiteral: + elif node.symbol.type == self.parser_class.BinaryLiteral: return BinaryConstant(node.getText(), from_parse_tree=True) - elif node.symbol.type == STIXPatternParser.StringLiteral: + elif node.symbol.type == self.parser_class.StringLiteral: if node.getText()[0] == "'" and node.getText()[-1] == "'": return StringConstant(node.getText()[1:-1], from_parse_tree=True) else: raise ParseException("The pattern does not start and end with a single quote") - elif node.symbol.type == STIXPatternParser.BoolLiteral: + elif node.symbol.type == self.parser_class.BoolLiteral: return BooleanConstant(node.getText()) - elif node.symbol.type == STIXPatternParser.TimestampLiteral: + elif node.symbol.type == self.parser_class.TimestampLiteral: return TimestampConstant(node.getText()) else: return node @@ -321,12 +315,44 @@ class STIXPatternVisitorForSTIX2(STIXPatternVisitor): aggregate = [nextResult] return aggregate +# This class defines a complete generic visitor for a parse tree produced by STIXPatternParser. +class STIXPatternVisitorForSTIX21(STIXPatternVisitorForSTIX2, STIXPatternVisitor21): + classes = {} -def create_pattern_object(pattern, module_suffix="", module_name=""): + def __init__(self, module_suffix, module_name): + if module_suffix and module_name: + self.module_suffix = module_suffix + if not STIXPatternVisitorForSTIX2.classes: + module = importlib.import_module(module_name) + for k, c in inspect.getmembers(module, inspect.isclass): + STIXPatternVisitorForSTIX2.classes[k] = c + else: + self.module_suffix = None + self.parser_class = STIXPatternParser21 + super(STIXPatternVisitor21, self).__init__() + + +class STIXPatternVisitorForSTIX20(STIXPatternVisitor20, STIXPatternVisitorForSTIX2): + classes = {} + + def __init__(self, module_suffix, module_name): + if module_suffix and module_name: + self.module_suffix = module_suffix + if not STIXPatternVisitorForSTIX2.classes: + module = importlib.import_module(module_name) + for k, c in inspect.getmembers(module, inspect.isclass): + STIXPatternVisitorForSTIX2.classes[k] = c + else: + self.module_suffix = None + self.parser_class = STIXPatternParser20 + super(STIXPatternVisitor20, self).__init__() + + +def create_pattern_object(pattern, module_suffix="", module_name="", version="2.1"): """ Create a STIX pattern AST from a pattern string. """ - pattern_obj = Pattern(pattern) - builder = STIXPatternVisitorForSTIX2(module_suffix, module_name) + pattern_obj = Pattern21(pattern) if version == "2.1" else Pattern20(pattern) + builder = STIXPatternVisitorForSTIX21(module_suffix, module_name) if version == "2.1" else STIXPatternVisitorForSTIX20(module_suffix, module_name) return pattern_obj.visit(builder) diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 0c298f8..502d7d3 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -175,20 +175,34 @@ def test_greater_than(): assert str(exp) == "[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]" +def test_parsing_greater_than(): + patt_obj = create_pattern_object("[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.478901]") + assert str(patt_obj) == "[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.478901]" + + def test_less_than(): exp = stix2.LessThanComparisonExpression("file:size", 1024) assert str(exp) == "file:size < 1024" +def test_parsing_less_than(): + patt_obj = create_pattern_object("[file:size < 1024]") + assert str(patt_obj) == "[file:size < 1024]" + + def test_greater_than_or_equal(): exp = stix2.GreaterThanEqualComparisonExpression( "file:size", 1024, ) - assert str(exp) == "file:size >= 1024" +def test_parsing_greater_than_or_equal(): + patt_obj = create_pattern_object("[file:size >= 1024]") + assert str(patt_obj) == "[file:size >= 1024]" + + def test_less_than_or_equal(): exp = stix2.LessThanEqualComparisonExpression( "file:size", @@ -197,6 +211,11 @@ def test_less_than_or_equal(): assert str(exp) == "file:size <= 1024" +def test_parsing_less_than_or_equal(): + patt_obj = create_pattern_object("[file:size <= 1024]") + assert str(patt_obj) == "[file:size <= 1024]" + + def test_not(): exp = stix2.LessThanComparisonExpression( "file:size", @@ -257,6 +276,67 @@ def test_and_observable_expression(): assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']" # noqa +def test_parsing_and_observable_expression(): + exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") + assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" + + +def test_or_observable_expression(): + exp1 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1007"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Peter", + ), + ]) + exp2 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1008"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Paul", + ), + ]) + exp3 = stix2.AndBooleanExpression([ + stix2.EqualityComparisonExpression( + "user-account:account_type", + "unix", + ), + stix2.EqualityComparisonExpression( + "user-account:user_id", + stix2.StringConstant("1009"), + ), + stix2.EqualityComparisonExpression( + "user-account:account_login", + "Mary", + ), + ]) + exp = stix2.OrObservationExpression([ + stix2.ObservationExpression(exp1), + stix2.ObservationExpression(exp2), + stix2.ObservationExpression(exp3), + ]) + assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']" # noqa + + +def test_parsing_or_observable_expression(): + exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") # noqa + assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" # noqa + + def test_invalid_and_observable_expression(): with pytest.raises(ValueError): stix2.AndBooleanExpression([ @@ -286,6 +366,11 @@ def test_hex(): assert str(exp) == "[file:mime_type = 'image/bmp' AND file:magic_number_hex = h'ffd8']" +def test_parsing_hex(): + patt_obj = create_pattern_object("[file:magic_number_hex = h'ffd8']") + assert str(patt_obj) == "[file:magic_number_hex = h'ffd8']" + + def test_multiple_qualifiers(): exp_and = stix2.AndBooleanExpression([ stix2.EqualityComparisonExpression( @@ -334,6 +419,11 @@ def test_binary(): assert str(exp) == "artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q='" +def test_parsing_binary(): + patt_obj = create_pattern_object("[artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q=']") + assert str(patt_obj) == "[artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q=']" + + def test_list(): exp = stix2.InComparisonExpression( "process:name", @@ -499,7 +589,7 @@ def test_parsing_comparison_expression(): assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" -def test_parsing_qualified_expression(): +def test_parsing_repeat_and_within_qualified_expression(): patt_obj = create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS", ) @@ -508,11 +598,25 @@ def test_parsing_qualified_expression(): ) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS" +def test_parsing_start_stop_qualified_expression(): + patt_obj = create_pattern_object( + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'", + ) + assert str( + patt_obj, + ) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'" + + def test_list_constant(): patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]") assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" +def test_parsing_boolean(): + patt_obj = create_pattern_object("[network-traffic:is_active = true]") + assert str(patt_obj) == "[network-traffic:is_active = true]" + + def test_parsing_multiple_slashes_quotes(): patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]") assert str(patt_obj) == "[file:name = 'weird_name\\'']" From e3ebb6393d9ecd0d44cd85a0545c53ccefc9ab3a Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 27 Mar 2020 12:33:24 -0400 Subject: [PATCH 24/37] flaky --- stix2/pattern_visitor.py | 14 ++++++++------ stix2/test/v21/test_pattern_expressions.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index 51e309f..f947437 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -3,14 +3,16 @@ import inspect from stix2patterns.exceptions import ParseException from stix2patterns.grammars.STIXPatternParser import TerminalNode - -from stix2patterns.v20.grammars.STIXPatternParser import STIXPatternParser as STIXPatternParser20 -from stix2patterns.v21.grammars.STIXPatternParser import STIXPatternParser as STIXPatternParser21 - -from stix2patterns.v20.grammars.STIXPatternVisitor import STIXPatternVisitor as STIXPatternVisitor20 -from stix2patterns.v21.grammars.STIXPatternVisitor import STIXPatternVisitor as STIXPatternVisitor21 from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor +from stix2patterns.v20.grammars.STIXPatternParser import \ + STIXPatternParser as STIXPatternParser20 +from stix2patterns.v20.grammars.STIXPatternVisitor import \ + STIXPatternVisitor as STIXPatternVisitor20 from stix2patterns.v20.pattern import Pattern as Pattern20 +from stix2patterns.v21.grammars.STIXPatternParser import \ + STIXPatternParser as STIXPatternParser21 +from stix2patterns.v21.grammars.STIXPatternVisitor import \ + STIXPatternVisitor as STIXPatternVisitor21 from stix2patterns.v21.pattern import Pattern as Pattern21 from .patterns import * diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 502d7d3..804e88c 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -277,8 +277,8 @@ def test_and_observable_expression(): def test_parsing_and_observable_expression(): - exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") - assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" + exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") # noqa + assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" # noqa def test_or_observable_expression(): @@ -600,11 +600,11 @@ def test_parsing_repeat_and_within_qualified_expression(): def test_parsing_start_stop_qualified_expression(): patt_obj = create_pattern_object( - "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'", + "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'", # noqa ) assert str( patt_obj, - ) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'" + ) == "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'" # noqa def test_list_constant(): From 9933f88975652b35f06e6cfb5d8f158f2578551b Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 27 Mar 2020 13:59:03 -0400 Subject: [PATCH 25/37] few more pattern op tests --- stix2/pattern_visitor.py | 6 ++++-- stix2/patterns.py | 2 +- stix2/test/v21/test_pattern_expressions.py | 25 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index f947437..a891e12 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -3,7 +3,6 @@ import inspect from stix2patterns.exceptions import ParseException from stix2patterns.grammars.STIXPatternParser import TerminalNode -from stix2patterns.grammars.STIXPatternVisitor import STIXPatternVisitor from stix2patterns.v20.grammars.STIXPatternParser import \ STIXPatternParser as STIXPatternParser20 from stix2patterns.v20.grammars.STIXPatternVisitor import \ @@ -102,7 +101,10 @@ class STIXPatternVisitorForSTIX2(): # Visit a parse tree produced by STIXPatternParser#observationExpressionCompound. def visitObservationExpressionCompound(self, ctx): children = self.visitChildren(ctx) - return self.instantiate("ObservationExpression", children[1]) + if isinstance(children[0], TerminalNode) and children[0].symbol.type == self.parser_class.LPAREN: + return self.instantiate("ParentheticalExpression", children[1]) + else: + return self.instantiate("ObservationExpression", children[0]) # Visit a parse tree produced by STIXPatternParser#observationExpressionWithin. def visitObservationExpressionWithin(self, ctx): diff --git a/stix2/patterns.py b/stix2/patterns.py index 2e149be..f0cceb8 100644 --- a/stix2/patterns.py +++ b/stix2/patterns.py @@ -551,7 +551,7 @@ class ObservationExpression(_PatternExpression): self.operand = operand def __str__(self): - return "[%s]" % self.operand + return "%s" % self.operand if isinstance(self.operand, (ObservationExpression, _CompoundObservationExpression)) else "[%s]" % self.operand class _CompoundObservationExpression(_PatternExpression): diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 804e88c..f176db7 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -216,6 +216,31 @@ def test_parsing_less_than_or_equal(): assert str(patt_obj) == "[file:size <= 1024]" +def test_parsing_issubset(): + patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']") + assert str(patt_obj) == "[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']" + + +def test_parsing_issuperset(): + patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUPERSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']") + assert str(patt_obj) == "[network-traffic:dst_ref.value ISSUPERSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']" + + +def test_parsing_like(): + patt_obj = create_pattern_object("[directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo']") + assert str(patt_obj) == "[directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo']" + + +def test_parsing_match(): + patt_obj = create_pattern_object("[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS") # noqa + assert str(patt_obj) == "[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS" + + +def test_parsing_followed_by(): + patt_obj = create_pattern_object("([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS") # noqa + assert str(patt_obj) == "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS" # noqa + + def test_not(): exp = stix2.LessThanComparisonExpression( "file:size", From 1a2b1367cfb8af8b3fac98a38487d9e51b25651d Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 27 Mar 2020 14:06:24 -0400 Subject: [PATCH 26/37] flaky 2 --- stix2/test/v21/test_pattern_expressions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index f176db7..627ae5b 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -233,7 +233,7 @@ def test_parsing_like(): def test_parsing_match(): patt_obj = create_pattern_object("[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS") # noqa - assert str(patt_obj) == "[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS" + assert str(patt_obj) == "[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS" # noqa def test_parsing_followed_by(): From c911cff97fe9d89855c83e805f0702f2d9bdc02e Mon Sep 17 00:00:00 2001 From: "Desai, Kartikey H" Date: Fri, 27 Mar 2020 14:58:18 -0400 Subject: [PATCH 27/37] Add duplicate checking to markings and observable extensions, and fix some tests and add some tests. Fixes #363 --- stix2/core.py | 13 +++++-- stix2/exceptions.py | 14 +++++--- stix2/test/v20/test_core.py | 53 ---------------------------- stix2/test/v20/test_custom.py | 65 ++++++++++++++++++++++++++++++++--- stix2/test/v21/test_core.py | 43 ----------------------- stix2/test/v21/test_custom.py | 56 ++++++++++++++++++++++++++---- 6 files changed, 129 insertions(+), 115 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index ffe77f2..2f8de3e 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -8,7 +8,7 @@ import re import stix2 from .base import _Observable, _STIXBase -from .exceptions import DuplicateObjectRegistrationError, ParseError +from .exceptions import DuplicateRegistrationError, ParseError from .markings import _MarkingsMixin from .utils import SCO21_EXT_REGEX, TYPE_REGEX, _get_dict @@ -218,7 +218,7 @@ def _register_object(new_type, version=None): OBJ_MAP = STIX2_OBJ_MAPS[v]['objects'] if new_type._type in OBJ_MAP.keys(): - raise DuplicateObjectRegistrationError("An object with type '%s' already exists and cannot be created again." % new_type._type) + raise DuplicateRegistrationError("STIX Object", new_type._type) OBJ_MAP[new_type._type] = new_type @@ -238,6 +238,8 @@ def _register_marking(new_marking, version=None): v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') OBJ_MAP_MARKING = STIX2_OBJ_MAPS[v]['markings'] + if new_marking._type in OBJ_MAP_MARKING.keys(): + raise DuplicateRegistrationError("STIX Marking", new_marking._type) OBJ_MAP_MARKING[new_marking._type] = new_marking @@ -258,7 +260,7 @@ def _register_observable(new_observable, version=None): OBJ_MAP_OBSERVABLE = STIX2_OBJ_MAPS[v]['observables'] if new_observable._type in OBJ_MAP_OBSERVABLE.keys(): - raise DuplicateObjectRegistrationError("An observable with type '%s' already exists and cannot be created again." % new_observable._type) + raise DuplicateRegistrationError("Cyber Observable", new_observable._type) OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable @@ -323,6 +325,11 @@ def _register_observable_extension( EXT_MAP = STIX2_OBJ_MAPS[v]['observable-extensions'] try: + try: + if ext_type in EXT_MAP[observable_type].keys(): + raise DuplicateRegistrationError("Observable Extension", ext_type) + except AttributeError: + pass EXT_MAP[observable_type][ext_type] = new_extension except KeyError: if observable_type not in OBJ_MAP_OBSERVABLE: diff --git a/stix2/exceptions.py b/stix2/exceptions.py index f65f48b..edcc352 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -235,8 +235,14 @@ class STIXDeprecationWarning(DeprecationWarning): pass -class DuplicateObjectRegistrationError(STIXError): - """An object (or observable) with the same type as an existing object (or observable) is being registered""" +class DuplicateRegistrationError(STIXError): + """A STIX object with the same type as an existing object is being registered""" - def __init__(self, msg): - super(DuplicateObjectRegistrationError, self).__init__(msg) + def __init__(self, obj_type, reg_obj_type): + super(DuplicateRegistrationError, self).__init__() + self.obj_type = obj_type + self.reg_obj_type = reg_obj_type + + def __str__(self): + msg = "A(n) {0} with type '{1}' already exists and cannot be registered again" + return msg.format(self.obj_type, self.reg_obj_type) diff --git a/stix2/test/v20/test_core.py b/stix2/test/v20/test_core.py index 2808ba5..080c6f6 100644 --- a/stix2/test/v20/test_core.py +++ b/stix2/test/v20/test_core.py @@ -3,8 +3,6 @@ import pytest import stix2 from stix2 import core, exceptions -from .constants import IDENTITY_ID - BUNDLE = { "type": "bundle", "spec_version": "2.0", @@ -74,54 +72,3 @@ def test_register_marking_with_version(): assert stix2.v20.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v20.TLP_WHITE.__class__) - - -@pytest.mark.xfail(reason="The default version is no longer 2.0", condition=stix2.DEFAULT_VERSION != "2.0") -def test_register_marking_with_no_version(): - # Uses default version (2.0 in this case) - core._register_marking(stix2.v20.TLP_WHITE.__class__) - v = 'v20' - - assert stix2.v20.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] - assert v in str(stix2.v20.TLP_WHITE.__class__) - - -def test_register_observable_extension_with_version(): - observed_data = stix2.v20.ObservedData( - id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - created_by_ref=IDENTITY_ID, - created="2016-04-06T19:58:16.000Z", - modified="2016-04-06T19:58:16.000Z", - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=50, - objects={ - "0": { - "name": "foo.exe", - "type": "file", - "extensions": { - "ntfs-ext": { - "alternate_data_streams": [ - { - "name": "second.stream", - "size": 25536, - }, - ], - }, - }, - }, - "1": { - "type": "directory", - "path": "/usr/home", - "contains_refs": ["0"], - }, - }, - ) - core._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__, version='2.0') - v = 'v20' - - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] - assert v in str(observed_data.objects['0'].__class__) - - assert observed_data.objects['0'].extensions['ntfs-ext']._type in core.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] - assert v in str(observed_data.objects['0'].extensions['ntfs-ext'].__class__) diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 19cb28a..4aee66e 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -4,7 +4,7 @@ import stix2 from stix2 import core import stix2.v20 -from ...exceptions import DuplicateObjectRegistrationError, InvalidValueError +from ...exceptions import DuplicateRegistrationError, InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v20.Identity( @@ -1040,7 +1040,7 @@ def test_register_custom_object_with_version(): def test_register_duplicate_object_with_version(): - with pytest.raises(DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v20.CustomObject( 'x-new-type-2', [ ('property1', stix2.properties.StringProperty()), @@ -1049,7 +1049,7 @@ def test_register_duplicate_object_with_version(): ) class NewType2(object): pass - assert "An object with type 'x-new-type-2' already exists and cannot be created again." in str(excinfo.value) + assert "cannot be registered again" in str(excinfo.value) @stix2.v20.CustomObservable( @@ -1069,7 +1069,7 @@ def test_register_observable_with_version(): def test_register_duplicate_observable_with_version(): - with pytest.raises(DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v20.CustomObservable( 'x-new-observable-2', [ ('property1', stix2.properties.StringProperty()), @@ -1077,4 +1077,59 @@ def test_register_duplicate_observable_with_version(): ) class NewObservable2(object): pass - assert "An observable with type 'x-new-observable-2' already exists and cannot be created again." in str(excinfo.value) + assert "cannot be registered again" in str(excinfo.value) + + +@pytest.mark.xfail(reason="The default version is no longer 2.0", condition=stix2.DEFAULT_VERSION != "2.0") +def test_register_marking_with_no_version(): + @stix2.v20.CustomMarking( + 'x-new-obj-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + v = 'v20' + + no = NewObj2(property1='something') + assert no._type in core.STIX2_OBJ_MAPS[v]['markings'] + + +def test_register_observable_extension_with_version(): + @stix2.v20.CustomExtension( + stix2.v20.UserAccount, 'some-extension-2', [ + ('keys', stix2.properties.StringProperty(required=True)), + ], + ) + class SomeCustomExtension2: + pass + + v = 'v20' + example = SomeCustomExtension2(keys='test123') + + assert example._type in core.STIX2_OBJ_MAPS[v]['observable-extensions']['user-account'] + + +def test_register_duplicate_observable_extension(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v20.CustomExtension( + stix2.v20.UserAccount, 'some-extension-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], + ) + class NewExtension2(): + pass + assert "cannot be registered again" in str(excinfo.value) + + +def test_register_duplicate_marking(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v20.CustomMarking( + 'x-new-obj-2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + assert "cannot be registered again" in str(excinfo.value) diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_core.py index f9f9de6..78e492a 100644 --- a/stix2/test/v21/test_core.py +++ b/stix2/test/v21/test_core.py @@ -3,8 +3,6 @@ import pytest import stix2 from stix2 import core, exceptions -from .constants import IDENTITY_ID, OBSERVED_DATA_ID - BUNDLE = { "type": "bundle", "id": "bundle--00000000-0000-4000-8000-000000000007", @@ -89,44 +87,3 @@ def test_register_marking_with_no_version(): assert stix2.v21.TLP_WHITE.definition._type in core.STIX2_OBJ_MAPS[v]['markings'] assert v in str(stix2.v21.TLP_WHITE.__class__) - - -def test_register_observable_extension_with_default_version(): - observed_data = stix2.v21.ObservedData( - id=OBSERVED_DATA_ID, - created_by_ref=IDENTITY_ID, - created="2016-04-06T19:58:16.000Z", - modified="2016-04-06T19:58:16.000Z", - first_observed="2015-12-21T19:00:00Z", - last_observed="2015-12-21T19:00:00Z", - number_observed=50, - objects={ - "0": { - "name": "foo.exe", - "type": "file", - "extensions": { - "ntfs-ext": { - "alternate_data_streams": [ - { - "name": "second.stream", - "size": 25536, - }, - ], - }, - }, - }, - "1": { - "type": "directory", - "path": "/usr/home", - "contains_refs": ["file--420bc087-8b53-5ae9-8210-20d27d5e96c8"], - }, - }, - ) - core._register_observable_extension(observed_data.objects['0'], observed_data.objects['0'].extensions['ntfs-ext'].__class__) - v = 'v21' - - assert observed_data.objects['0'].type in core.STIX2_OBJ_MAPS[v]['observables'] - assert v in str(observed_data.objects['0'].__class__) - - assert observed_data.objects['0'].extensions['ntfs-ext']._type in core.STIX2_OBJ_MAPS[v]['observable-extensions']['file'] - assert v in str(observed_data.objects['0'].extensions['ntfs-ext'].__class__) diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 92cacf3..4522463 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -6,7 +6,7 @@ import stix2 import stix2.base import stix2.v21 -from ...exceptions import DuplicateObjectRegistrationError, InvalidValueError +from ...exceptions import DuplicateRegistrationError, InvalidValueError from .constants import FAKE_TIME, IDENTITY_ID, MARKING_DEFINITION_ID IDENTITY_CUSTOM_PROP = stix2.v21.Identity( @@ -1092,7 +1092,7 @@ def test_register_custom_object_with_version(): def test_register_duplicate_object_with_version(): - with pytest.raises(DuplicateObjectRegistrationError) as excinfo: + with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v21.CustomObject( 'x-new-type-2', [ ('property1', stix2.properties.StringProperty()), @@ -1101,7 +1101,7 @@ def test_register_duplicate_object_with_version(): ) class NewType2(object): pass - assert "An object with type 'x-new-type-2' already exists and cannot be created again." in str(excinfo.value) + assert "cannot be registered again" in str(excinfo.value) @stix2.v21.CustomObservable( @@ -1113,15 +1113,15 @@ class NewObservable3(object): pass -def test_register_observable_with_version(): +def test_register_observable(): custom_obs = NewObservable3(property1="Test Observable") v = 'v21' assert custom_obs.type in stix2.core.STIX2_OBJ_MAPS[v]['observables'] -def test_register_duplicate_observable_with_version(): - with pytest.raises(DuplicateObjectRegistrationError) as excinfo: +def test_register_duplicate_observable(): + with pytest.raises(DuplicateRegistrationError) as excinfo: @stix2.v21.CustomObservable( 'x-new-observable-2', [ ('property1', stix2.properties.StringProperty()), @@ -1129,4 +1129,46 @@ def test_register_duplicate_observable_with_version(): ) class NewObservable2(object): pass - assert "An observable with type 'x-new-observable-2' already exists and cannot be created again." in str(excinfo.value) + assert "cannot be registered again" in str(excinfo.value) + + +def test_register_observable_custom_extension(): + @stix2.v21.CustomExtension( + stix2.v21.DomainName, 'x-new-2-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], + ) + class NewExtension2(): + pass + + example = NewExtension2(property1="Hi there") + v = 'v21' + + assert 'domain-name' in stix2.core.STIX2_OBJ_MAPS[v]['observables'] + assert example._type in stix2.core.STIX2_OBJ_MAPS[v]['observable-extensions']['domain-name'] + + +def test_register_duplicate_observable_extension(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v21.CustomExtension( + stix2.v21.DomainName, 'x-new-2-ext', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ], + ) + class NewExtension2(): + pass + assert "cannot be registered again" in str(excinfo.value) + + +def test_register_duplicate_marking(): + with pytest.raises(DuplicateRegistrationError) as excinfo: + @stix2.v21.CustomMarking( + 'x-new-obj', [ + ('property1', stix2.properties.StringProperty(required=True)), + ], + ) + class NewObj2(): + pass + assert "cannot be registered again" in str(excinfo.value) From c494a2e47798f9df0e189158f9022da88bca9378 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 1 Apr 2020 21:52:04 -0400 Subject: [PATCH 28/37] Use TypeProperty.clean() to verify type format --- stix2/custom.py | 42 +----------------------- stix2/parsing.py | 62 ++++++++++------------------------- stix2/properties.py | 31 ++++++++++++++++-- stix2/test/v21/test_custom.py | 6 ++-- stix2/utils.py | 1 - stix2/v20/bundle.py | 2 +- stix2/v20/common.py | 2 +- stix2/v20/observables.py | 38 ++++++++++----------- stix2/v20/sdo.py | 26 +++++++-------- stix2/v20/sro.py | 4 +-- stix2/v21/bundle.py | 2 +- stix2/v21/common.py | 4 +-- stix2/v21/observables.py | 38 ++++++++++----------- stix2/v21/sdo.py | 38 ++++++++++----------- stix2/v21/sro.py | 4 +-- 15 files changed, 129 insertions(+), 171 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 6d1abc5..387906b 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -8,34 +8,12 @@ from .parsing import ( _register_marking, _register_object, _register_observable, _register_observable_extension, ) -from .utils import ( - PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, get_class_hierarchy_names, -) +from .utils import PREFIX_21_REGEX, get_class_hierarchy_names def _custom_object_builder(cls, type, properties, version, base_class): class _CustomObject(cls, base_class): - if version == "2.0": - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - type, - ) - else: # 2.1+ - if not re.match(TYPE_21_REGEX, type): - raise ValueError( - "Invalid type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " - "and must begin with an a-z character" % type, - ) - - if len(type) < 3 or len(type) > 250: - raise ValueError( - "Invalid type name '%s': must be between 3 and 250 characters." % type, - ) - if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") @@ -79,24 +57,6 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co class _CustomObservable(cls, base_class): - if version == "2.0": - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid observable type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - type, - ) - else: # 2.1+ - if not re.match(TYPE_21_REGEX, type): - raise ValueError( - "Invalid observable type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " - "and must begin with an a-z character" % type, - ) - - if len(type) < 3 or len(type) > 250: - raise ValueError("Invalid observable type name '%s': must be between 3 and 250 characters." % type) - if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") diff --git a/stix2/parsing.py b/stix2/parsing.py index 7dedc15..f355f41 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -7,11 +7,9 @@ import re import stix2 -from .base import _Observable +from .base import _DomainObject, _Observable from .exceptions import ParseError -from .utils import ( - EXT_21_REGEX, PREFIX_21_REGEX, TYPE_21_REGEX, TYPE_REGEX, _get_dict, -) +from .utils import PREFIX_21_REGEX, _get_dict STIX2_OBJ_MAPS = {} @@ -204,6 +202,14 @@ def _register_object(new_type, version=None): """ + if not issubclass(new_type, _DomainObject): + raise ValueError( + "'%s' must be created with the @CustomObject decorator." % + new_type.__name__, + ) + + new_type._properties['type'].clean(new_type._type) + if version: v = 'v' + version.replace('.', '') else: @@ -224,28 +230,6 @@ def _register_marking(new_marking, version=None): """ - type = new_marking._type - - if version == "2.0": - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid marking type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - type, - ) - else: # 2.1+ - if not re.match(TYPE_21_REGEX, type): - raise ValueError( - "Invalid marking type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " - "and must begin with an a-z character" % type, - ) - - if len(type) < 3 or len(type) > 250: - raise ValueError( - "Invalid marking type name '%s': must be between 3 and 250 characters." % type, - ) - properties = new_marking._properties if version == "2.1": @@ -273,6 +257,8 @@ def _register_observable(new_observable, version=None): """ + new_observable._properties['type'].clean(new_observable._type) + if version: v = 'v' + version.replace('.', '') else: @@ -304,26 +290,12 @@ def _register_observable_extension( if not issubclass(obs_class, _Observable): raise ValueError("'observable' must be a valid Observable class!") - if version == "2.0": - if not re.match(TYPE_REGEX, ext_type): - raise ValueError( - "Invalid extension type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - ext_type, - ) - else: # 2.1+ - if not re.match(EXT_21_REGEX, ext_type): - raise ValueError( - "Invalid extension type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, hyphen (-), " - "must begin with an a-z character" - "and end with '-ext'." % ext_type, - ) - - if len(ext_type) < 3 or len(ext_type) > 250: + temp_prop = stix2.properties.TypeProperty(ext_type, spec_version=version) + temp_prop.clean(ext_type) + if not ext_type.endswith('-ext'): raise ValueError( - "Invalid extension type name '%s': must be between 3 and 250" - " characters." % ext_type, + "Invalid extension type name '%s': must end with '-ext'." % + ext_type, ) if not new_extension._properties: diff --git a/stix2/properties.py b/stix2/properties.py index be66533..4ccfa89 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -17,7 +17,10 @@ from .exceptions import ( MutuallyExclusivePropertiesError, ) from .parsing import STIX2_OBJ_MAPS, parse, parse_observable -from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime +from .utils import ( + TYPE_21_REGEX, TYPE_REGEX, _get_dict, get_class_hierarchy_names, + parse_into_datetime, +) try: from collections.abc import Mapping @@ -232,9 +235,33 @@ class StringProperty(Property): class TypeProperty(Property): - def __init__(self, type): + def __init__(self, type, spec_version=stix2.DEFAULT_VERSION): + self.spec_version = spec_version super(TypeProperty, self).__init__(fixed=type) + def clean(self, value): + if self.spec_version == "2.0": + if not re.match(TYPE_REGEX, type): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type, + ) + else: # 2.1+ + if not re.match(TYPE_21_REGEX, type): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type, + ) + + if len(type) < 3 or len(type) > 250: + raise ValueError( + "Invalid type name '%s': must be between 3 and 250 characters." % type, + ) + + return value + class IDProperty(Property): diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index a9969b8..2183d4f 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -1186,9 +1186,9 @@ def test_register_custom_object(): class CustomObject2(object): _type = 'awesome-object' - stix2.parsing._register_object(CustomObject2, version="2.1") - # Note that we will always check against newest OBJ_MAP. - assert (CustomObject2._type, CustomObject2) in stix2.v21.OBJ_MAP.items() + with pytest.raises(ValueError) as excinfo: + stix2.parsing._register_object(CustomObject2, version="2.1") + assert '@CustomObject decorator' in str(excinfo) def test_extension_property_location(): diff --git a/stix2/utils.py b/stix2/utils.py index 69489d9..7acf523 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -28,7 +28,6 @@ STIX_UNMOD_PROPERTIES = ['created', 'created_by_ref', 'id', 'type'] TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') -EXT_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-ext$') PREFIX_21_REGEX = re.compile(r'^[a-z].*') diff --git a/stix2/v20/bundle.py b/stix2/v20/bundle.py index b811a79..25b948d 100644 --- a/stix2/v20/bundle.py +++ b/stix2/v20/bundle.py @@ -15,7 +15,7 @@ class Bundle(_STIXBase20): _type = 'bundle' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), # Not technically correct: STIX 2.0 spec doesn't say spec_version must # have this value, but it's all we support for now. diff --git a/stix2/v20/common.py b/stix2/v20/common.py index 55f2c3e..a92f81d 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -120,7 +120,7 @@ class MarkingDefinition(_STIXBase20, _MarkingsMixin): _type = 'marking-definition' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW)), diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index eb9bfde..3491c47 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -26,7 +26,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), ('url', StringProperty()), @@ -47,7 +47,7 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), @@ -62,7 +62,7 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself @@ -81,7 +81,7 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), @@ -95,7 +95,7 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), @@ -127,7 +127,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), ('content_type', StringProperty()), @@ -306,7 +306,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('hashes', HashesProperty()), ('size', IntegerProperty()), ('name', StringProperty()), @@ -339,7 +339,7 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -354,7 +354,7 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), @@ -369,7 +369,7 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -382,7 +382,7 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -483,7 +483,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('start', TimestampProperty()), ('end', TimestampProperty()), ('is_active', BooleanProperty()), @@ -575,7 +575,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), ('name', StringProperty()), @@ -615,7 +615,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('name', StringProperty(required=True)), ('cpe', StringProperty()), ('languages', ListProperty(StringProperty)), @@ -632,7 +632,7 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=_type)), ]) @@ -659,7 +659,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('user_id', StringProperty(required=True)), ('account_login', StringProperty()), ('account_type', StringProperty()), # open vocab @@ -713,7 +713,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('key', StringProperty(required=True)), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself @@ -757,7 +757,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty()), ('version', StringProperty()), @@ -791,7 +791,7 @@ def CustomObservable(type='x-custom-observable', properties=None): """ def wrapper(cls): _properties = list(itertools.chain.from_iterable([ - [('type', TypeProperty(type))], + [('type', TypeProperty(type, spec_version='2.0'))], properties, [('extensions', ExtensionsProperty(spec_version="2.0", enclosing_type=type))], ])) diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py index cb5ba04..8cbd94b 100644 --- a/stix2/v20/sdo.py +++ b/stix2/v20/sdo.py @@ -24,7 +24,7 @@ class AttackPattern(_DomainObject): _type = 'attack-pattern' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -47,7 +47,7 @@ class Campaign(_DomainObject): _type = 'campaign' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -73,7 +73,7 @@ class CourseOfAction(_DomainObject): _type = 'course-of-action' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -95,7 +95,7 @@ class Identity(_DomainObject): _type = 'identity' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -120,7 +120,7 @@ class Indicator(_DomainObject): _type = 'indicator' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -151,7 +151,7 @@ class IntrusionSet(_DomainObject): _type = 'intrusion-set' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -180,7 +180,7 @@ class Malware(_DomainObject): _type = 'malware' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -203,7 +203,7 @@ class ObservedData(_DomainObject): _type = 'observed-data' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -233,7 +233,7 @@ class Report(_DomainObject): _type = 'report' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -257,7 +257,7 @@ class ThreatActor(_DomainObject): _type = 'threat-actor' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -287,7 +287,7 @@ class Tool(_DomainObject): _type = 'tool' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -311,7 +311,7 @@ class Vulnerability(_DomainObject): _type = 'vulnerability' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -358,7 +358,7 @@ def CustomObject(type='x-custom-type', properties=None): def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [ - ('type', TypeProperty(type)), + ('type', TypeProperty(type, spec_version='2.0')), ('id', IDProperty(type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py index b7d1ad3..1372a5e 100644 --- a/stix2/v20/sro.py +++ b/stix2/v20/sro.py @@ -20,7 +20,7 @@ class Relationship(_RelationshipObject): _type = 'relationship' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -59,7 +59,7 @@ class Sighting(_RelationshipObject): _type = 'sighting' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.0')), ('id', IDProperty(_type, spec_version='2.0')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.0')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), diff --git a/stix2/v21/bundle.py b/stix2/v21/bundle.py index ca60fe0..1b8c652 100644 --- a/stix2/v21/bundle.py +++ b/stix2/v21/bundle.py @@ -16,7 +16,7 @@ class Bundle(_STIXBase21): _type = 'bundle' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('objects', ListProperty(STIXObjectProperty(spec_version='2.1'))), ]) diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 4254767..115bdb5 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -87,7 +87,7 @@ class LanguageContent(_STIXBase21): _type = 'language-content' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -158,7 +158,7 @@ class MarkingDefinition(_STIXBase21, _MarkingsMixin): _type = 'marking-definition' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type)), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py index e71e445..275bbbe 100644 --- a/stix2/v21/observables.py +++ b/stix2/v21/observables.py @@ -29,7 +29,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), @@ -59,7 +59,7 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('number', IntegerProperty(required=True)), ('name', StringProperty()), @@ -81,7 +81,7 @@ class Directory(_Observable): _type = 'directory' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), @@ -107,7 +107,7 @@ class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'], spec_version='2.1'))), @@ -128,7 +128,7 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('display_name', StringProperty()), @@ -168,7 +168,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), @@ -361,7 +361,7 @@ class File(_Observable): _type = 'file' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('hashes', HashesProperty(spec_version='2.1')), ('size', IntegerProperty(min=0)), @@ -397,7 +397,7 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), @@ -419,7 +419,7 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ReferenceProperty(valid_types='mac-addr', spec_version='2.1'))), @@ -441,7 +441,7 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -461,7 +461,7 @@ class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -586,7 +586,7 @@ class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('start', TimestampProperty()), ('end', TimestampProperty()), @@ -711,7 +711,7 @@ class Process(_Observable): _type = 'process' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_hidden', BooleanProperty()), ('pid', IntegerProperty()), @@ -756,7 +756,7 @@ class Software(_Observable): _type = 'software' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('name', StringProperty(required=True)), ('cpe', StringProperty()), @@ -781,7 +781,7 @@ class URL(_Observable): _type = 'url' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('value', StringProperty(required=True)), ('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=_type)), @@ -816,7 +816,7 @@ class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('user_id', StringProperty()), ('credential', StringProperty()), @@ -879,7 +879,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('key', StringProperty()), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), @@ -931,7 +931,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty(spec_version='2.1')), @@ -983,7 +983,7 @@ def CustomObservable(type='x-custom-observable', properties=None, id_contrib_pro """ def wrapper(cls): _properties = list(itertools.chain.from_iterable([ - [('type', TypeProperty(type))], + [('type', TypeProperty(type, spec_version='2.1'))], [('id', IDProperty(type, spec_version='2.1'))], properties, [('extensions', ExtensionsProperty(spec_version='2.1', enclosing_type=type))], diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py index 1e9633e..396b030 100644 --- a/stix2/v21/sdo.py +++ b/stix2/v21/sdo.py @@ -29,7 +29,7 @@ class AttackPattern(_DomainObject): _type = 'attack-pattern' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -57,7 +57,7 @@ class Campaign(_DomainObject): _type = 'campaign' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -97,7 +97,7 @@ class CourseOfAction(_DomainObject): _type = 'course-of-action' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -123,7 +123,7 @@ class Grouping(_DomainObject): _type = 'grouping' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -151,7 +151,7 @@ class Identity(_DomainObject): _type = 'identity' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -181,7 +181,7 @@ class Indicator(_DomainObject): _type = 'indicator' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -241,7 +241,7 @@ class Infrastructure(_DomainObject): _type = 'infrastructure' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -282,7 +282,7 @@ class IntrusionSet(_DomainObject): _type = 'intrusion-set' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -325,7 +325,7 @@ class Location(_DomainObject): _type = 'location' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -433,7 +433,7 @@ class Malware(_DomainObject): _type = 'malware' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -486,7 +486,7 @@ class MalwareAnalysis(_DomainObject): _type = 'malware-analysis' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), @@ -531,7 +531,7 @@ class Note(_DomainObject): _type = 'note' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -559,7 +559,7 @@ class ObservedData(_DomainObject): _type = 'observed-data' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -615,7 +615,7 @@ class Opinion(_DomainObject): _type = 'opinion' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -653,7 +653,7 @@ class Report(_DomainObject): _type = 'report' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -682,7 +682,7 @@ class ThreatActor(_DomainObject): _type = 'threat-actor' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -729,7 +729,7 @@ class Tool(_DomainObject): _type = 'tool' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -759,7 +759,7 @@ class Vulnerability(_DomainObject): _type = 'vulnerability' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -809,7 +809,7 @@ def CustomObject(type='x-custom-type', properties=None): def wrapper(cls): _properties = list(itertools.chain.from_iterable([ [ - ('type', TypeProperty(type)), + ('type', TypeProperty(type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py index 3ffdd77..9f85004 100644 --- a/stix2/v21/sro.py +++ b/stix2/v21/sro.py @@ -21,7 +21,7 @@ class Relationship(_RelationshipObject): _type = 'relationship' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), @@ -76,7 +76,7 @@ class Sighting(_RelationshipObject): _type = 'sighting' _properties = OrderedDict([ - ('type', TypeProperty(_type)), + ('type', TypeProperty(_type, spec_version='2.1')), ('spec_version', StringProperty(fixed='2.1')), ('id', IDProperty(_type, spec_version='2.1')), ('created_by_ref', ReferenceProperty(valid_types='identity', spec_version='2.1')), From 897e884217e93aa2b3bc76cb6bf242be71400683 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 2 Apr 2020 04:46:11 -0400 Subject: [PATCH 29/37] Fix some testing --- .travis.yml | 2 +- stix2/core.py | 7 ++----- stix2/test/v20/test_custom.py | 3 +-- tox.ini | 4 ++-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 018aefe..81141d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +os: linux language: python cache: pip dist: xenial diff --git a/stix2/core.py b/stix2/core.py index 2f8de3e..dd7aac1 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -325,11 +325,8 @@ def _register_observable_extension( EXT_MAP = STIX2_OBJ_MAPS[v]['observable-extensions'] try: - try: - if ext_type in EXT_MAP[observable_type].keys(): - raise DuplicateRegistrationError("Observable Extension", ext_type) - except AttributeError: - pass + if ext_type in EXT_MAP[observable_type].keys(): + raise DuplicateRegistrationError("Observable Extension", ext_type) EXT_MAP[observable_type][ext_type] = new_extension except KeyError: if observable_type not in OBJ_MAP_OBSERVABLE: diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 4aee66e..346f06b 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -1080,8 +1080,7 @@ def test_register_duplicate_observable_with_version(): assert "cannot be registered again" in str(excinfo.value) -@pytest.mark.xfail(reason="The default version is no longer 2.0", condition=stix2.DEFAULT_VERSION != "2.0") -def test_register_marking_with_no_version(): +def test_register_marking_with_version(): @stix2.v20.CustomMarking( 'x-new-obj-2', [ ('property1', stix2.properties.StringProperty(required=True)), diff --git a/tox.ini b/tox.ini index d34aac1..aa0472a 100644 --- a/tox.ini +++ b/tox.ini @@ -8,11 +8,11 @@ deps = pytest pytest-cov coverage - taxii2-client + taxii2-client<1.0.0 fuzzywuzzy haversine python-Levenshtein - medallion + medallion<2.0.0 commands = python -m pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning From 13cddf9d6d4e2f97b7e8f4db20022f553369e747 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 2 Apr 2020 08:17:34 -0400 Subject: [PATCH 30/37] Move TypeProperty format checks to __init__ TypeProperty uses a fixed value, so check() was never called. This way also runs the check at object registration time because the wrapper creates an instance of TypeProperty and doesn't have to wait for the object to be instantiated so clean() can be called. Also fix some tests. --- stix2/parsing.py | 21 ++++++++------ stix2/properties.py | 54 ++++++++++++++++++++--------------- stix2/test/v20/test_custom.py | 13 ++++----- stix2/test/v21/test_custom.py | 18 ++++++------ 4 files changed, 58 insertions(+), 48 deletions(-) diff --git a/stix2/parsing.py b/stix2/parsing.py index b81a07d..db0c693 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -200,6 +200,11 @@ def _register_object(new_type, version=None): version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. + Raises: + ValueError: If the class being registered wasn't created with the + @CustomObject decorator. + DuplicateRegistrationError: If the class has already been registered. + """ if not issubclass(new_type, _DomainObject): @@ -208,8 +213,6 @@ def _register_object(new_type, version=None): new_type.__name__, ) - new_type._properties['type'].clean(new_type._type) - if version: v = 'v' + version.replace('.', '') else: @@ -232,8 +235,11 @@ def _register_marking(new_marking, version=None): """ + mark_type = new_marking._type properties = new_marking._properties + stix2.properties._validate_type(mark_type, version) + if version == "2.1": for prop_name, prop_value in properties.items(): if not re.match(PREFIX_21_REGEX, prop_name): @@ -246,9 +252,9 @@ def _register_marking(new_marking, version=None): v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') OBJ_MAP_MARKING = STIX2_OBJ_MAPS[v]['markings'] - if new_marking._type in OBJ_MAP_MARKING.keys(): - raise DuplicateRegistrationError("STIX Marking", new_marking._type) - OBJ_MAP_MARKING[new_marking._type] = new_marking + if mark_type in OBJ_MAP_MARKING.keys(): + raise DuplicateRegistrationError("STIX Marking", mark_type) + OBJ_MAP_MARKING[mark_type] = new_marking def _register_observable(new_observable, version=None): @@ -261,8 +267,6 @@ def _register_observable(new_observable, version=None): """ - new_observable._properties['type'].clean(new_observable._type) - if version: v = 'v' + version.replace('.', '') else: @@ -296,8 +300,7 @@ def _register_observable_extension( if not issubclass(obs_class, _Observable): raise ValueError("'observable' must be a valid Observable class!") - temp_prop = stix2.properties.TypeProperty(ext_type, spec_version=version) - temp_prop.clean(ext_type) + stix2.properties._validate_type(ext_type, version) if not new_extension._properties: raise ValueError( diff --git a/stix2/properties.py b/stix2/properties.py index 4ccfa89..39c409c 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -84,6 +84,36 @@ def _validate_id(id_, spec_version, required_prefix): raise ValueError(ERROR_INVALID_ID.format(id_)) +def _validate_type(type_, spec_version): + """ + Check the STIX type name for correctness, raise an exception if there are + errors. + + :param type_: The STIX type name + :param spec_version: The STIX specification version to use + :raises ValueError: If there are any errors with the identifier + """ + if spec_version == "2.0": + if not re.match(TYPE_REGEX, type_): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % + type_, + ) + else: # 2.1+ + if not re.match(TYPE_21_REGEX, type_): + raise ValueError( + "Invalid type name '%s': must only contain the " + "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " + "and must begin with an a-z character" % type_, + ) + + if len(type_) < 3 or len(type_) > 250: + raise ValueError( + "Invalid type name '%s': must be between 3 and 250 characters." % type_, + ) + + class Property(object): """Represent a property of STIX data type. @@ -236,32 +266,10 @@ class StringProperty(Property): class TypeProperty(Property): def __init__(self, type, spec_version=stix2.DEFAULT_VERSION): + _validate_type(type, spec_version) self.spec_version = spec_version super(TypeProperty, self).__init__(fixed=type) - def clean(self, value): - if self.spec_version == "2.0": - if not re.match(TYPE_REGEX, type): - raise ValueError( - "Invalid type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % - type, - ) - else: # 2.1+ - if not re.match(TYPE_21_REGEX, type): - raise ValueError( - "Invalid type name '%s': must only contain the " - "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " - "and must begin with an a-z character" % type, - ) - - if len(type) < 3 or len(type) > 250: - raise ValueError( - "Invalid type name '%s': must be between 3 and 250 characters." % type, - ) - - return value - class IDProperty(Property): diff --git a/stix2/test/v20/test_custom.py b/stix2/test/v20/test_custom.py index 8342fec..57f8aac 100644 --- a/stix2/test/v20/test_custom.py +++ b/stix2/test/v20/test_custom.py @@ -483,7 +483,7 @@ def test_custom_observable_object_invalid_type_name(): ) class NewObs(object): pass # pragma: no cover - assert "Invalid observable type name 'x':" in str(excinfo.value) + assert "Invalid type name 'x':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v20.CustomObservable( @@ -493,7 +493,7 @@ def test_custom_observable_object_invalid_type_name(): ) class NewObs2(object): pass # pragma: no cover - assert "Invalid observable type name 'x_new_obs':" in str(excinfo.value) + assert "Invalid type name 'x_new_obs':" in str(excinfo.value) def test_custom_observable_object_invalid_ref_property(): @@ -808,7 +808,7 @@ def test_custom_extension_invalid_type_name(): ) class FooExtension(): pass # pragma: no cover - assert "Invalid extension type name 'x':" in str(excinfo.value) + assert "Invalid type name 'x':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v20.CustomExtension( @@ -818,7 +818,7 @@ def test_custom_extension_invalid_type_name(): ) class BlaExtension(): pass # pragma: no cover - assert "Invalid extension type name 'x_new_ext':" in str(excinfo.value) + assert "Invalid type name 'x_new_ext':" in str(excinfo.value) def test_custom_extension_no_properties(): @@ -968,9 +968,8 @@ def test_register_custom_object(): class CustomObject2(object): _type = 'awesome-object' - stix2.parsing._register_object(CustomObject2, version="2.0") - # Note that we will always check against newest OBJ_MAP. - assert (CustomObject2._type, CustomObject2) in stix2.v20.OBJ_MAP.items() + with pytest.raises(ValueError): + stix2.parsing._register_object(CustomObject2, version="2.0") def test_extension_property_location(): diff --git a/stix2/test/v21/test_custom.py b/stix2/test/v21/test_custom.py index 59eb53e..3646f11 100644 --- a/stix2/test/v21/test_custom.py +++ b/stix2/test/v21/test_custom.py @@ -375,7 +375,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj(object): pass # pragma: no cover - assert "Invalid marking type name 'x': " in str(excinfo.value) + assert "Invalid type name 'x': " in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomMarking( @@ -385,7 +385,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj2(object): pass # pragma: no cover - assert "Invalid marking type name 'x_new_marking':" in str(excinfo.value) + assert "Invalid type name 'x_new_marking':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomMarking( @@ -395,7 +395,7 @@ def test_custom_marking_invalid_type_name(): ) class NewObj3(object): pass # pragma: no cover - assert "Invalid marking type name '7x-new-marking':" in str(excinfo.value) + assert "Invalid type name '7x-new-marking':" in str(excinfo.value) # Custom Objects @@ -619,7 +619,7 @@ def test_custom_observable_object_invalid_type_name(): ) class NewObs(object): pass # pragma: no cover - assert "Invalid observable type name 'x':" in str(excinfo.value) + assert "Invalid type name 'x':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomObservable( @@ -629,7 +629,7 @@ def test_custom_observable_object_invalid_type_name(): ) class NewObs2(object): pass # pragma: no cover - assert "Invalid observable type name 'x_new_obs':" in str(excinfo.value) + assert "Invalid type name 'x_new_obs':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomObservable( @@ -639,7 +639,7 @@ def test_custom_observable_object_invalid_type_name(): ) class NewObs3(object): pass # pragma: no cover - assert "Invalid observable type name '7x-new-obs':" in str(excinfo.value) + assert "Invalid type name '7x-new-obs':" in str(excinfo.value) def test_custom_observable_object_invalid_ref_property(): @@ -1005,7 +1005,7 @@ def test_custom_extension_invalid_type_name(): ) class FooExtension(): pass # pragma: no cover - assert "Invalid extension type name 'x':" in str(excinfo.value) + assert "Invalid type name 'x':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( @@ -1015,7 +1015,7 @@ def test_custom_extension_invalid_type_name(): ) class BlaExtension(): pass # pragma: no cover - assert "Invalid extension type name 'x_new_ext':" in str(excinfo.value) + assert "Invalid type name 'x_new_ext':" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: @stix2.v21.CustomExtension( @@ -1025,7 +1025,7 @@ def test_custom_extension_invalid_type_name(): ) class Bla2Extension(): pass # pragma: no cover - assert "Invalid extension type name '7x-new-ext':" in str(excinfo.value) + assert "Invalid type name '7x-new-ext':" in str(excinfo.value) def test_custom_extension_no_properties(): From d33adbcc716d53c12f8d4672e62ac9f41d748728 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 2 Apr 2020 08:22:49 -0400 Subject: [PATCH 31/37] Rename test files to align with module renaming --- stix2/test/v20/{test_core.py => test_parsing.py} | 0 stix2/test/v21/{test_core.py => test_parsing.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename stix2/test/v20/{test_core.py => test_parsing.py} (100%) rename stix2/test/v21/{test_core.py => test_parsing.py} (100%) diff --git a/stix2/test/v20/test_core.py b/stix2/test/v20/test_parsing.py similarity index 100% rename from stix2/test/v20/test_core.py rename to stix2/test/v20/test_parsing.py diff --git a/stix2/test/v21/test_core.py b/stix2/test/v21/test_parsing.py similarity index 100% rename from stix2/test/v21/test_core.py rename to stix2/test/v21/test_parsing.py From 14540c0ea16d9ec34851623c15f4471501e84cd5 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 2 Apr 2020 14:15:45 -0400 Subject: [PATCH 32/37] Clean up _register_* functions Made them consistent with _register_observable_extension, by: - moving validation logic there from _custom_*_builder functions - using a new function for ensuring properties are dict-like - using the library default spec version instead of None Fix #371, fix #372, fix #373. --- stix2/custom.py | 85 +++++++++------------------------ stix2/parsing.py | 47 ++++++++++++++++-- stix2/test/v20/test_markings.py | 4 +- stix2/test/v21/test_markings.py | 4 +- 4 files changed, 68 insertions(+), 72 deletions(-) diff --git a/stix2/custom.py b/stix2/custom.py index 387906b..f749b04 100644 --- a/stix2/custom.py +++ b/stix2/custom.py @@ -1,5 +1,4 @@ from collections import OrderedDict -import re import six @@ -8,22 +7,29 @@ from .parsing import ( _register_marking, _register_object, _register_observable, _register_observable_extension, ) -from .utils import PREFIX_21_REGEX, get_class_hierarchy_names + + +def _get_properties_dict(properties): + try: + return OrderedDict(properties) + except TypeError as e: + six.raise_from( + ValueError( + "properties must be dict-like, e.g. a list " + "containing tuples. For example, " + "[('property1', IntegerProperty())]", + ), + e, + ) def _custom_object_builder(cls, type, properties, version, base_class): + prop_dict = _get_properties_dict(properties) + class _CustomObject(cls, base_class): - if not properties or not isinstance(properties, list): - raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - - if version == "2.1": - for prop_name, prop in properties: - if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name '%s' must begin with an alpha character" % prop_name) - _type = type - _properties = OrderedDict(properties) + _properties = prop_dict def __init__(self, **kwargs): base_class.__init__(self, **kwargs) @@ -34,14 +40,12 @@ def _custom_object_builder(cls, type, properties, version, base_class): def _custom_marking_builder(cls, type, properties, version, base_class): + prop_dict = _get_properties_dict(properties) class _CustomMarking(cls, base_class): - if not properties or not isinstance(properties, list): - raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - _type = type - _properties = OrderedDict(properties) + _properties = prop_dict def __init__(self, **kwargs): base_class.__init__(self, **kwargs) @@ -55,44 +59,12 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co if id_contrib_props is None: id_contrib_props = [] + prop_dict = _get_properties_dict(properties) + class _CustomObservable(cls, base_class): - if not properties or not isinstance(properties, list): - raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - - if version == "2.0": - # If using STIX2.0, check properties ending in "_ref/s" are ObjectReferenceProperties - for prop_name, prop in properties: - if prop_name.endswith('_ref') and ('ObjectReferenceProperty' not in get_class_hierarchy_names(prop)): - raise ValueError( - "'%s' is named like an object reference property but " - "is not an ObjectReferenceProperty." % prop_name, - ) - elif (prop_name.endswith('_refs') and ('ListProperty' not in get_class_hierarchy_names(prop) or - 'ObjectReferenceProperty' not in get_class_hierarchy_names(prop.contained))): - raise ValueError( - "'%s' is named like an object reference list property but " - "is not a ListProperty containing ObjectReferenceProperty." % prop_name, - ) - else: - # If using STIX2.1 (or newer...), check properties ending in "_ref/s" are ReferenceProperties - for prop_name, prop in properties: - if not re.match(PREFIX_21_REGEX, prop_name): - raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) - elif prop_name.endswith('_ref') and ('ReferenceProperty' not in get_class_hierarchy_names(prop)): - raise ValueError( - "'%s' is named like a reference property but " - "is not a ReferenceProperty." % prop_name, - ) - elif (prop_name.endswith('_refs') and ('ListProperty' not in get_class_hierarchy_names(prop) or - 'ReferenceProperty' not in get_class_hierarchy_names(prop.contained))): - raise ValueError( - "'%s' is named like a reference list property but " - "is not a ListProperty containing ReferenceProperty." % prop_name, - ) - _type = type - _properties = OrderedDict(properties) + _properties = prop_dict if version != '2.0': _id_contributing_properties = id_contrib_props @@ -105,18 +77,7 @@ def _custom_observable_builder(cls, type, properties, version, base_class, id_co def _custom_extension_builder(cls, observable, type, properties, version, base_class): - - try: - prop_dict = OrderedDict(properties) - except TypeError as e: - six.raise_from( - ValueError( - "Extension properties must be dict-like, e.g. a list " - "containing tuples. For example, " - "[('property1', IntegerProperty())]", - ), - e, - ) + prop_dict = _get_properties_dict(properties) class _CustomExtension(cls, base_class): diff --git a/stix2/parsing.py b/stix2/parsing.py index db0c693..baa3b7b 100644 --- a/stix2/parsing.py +++ b/stix2/parsing.py @@ -9,7 +9,7 @@ import stix2 from .base import _DomainObject, _Observable from .exceptions import DuplicateRegistrationError, ParseError -from .utils import PREFIX_21_REGEX, _get_dict +from .utils import PREFIX_21_REGEX, _get_dict, get_class_hierarchy_names STIX2_OBJ_MAPS = {} @@ -192,7 +192,7 @@ def parse_observable(data, _valid_refs=None, allow_custom=False, version=None): return obj_class(allow_custom=allow_custom, **obj) -def _register_object(new_type, version=None): +def _register_object(new_type, version=stix2.DEFAULT_VERSION): """Register a custom STIX Object type. Args: @@ -213,6 +213,13 @@ def _register_object(new_type, version=None): new_type.__name__, ) + properties = new_type._properties + + if version == "2.1": + for prop_name, prop in properties.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name '%s' must begin with an alpha character" % prop_name) + if version: v = 'v' + version.replace('.', '') else: @@ -225,7 +232,7 @@ def _register_object(new_type, version=None): OBJ_MAP[new_type._type] = new_type -def _register_marking(new_marking, version=None): +def _register_marking(new_marking, version=stix2.DEFAULT_VERSION): """Register a custom STIX Marking Definition type. Args: @@ -257,7 +264,7 @@ def _register_marking(new_marking, version=None): OBJ_MAP_MARKING[mark_type] = new_marking -def _register_observable(new_observable, version=None): +def _register_observable(new_observable, version=stix2.DEFAULT_VERSION): """Register a custom STIX Cyber Observable type. Args: @@ -266,6 +273,38 @@ def _register_observable(new_observable, version=None): None, use latest version. """ + properties = new_observable._properties + + if version == "2.0": + # If using STIX2.0, check properties ending in "_ref/s" are ObjectReferenceProperties + for prop_name, prop in properties.items(): + if prop_name.endswith('_ref') and ('ObjectReferenceProperty' not in get_class_hierarchy_names(prop)): + raise ValueError( + "'%s' is named like an object reference property but " + "is not an ObjectReferenceProperty." % prop_name, + ) + elif (prop_name.endswith('_refs') and ('ListProperty' not in get_class_hierarchy_names(prop) or + 'ObjectReferenceProperty' not in get_class_hierarchy_names(prop.contained))): + raise ValueError( + "'%s' is named like an object reference list property but " + "is not a ListProperty containing ObjectReferenceProperty." % prop_name, + ) + else: + # If using STIX2.1 (or newer...), check properties ending in "_ref/s" are ReferenceProperties + for prop_name, prop in properties.items(): + if not re.match(PREFIX_21_REGEX, prop_name): + raise ValueError("Property name '%s' must begin with an alpha character." % prop_name) + elif prop_name.endswith('_ref') and ('ReferenceProperty' not in get_class_hierarchy_names(prop)): + raise ValueError( + "'%s' is named like a reference property but " + "is not a ReferenceProperty." % prop_name, + ) + elif (prop_name.endswith('_refs') and ('ListProperty' not in get_class_hierarchy_names(prop) or + 'ReferenceProperty' not in get_class_hierarchy_names(prop.contained))): + raise ValueError( + "'%s' is named like a reference list property but " + "is not a ListProperty containing ReferenceProperty." % prop_name, + ) if version: v = 'v' + version.replace('.', '') diff --git a/stix2/test/v20/test_markings.py b/stix2/test/v20/test_markings.py index b34ef43..0753e86 100644 --- a/stix2/test/v20/test_markings.py +++ b/stix2/test/v20/test_markings.py @@ -249,14 +249,12 @@ def test_not_registered_marking_raises_exception(): def test_marking_wrong_type_construction(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): # Test passing wrong type for properties. @stix2.v20.CustomMarking('x-new-marking-type2', ("a", "b")) class NewObject3(object): pass - assert str(excinfo.value) == "Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]" - def test_campaign_add_markings(): campaign = stix2.v20.Campaign( diff --git a/stix2/test/v21/test_markings.py b/stix2/test/v21/test_markings.py index a2fca51..b0a0d09 100644 --- a/stix2/test/v21/test_markings.py +++ b/stix2/test/v21/test_markings.py @@ -277,14 +277,12 @@ def test_not_registered_marking_raises_exception(): def test_marking_wrong_type_construction(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError): # Test passing wrong type for properties. @stix2.v21.CustomMarking('x-new-marking-type2', ("a", "b")) class NewObject3(object): pass - assert str(excinfo.value) == "Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]" - def test_campaign_add_markings(): campaign = stix2.v21.Campaign( From e730d45d44a4c71e5ab494216ddeede1ff27ae63 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 3 Apr 2020 10:45:36 -0400 Subject: [PATCH 33/37] Use DEFAULT_VERSION in create_pattern_object() --- stix2/pattern_visitor.py | 17 +++++++--- stix2/test/v20/test_pattern_expressions.py | 5 +-- stix2/test/v21/test_pattern_expressions.py | 38 ++++++++++++---------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/stix2/pattern_visitor.py b/stix2/pattern_visitor.py index a891e12..c0a0fdb 100644 --- a/stix2/pattern_visitor.py +++ b/stix2/pattern_visitor.py @@ -14,6 +14,8 @@ from stix2patterns.v21.grammars.STIXPatternVisitor import \ STIXPatternVisitor as STIXPatternVisitor21 from stix2patterns.v21.pattern import Pattern as Pattern21 +import stix2 + from .patterns import * from .patterns import _BooleanExpression @@ -336,7 +338,7 @@ class STIXPatternVisitorForSTIX21(STIXPatternVisitorForSTIX2, STIXPatternVisitor super(STIXPatternVisitor21, self).__init__() -class STIXPatternVisitorForSTIX20(STIXPatternVisitor20, STIXPatternVisitorForSTIX2): +class STIXPatternVisitorForSTIX20(STIXPatternVisitorForSTIX2, STIXPatternVisitor20): classes = {} def __init__(self, module_suffix, module_name): @@ -352,11 +354,18 @@ class STIXPatternVisitorForSTIX20(STIXPatternVisitor20, STIXPatternVisitorForSTI super(STIXPatternVisitor20, self).__init__() -def create_pattern_object(pattern, module_suffix="", module_name="", version="2.1"): +def create_pattern_object(pattern, module_suffix="", module_name="", version=stix2.DEFAULT_VERSION): """ Create a STIX pattern AST from a pattern string. """ - pattern_obj = Pattern21(pattern) if version == "2.1" else Pattern20(pattern) - builder = STIXPatternVisitorForSTIX21(module_suffix, module_name) if version == "2.1" else STIXPatternVisitorForSTIX20(module_suffix, module_name) + if version == "2.1": + pattern_class = Pattern21 + visitor_class = STIXPatternVisitorForSTIX21 + else: + pattern_class = Pattern20 + visitor_class = STIXPatternVisitorForSTIX20 + + pattern_obj = pattern_class(pattern) + builder = visitor_class(module_suffix, module_name) return pattern_obj.visit(builder) diff --git a/stix2/test/v20/test_pattern_expressions.py b/stix2/test/v20/test_pattern_expressions.py index 23a401b..d5cbb5b 100644 --- a/stix2/test/v20/test_pattern_expressions.py +++ b/stix2/test/v20/test_pattern_expressions.py @@ -494,13 +494,14 @@ def test_make_constant_already_a_constant(): def test_parsing_comparison_expression(): - patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") + patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']", version="2.0") assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" def test_parsing_qualified_expression(): patt_obj = create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS", + version="2.0", ) assert str( patt_obj, @@ -508,5 +509,5 @@ def test_parsing_qualified_expression(): def test_list_constant(): - patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]") + patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]", version="2.0") assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" diff --git a/stix2/test/v21/test_pattern_expressions.py b/stix2/test/v21/test_pattern_expressions.py index 627ae5b..198edac 100644 --- a/stix2/test/v21/test_pattern_expressions.py +++ b/stix2/test/v21/test_pattern_expressions.py @@ -176,7 +176,7 @@ def test_greater_than(): def test_parsing_greater_than(): - patt_obj = create_pattern_object("[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.478901]") + patt_obj = create_pattern_object("[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.478901]", version="2.1") assert str(patt_obj) == "[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.478901]" @@ -186,7 +186,7 @@ def test_less_than(): def test_parsing_less_than(): - patt_obj = create_pattern_object("[file:size < 1024]") + patt_obj = create_pattern_object("[file:size < 1024]", version="2.1") assert str(patt_obj) == "[file:size < 1024]" @@ -199,7 +199,7 @@ def test_greater_than_or_equal(): def test_parsing_greater_than_or_equal(): - patt_obj = create_pattern_object("[file:size >= 1024]") + patt_obj = create_pattern_object("[file:size >= 1024]", version="2.1") assert str(patt_obj) == "[file:size >= 1024]" @@ -212,32 +212,32 @@ def test_less_than_or_equal(): def test_parsing_less_than_or_equal(): - patt_obj = create_pattern_object("[file:size <= 1024]") + patt_obj = create_pattern_object("[file:size <= 1024]", version="2.1") assert str(patt_obj) == "[file:size <= 1024]" def test_parsing_issubset(): - patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']") + patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']", version="2.1") assert str(patt_obj) == "[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']" def test_parsing_issuperset(): - patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUPERSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']") + patt_obj = create_pattern_object("[network-traffic:dst_ref.value ISSUPERSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']", version="2.1") assert str(patt_obj) == "[network-traffic:dst_ref.value ISSUPERSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']" def test_parsing_like(): - patt_obj = create_pattern_object("[directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo']") + patt_obj = create_pattern_object("[directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo']", version="2.1") assert str(patt_obj) == "[directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo']" def test_parsing_match(): - patt_obj = create_pattern_object("[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS") # noqa + patt_obj = create_pattern_object("[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS", version="2.1") # noqa assert str(patt_obj) == "[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS" # noqa def test_parsing_followed_by(): - patt_obj = create_pattern_object("([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS") # noqa + patt_obj = create_pattern_object("([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS", version="2.1") # noqa assert str(patt_obj) == "([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\\\foo\\\\bar']) WITHIN 300 SECONDS" # noqa @@ -302,7 +302,7 @@ def test_and_observable_expression(): def test_parsing_and_observable_expression(): - exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") # noqa + exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']", version="2.1") # noqa assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" # noqa @@ -358,7 +358,7 @@ def test_or_observable_expression(): def test_parsing_or_observable_expression(): - exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']") # noqa + exp = create_pattern_object("[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']", version="2.1") # noqa assert str(exp) == "[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] OR [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul']" # noqa @@ -392,7 +392,7 @@ def test_hex(): def test_parsing_hex(): - patt_obj = create_pattern_object("[file:magic_number_hex = h'ffd8']") + patt_obj = create_pattern_object("[file:magic_number_hex = h'ffd8']", version="2.1") assert str(patt_obj) == "[file:magic_number_hex = h'ffd8']" @@ -445,7 +445,7 @@ def test_binary(): def test_parsing_binary(): - patt_obj = create_pattern_object("[artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q=']") + patt_obj = create_pattern_object("[artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q=']", version="2.1") assert str(patt_obj) == "[artifact:payload_bin = b'dGhpcyBpcyBhIHRlc3Q=']" @@ -610,13 +610,14 @@ def test_make_constant_already_a_constant(): def test_parsing_comparison_expression(): - patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']") + patt_obj = create_pattern_object("[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']", version="2.1") assert str(patt_obj) == "[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']" def test_parsing_repeat_and_within_qualified_expression(): patt_obj = create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS", + version="2.1", ) assert str( patt_obj, @@ -626,6 +627,7 @@ def test_parsing_repeat_and_within_qualified_expression(): def test_parsing_start_stop_qualified_expression(): patt_obj = create_pattern_object( "[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] START t'2016-06-01T00:00:00Z' STOP t'2017-03-12T08:30:00Z'", # noqa + version="2.1", ) assert str( patt_obj, @@ -633,20 +635,20 @@ def test_parsing_start_stop_qualified_expression(): def test_list_constant(): - patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]") + patt_obj = create_pattern_object("[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]", version="2.1") assert str(patt_obj) == "[network-traffic:src_ref.value IN ('10.0.0.0', '10.0.0.1', '10.0.0.2')]" def test_parsing_boolean(): - patt_obj = create_pattern_object("[network-traffic:is_active = true]") + patt_obj = create_pattern_object("[network-traffic:is_active = true]", version="2.1") assert str(patt_obj) == "[network-traffic:is_active = true]" def test_parsing_multiple_slashes_quotes(): - patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]") + patt_obj = create_pattern_object("[ file:name = 'weird_name\\'' ]", version="2.1") assert str(patt_obj) == "[file:name = 'weird_name\\'']" def test_parse_error(): with pytest.raises(ParseException): - create_pattern_object("[ file: name = 'weirdname]") + create_pattern_object("[ file: name = 'weirdname]", version="2.1") From c7fb79d195404ed287934b38e6a9e52e359d08b5 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 3 Apr 2020 15:58:56 -0400 Subject: [PATCH 34/37] Fix some TAXII DataStore tests --- stix2/test/v20/test_datastore_taxii.py | 9 ++++++--- stix2/test/v21/test_datastore_taxii.py | 10 +++++++--- tox.ini | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/stix2/test/v20/test_datastore_taxii.py b/stix2/test/v20/test_datastore_taxii.py index 5a3c0cb..bf73ab4 100644 --- a/stix2/test/v20/test_datastore_taxii.py +++ b/stix2/test/v20/test_datastore_taxii.py @@ -4,7 +4,8 @@ from medallion.filters.basic_filter import BasicFilter import pytest from requests.models import Response import six -from taxii2client import Collection, _filter_kwargs_to_query_params +from taxii2client import Collection +from taxii2client.common import _filter_kwargs_to_query_params import stix2 from stix2.datastore import DataSourceError @@ -38,7 +39,8 @@ class MockTAXIICollectionEndpoint(Collection): self.objects, ("id", "type", "version"), [], - ) + None, + )[0] if objs: return stix2.v20.Bundle(objects=objs) else: @@ -59,7 +61,8 @@ class MockTAXIICollectionEndpoint(Collection): objects, ("version",), [], - ) + None, + )[0] else: filtered_objects = [] if filtered_objects: diff --git a/stix2/test/v21/test_datastore_taxii.py b/stix2/test/v21/test_datastore_taxii.py index 528546a..35dc0ba 100644 --- a/stix2/test/v21/test_datastore_taxii.py +++ b/stix2/test/v21/test_datastore_taxii.py @@ -4,7 +4,8 @@ from medallion.filters.basic_filter import BasicFilter import pytest from requests.models import Response import six -from taxii2client import Collection, _filter_kwargs_to_query_params +from taxii2client import Collection +from taxii2client.common import _filter_kwargs_to_query_params import stix2 from stix2.datastore import DataSourceError @@ -38,8 +39,10 @@ class MockTAXIICollectionEndpoint(Collection): self.objects, ("id", "type", "version"), [], - ) + None, + )[0] if objs: + print(objs) return stix2.v21.Bundle(objects=objs) else: resp = Response() @@ -59,7 +62,8 @@ class MockTAXIICollectionEndpoint(Collection): objects, ("version",), [], - ) + None, + )[0] else: filtered_objects = [] if filtered_objects: diff --git a/tox.ini b/tox.ini index aa0472a..d34aac1 100644 --- a/tox.ini +++ b/tox.ini @@ -8,11 +8,11 @@ deps = pytest pytest-cov coverage - taxii2-client<1.0.0 + taxii2-client fuzzywuzzy haversine python-Levenshtein - medallion<2.0.0 + medallion commands = python -m pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning From 2b0d63c4b1d83ab9907839a1dca4f7ae02805fb8 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Apr 2020 17:19:36 -0400 Subject: [PATCH 35/37] update test_datastore_taxii.py conftest.py for latest changes in medallion. add extra data used by filter --- stix2/datastore/taxii.py | 2 +- stix2/test/v20/conftest.py | 36 ++++++++++++++++++++++++++ stix2/test/v20/test_datastore_taxii.py | 26 ++++++++++++++----- stix2/test/v21/conftest.py | 36 ++++++++++++++++++++++++++ stix2/test/v21/test_datastore_taxii.py | 27 +++++++++++++------ 5 files changed, 111 insertions(+), 16 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 41c968f..08d0e14 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -12,7 +12,7 @@ from stix2.datastore.filters import Filter, FilterSet, apply_common_filters from stix2.utils import deduplicate try: - from taxii2client import ValidationError + from taxii2client.exceptions import ValidationError _taxii2_client = True except ImportError: _taxii2_client = False diff --git a/stix2/test/v20/conftest.py b/stix2/test/v20/conftest.py index 48e4532..6bb8fae 100644 --- a/stix2/test/v20/conftest.py +++ b/stix2/test/v20/conftest.py @@ -114,6 +114,42 @@ def stix_objs1(): return [ind1, ind2, ind3, ind4, ind5] +@pytest.fixture +def stix_objs1_manifests(): + # Tests against latest medallion (TAXII 2.1) + ind1 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind2 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind3 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.936Z", + } + ind4 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000002", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind5 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000002", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + return [ind1, ind2, ind3, ind4, ind5] + + @pytest.fixture def stix_objs2(): ind6 = { diff --git a/stix2/test/v20/test_datastore_taxii.py b/stix2/test/v20/test_datastore_taxii.py index bf73ab4..b0c6b77 100644 --- a/stix2/test/v20/test_datastore_taxii.py +++ b/stix2/test/v20/test_datastore_taxii.py @@ -4,12 +4,13 @@ from medallion.filters.basic_filter import BasicFilter import pytest from requests.models import Response import six -from taxii2client import Collection from taxii2client.common import _filter_kwargs_to_query_params +from taxii2client.v20 import Collection import stix2 from stix2.datastore import DataSourceError from stix2.datastore.filters import Filter +from stix2.utils import get_timestamp COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -22,6 +23,7 @@ class MockTAXIICollectionEndpoint(Collection): url, collection_info=collection_info, ) self.objects = [] + self.manifests = [] def add_objects(self, bundle): self._verify_can_write() @@ -29,6 +31,14 @@ class MockTAXIICollectionEndpoint(Collection): bundle = json.loads(bundle, encoding='utf-8') for object in bundle.get("objects", []): self.objects.append(object) + self.manifests.append( + { + "date_added": get_timestamp(), + "id": object["id"], + "media_type": "application/stix+json;version=2.1", + "version": object.get("modified", object.get("created", get_timestamp())), + }, + ) def get_objects(self, **filter_kwargs): self._verify_can_read() @@ -38,8 +48,8 @@ class MockTAXIICollectionEndpoint(Collection): objs = full_filter.process_filter( self.objects, ("id", "type", "version"), - [], - None, + self.manifests, + 100, )[0] if objs: return stix2.v20.Bundle(objects=objs) @@ -60,8 +70,8 @@ class MockTAXIICollectionEndpoint(Collection): filtered_objects = full_filter.process_filter( objects, ("version",), - [], - None, + self.manifests, + 100, )[0] else: filtered_objects = [] @@ -74,7 +84,7 @@ class MockTAXIICollectionEndpoint(Collection): @pytest.fixture -def collection(stix_objs1): +def collection(stix_objs1, stix_objs1_manifests): mock = MockTAXIICollectionEndpoint( COLLECTION_URL, { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", @@ -89,11 +99,12 @@ def collection(stix_objs1): ) mock.objects.extend(stix_objs1) + mock.manifests.extend(stix_objs1_manifests) return mock @pytest.fixture -def collection_no_rw_access(stix_objs1): +def collection_no_rw_access(stix_objs1, stix_objs1_manifests): mock = MockTAXIICollectionEndpoint( COLLECTION_URL, { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", @@ -108,6 +119,7 @@ def collection_no_rw_access(stix_objs1): ) mock.objects.extend(stix_objs1) + mock.manifests.extend(stix_objs1_manifests) return mock diff --git a/stix2/test/v21/conftest.py b/stix2/test/v21/conftest.py index 103db34..d602f42 100644 --- a/stix2/test/v21/conftest.py +++ b/stix2/test/v21/conftest.py @@ -135,6 +135,42 @@ def stix_objs1(): return [ind1, ind2, ind3, ind4, ind5] +@pytest.fixture +def stix_objs1_manifests(): + # Tests against latest medallion (TAXII 2.1) + ind1 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind2 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind3 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000001", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.936Z", + } + ind4 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000002", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + ind5 = { + "date_added": "2017-01-27T13:49:53.935Z", + "id": "indicator--00000000-0000-4000-8000-000000000002", + "media_type": "application/stix+json;version=2.1", + "version": "2017-01-27T13:49:53.935Z", + } + return [ind1, ind2, ind3, ind4, ind5] + + @pytest.fixture def stix_objs2(): ind6 = { diff --git a/stix2/test/v21/test_datastore_taxii.py b/stix2/test/v21/test_datastore_taxii.py index 35dc0ba..28a7368 100644 --- a/stix2/test/v21/test_datastore_taxii.py +++ b/stix2/test/v21/test_datastore_taxii.py @@ -4,12 +4,13 @@ from medallion.filters.basic_filter import BasicFilter import pytest from requests.models import Response import six -from taxii2client import Collection from taxii2client.common import _filter_kwargs_to_query_params +from taxii2client.v21 import Collection import stix2 from stix2.datastore import DataSourceError from stix2.datastore.filters import Filter +from stix2.utils import get_timestamp COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -22,6 +23,7 @@ class MockTAXIICollectionEndpoint(Collection): url, collection_info=collection_info, ) self.objects = [] + self.manifests = [] def add_objects(self, bundle): self._verify_can_write() @@ -29,6 +31,14 @@ class MockTAXIICollectionEndpoint(Collection): bundle = json.loads(bundle, encoding='utf-8') for object in bundle.get("objects", []): self.objects.append(object) + self.manifests.append( + { + "date_added": get_timestamp(), + "id": object["id"], + "media_type": "application/stix+json;version=2.1", + "version": object.get("modified", object.get("created", get_timestamp())), + }, + ) def get_objects(self, **filter_kwargs): self._verify_can_read() @@ -38,11 +48,10 @@ class MockTAXIICollectionEndpoint(Collection): objs = full_filter.process_filter( self.objects, ("id", "type", "version"), - [], - None, + self.manifests, + 100, )[0] if objs: - print(objs) return stix2.v21.Bundle(objects=objs) else: resp = Response() @@ -61,8 +70,8 @@ class MockTAXIICollectionEndpoint(Collection): filtered_objects = full_filter.process_filter( objects, ("version",), - [], - None, + self.manifests, + 100, )[0] else: filtered_objects = [] @@ -75,7 +84,7 @@ class MockTAXIICollectionEndpoint(Collection): @pytest.fixture -def collection(stix_objs1): +def collection(stix_objs1, stix_objs1_manifests): mock = MockTAXIICollectionEndpoint( COLLECTION_URL, { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", @@ -90,11 +99,12 @@ def collection(stix_objs1): ) mock.objects.extend(stix_objs1) + mock.manifests.extend(stix_objs1_manifests) return mock @pytest.fixture -def collection_no_rw_access(stix_objs1): +def collection_no_rw_access(stix_objs1, stix_objs1_manifests): mock = MockTAXIICollectionEndpoint( COLLECTION_URL, { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", @@ -109,6 +119,7 @@ def collection_no_rw_access(stix_objs1): ) mock.objects.extend(stix_objs1) + mock.manifests.extend(stix_objs1_manifests) return mock From c68dd055c16dd59ff437298b9dfea597ddf3d34f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 3 Apr 2020 17:44:20 -0400 Subject: [PATCH 36/37] Update CHANGELOG for v1.4.0 --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index dc0d91e..fd22828 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,12 @@ CHANGELOG ========= +1.4.0 - 2020-04-03 + +* #347, #355, #356, #357, #358, #360, #362, #369, #370, #379, #374, #384 Updates STIX 2.1 support to CS01 +* #376 Fixes bug where registering object of same name would overwrite it; will + now raise an error + 1.3.1 - 2020-03-06 * #322 Adds encoding option FileSystemSource and MemorySource From 31cb2f85be6c17f81aedf744d49b9f27aa331285 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 3 Apr 2020 17:44:52 -0400 Subject: [PATCH 37/37] =?UTF-8?q?Bump=20version:=201.3.1=20=E2=86=92=201.4?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- stix2/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7e89c66..4fefdcc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.1 +current_version = 1.4.0 commit = True tag = True diff --git a/stix2/version.py b/stix2/version.py index 9c73af2..3e8d9f9 100644 --- a/stix2/version.py +++ b/stix2/version.py @@ -1 +1 @@ -__version__ = "1.3.1" +__version__ = "1.4.0"