diff --git a/stix2/base.py b/stix2/base.py index 898f489..3219007 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -6,9 +6,9 @@ import datetime as dt import simplejson as json -from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, - ExtraPropertiesError, ImmutableError, - InvalidObjRefError, InvalidValueError, +from .exceptions import (AtLeastOnePropertyError, CustomContentError, + DependentPropertiesError, ExtraPropertiesError, + ImmutableError, InvalidObjRefError, InvalidValueError, MissingPropertiesError, MutuallyExclusivePropertiesError) from .markings.utils import validate @@ -61,6 +61,8 @@ class _STIXBase(collections.Mapping): try: kwargs[prop_name] = prop.clean(kwargs[prop_name]) except ValueError as exc: + if self.__allow_custom and isinstance(exc, CustomContentError): + return raise InvalidValueError(self.__class__, prop_name, reason=str(exc)) # interproperty constraint methods @@ -97,6 +99,7 @@ class _STIXBase(collections.Mapping): def __init__(self, allow_custom=False, **kwargs): cls = self.__class__ + self.__allow_custom = allow_custom # Use the same timestamp for any auto-generated datetimes self.__now = get_timestamp() diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 841a8e9..79c5a81 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -163,6 +163,13 @@ class ParseError(STIXError, ValueError): super(ParseError, self).__init__(msg) +class CustomContentError(STIXError, ValueError): + """Custom STIX Content (SDO, Observable, Extension, etc.) detected.""" + + def __init__(self, msg): + super(CustomContentError, self).__init__(msg) + + class InvalidSelectorError(STIXError, AssertionError): """Granular Marking selector violation. The selector must resolve into an existing STIX object property.""" diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 417be00..5829e29 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -364,6 +364,7 @@ def test_parse_custom_observable_object(): }""" nt = stix2.parse_observable(nt_string, []) + assert isinstance(nt, stix2.core._STIXBase) assert nt.property1 == 'something' @@ -373,10 +374,46 @@ def test_parse_unregistered_custom_observable_object(): "property1": "something" }""" - with pytest.raises(stix2.exceptions.ParseError) as excinfo: + with pytest.raises(stix2.exceptions.CustomContentError) as excinfo: stix2.parse_observable(nt_string) assert "Can't parse unknown observable type" in str(excinfo.value) + parsed_custom = stix2.parse_observable(nt_string, allow_custom=True) + assert parsed_custom['property1'] == 'something' + with pytest.raises(AttributeError) as excinfo: + assert parsed_custom.property1 == 'something' + assert not isinstance(parsed_custom, stix2.core._STIXBase) + + +def test_parse_unregistered_custom_observable_object_with_no_type(): + nt_string = """{ + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse_observable(nt_string, allow_custom=True) + assert "Can't parse observable with no 'type' property" in str(excinfo.value) + + +def test_parse_observed_data_with_custom_observable(): + input_str = """{ + "type": "observed-data", + "id": "observed-data--dc20c4ca-a2a3-4090-a5d5-9558c3af4758", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "x-foobar-observable", + "property1": "something" + } + } + }""" + parsed = stix2.parse(input_str, allow_custom=True) + assert parsed.objects['0']['property1'] == 'something' + def test_parse_invalid_custom_observable_object(): nt_string = """{ @@ -592,7 +629,11 @@ def test_parse_observable_with_unregistered_custom_extension(): with pytest.raises(ValueError) as excinfo: stix2.parse_observable(input_str) - assert "Can't parse Unknown extension type" in str(excinfo.value) + assert "Can't parse unknown extension type" in str(excinfo.value) + + parsed_ob = stix2.parse_observable(input_str, allow_custom=True) + assert parsed_ob['extensions']['x-foobar-ext']['property1'] == 'foo' + assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase) def test_register_custom_object(): diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py index 2b79f01..9b90998 100644 --- a/stix2/test/test_external_reference.py +++ b/stix2/test/test_external_reference.py @@ -42,7 +42,7 @@ def test_external_reference_capec(): ) assert str(ref) == CAPEC - assert re.match("ExternalReference\(source_name=u?'capec', external_id=u?'CAPEC-550'\)", repr(ref)) + assert re.match("ExternalReference\\(source_name=u?'capec', external_id=u?'CAPEC-550'\\)", repr(ref)) CAPEC_URL = """{ @@ -109,7 +109,7 @@ def test_external_reference_offline(): ) assert str(ref) == OFFLINE - assert re.match("ExternalReference\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\)", repr(ref)) + assert re.match("ExternalReference\\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\\)", repr(ref)) # Yikes! This works assert eval("stix2." + repr(ref)) == ref diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py index 8c565cd..2228885 100644 --- a/stix2/test/test_malware.py +++ b/stix2/test/test_malware.py @@ -126,7 +126,7 @@ def test_parse_malware(data): def test_parse_malware_invalid_labels(): - data = re.compile('\[.+\]', re.DOTALL).sub('1', EXPECTED_MALWARE) + data = re.compile('\\[.+\\]', re.DOTALL).sub('1', EXPECTED_MALWARE) with pytest.raises(ValueError) as excinfo: stix2.parse(data) assert "Invalid value for Malware 'labels'" in str(excinfo.value) diff --git a/stix2/test/test_pattern_expressions.py b/stix2/test/test_pattern_expressions.py index 363458a..74a7d0f 100644 --- a/stix2/test/test_pattern_expressions.py +++ b/stix2/test/test_pattern_expressions.py @@ -319,13 +319,13 @@ def test_invalid_binary_constant(): def test_escape_quotes_and_backslashes(): exp = stix2.MatchesComparisonExpression("file:name", - "^Final Report.+\.exe$") + "^Final Report.+\\.exe$") assert str(exp) == "file:name MATCHES '^Final Report.+\\\\.exe$'" def test_like(): exp = stix2.LikeComparisonExpression("directory:path", - "C:\Windows\%\\foo") + "C:\\Windows\\%\\foo") assert str(exp) == "directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo'" diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py index 75887ee..254498b 100644 --- a/stix2/v20/observables.py +++ b/stix2/v20/observables.py @@ -9,8 +9,8 @@ from collections import OrderedDict import copy from ..base import _Extension, _Observable, _STIXBase -from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError, - ParseError) +from ..exceptions import (AtLeastOnePropertyError, CustomContentError, + DependentPropertiesError, ParseError) from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, HashesProperty, HexProperty, IntegerProperty, @@ -76,7 +76,7 @@ class ExtensionsProperty(DictionaryProperty): else: raise ValueError("Cannot determine extension type.") else: - raise ValueError("The key used in the extensions dictionary is not an extension type name") + raise CustomContentError("Can't parse unknown extension type: {}".format(key)) else: raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type) return dictified @@ -937,15 +937,23 @@ def parse_observable(data, _valid_refs=None, allow_custom=False): try: obj_class = OBJ_MAP_OBSERVABLE[obj['type']] except KeyError: - raise ParseError("Can't parse unknown observable type '%s'! For custom observables, " - "use the CustomObservable decorator." % obj['type']) + if allow_custom: + # flag allows for unknown custom objects too, but will not + # be parsed into STIX observable object, just returned as is + return obj + raise CustomContentError("Can't parse unknown observable type '%s'! For custom observables, " + "use the CustomObservable decorator." % obj['type']) if 'extensions' in obj and obj['type'] in EXT_MAP: for name, ext in obj['extensions'].items(): - if name not in EXT_MAP[obj['type']]: - raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type'])) - ext_class = EXT_MAP[obj['type']][name] - obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) + try: + ext_class = EXT_MAP[obj['type']][name] + except KeyError: + if not allow_custom: + raise CustomContentError("Can't parse unknown extension type '%s'" + "for observable type '%s'!" % (name, obj['type'])) + else: # extension was found + obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) return obj_class(allow_custom=allow_custom, **obj) diff --git a/tox.ini b/tox.ini index ac4e89f..86cd4ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,pycodestyle,isort-check +envlist = py27,py34,py35,py36,style,isort-check [testenv] deps = @@ -10,22 +10,17 @@ deps = coverage taxii2-client commands = - py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing - py.test stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append + pytest --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing + pytest stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append passenv = CI TRAVIS TRAVIS_* -[testenv:pycodestyle] +[testenv:style] deps = flake8 - pycodestyle commands = - pycodestyle ./stix2 flake8 -[pycodestyle] -max-line-length=160 - [flake8] max-line-length=160 @@ -37,7 +32,7 @@ commands = [travis] python = - 2.7: py27, pycodestyle - 3.4: py34, pycodestyle - 3.5: py35, pycodestyle - 3.6: py36, pycodestyle + 2.7: py27, style + 3.4: py34, style + 3.5: py35, style + 3.6: py36, style