Merge branch 'master' of https://github.com/oasis-open/cti-python-stix2 into 139-dict-filter-value

stix2/base.py

@@ -6,9 +6,9 @@ import datetime as dt
 
 import simplejson as json
 
-from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
+from .exceptions import (AtLeastOnePropertyError, CustomContentError,
-                         ExtraPropertiesError, ImmutableError,
+                         DependentPropertiesError, ExtraPropertiesError,
-                         InvalidObjRefError, InvalidValueError,
+                         ImmutableError, InvalidObjRefError, InvalidValueError,
                          MissingPropertiesError,
                          MutuallyExclusivePropertiesError)
 from .markings.utils import validate
@@ -61,6 +61,8 @@ class _STIXBase(collections.Mapping):
             try:
                 kwargs[prop_name] = prop.clean(kwargs[prop_name])
             except ValueError as exc:
+                if self.__allow_custom and isinstance(exc, CustomContentError):
+                    return
                 raise InvalidValueError(self.__class__, prop_name, reason=str(exc))
 
     # interproperty constraint methods
@@ -97,6 +99,7 @@ class _STIXBase(collections.Mapping):
 
     def __init__(self, allow_custom=False, **kwargs):
         cls = self.__class__
+        self.__allow_custom = allow_custom
 
         # Use the same timestamp for any auto-generated datetimes
         self.__now = get_timestamp()

stix2/exceptions.py

@@ -163,6 +163,13 @@ class ParseError(STIXError, ValueError):
         super(ParseError, self).__init__(msg)
 
 
+class CustomContentError(STIXError, ValueError):
+    """Custom STIX Content (SDO, Observable, Extension, etc.) detected."""
+
+    def __init__(self, msg):
+        super(CustomContentError, self).__init__(msg)
+
+
 class InvalidSelectorError(STIXError, AssertionError):
     """Granular Marking selector violation. The selector must resolve into an existing STIX object property."""
 

stix2/test/test_custom.py

@@ -364,6 +364,7 @@ def test_parse_custom_observable_object():
     }"""
 
     nt = stix2.parse_observable(nt_string, [])
+    assert isinstance(nt, stix2.core._STIXBase)
     assert nt.property1 == 'something'
 
 
@@ -373,10 +374,46 @@ def test_parse_unregistered_custom_observable_object():
         "property1": "something"
     }"""
 
-    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+    with pytest.raises(stix2.exceptions.CustomContentError) as excinfo:
         stix2.parse_observable(nt_string)
     assert "Can't parse unknown observable type" in str(excinfo.value)
 
+    parsed_custom = stix2.parse_observable(nt_string, allow_custom=True)
+    assert parsed_custom['property1'] == 'something'
+    with pytest.raises(AttributeError) as excinfo:
+        assert parsed_custom.property1 == 'something'
+    assert not isinstance(parsed_custom, stix2.core._STIXBase)
+
+
+def test_parse_unregistered_custom_observable_object_with_no_type():
+    nt_string = """{
+        "property1": "something"
+    }"""
+
+    with pytest.raises(stix2.exceptions.ParseError) as excinfo:
+        stix2.parse_observable(nt_string, allow_custom=True)
+    assert "Can't parse observable with no 'type' property" in str(excinfo.value)
+
+
+def test_parse_observed_data_with_custom_observable():
+    input_str = """{
+        "type": "observed-data",
+        "id": "observed-data--dc20c4ca-a2a3-4090-a5d5-9558c3af4758",
+        "created": "2016-04-06T19:58:16.000Z",
+        "modified": "2016-04-06T19:58:16.000Z",
+        "first_observed": "2015-12-21T19:00:00Z",
+        "last_observed": "2015-12-21T19:00:00Z",
+        "number_observed": 1,
+        "objects": {
+            "0": {
+                "type": "x-foobar-observable",
+                "property1": "something"
+            }
+        }
+    }"""
+    parsed = stix2.parse(input_str, allow_custom=True)
+    assert parsed.objects['0']['property1'] == 'something'
+
 
 def test_parse_invalid_custom_observable_object():
     nt_string = """{
@@ -592,7 +629,11 @@ def test_parse_observable_with_unregistered_custom_extension():
 
     with pytest.raises(ValueError) as excinfo:
         stix2.parse_observable(input_str)
-    assert "Can't parse Unknown extension type" in str(excinfo.value)
+    assert "Can't parse unknown extension type" in str(excinfo.value)
+
+    parsed_ob = stix2.parse_observable(input_str, allow_custom=True)
+    assert parsed_ob['extensions']['x-foobar-ext']['property1'] == 'foo'
+    assert not isinstance(parsed_ob['extensions']['x-foobar-ext'], stix2.core._STIXBase)
 
 
 def test_register_custom_object():

stix2/test/test_external_reference.py

@@ -42,7 +42,7 @@ def test_external_reference_capec():
     )
 
     assert str(ref) == CAPEC
-    assert re.match("ExternalReference\(source_name=u?'capec', external_id=u?'CAPEC-550'\)", repr(ref))
+    assert re.match("ExternalReference\\(source_name=u?'capec', external_id=u?'CAPEC-550'\\)", repr(ref))
 
 
 CAPEC_URL = """{
@@ -109,7 +109,7 @@ def test_external_reference_offline():
     )
 
     assert str(ref) == OFFLINE
-    assert re.match("ExternalReference\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\)", repr(ref))
+    assert re.match("ExternalReference\\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\\)", repr(ref))
     # Yikes! This works
     assert eval("stix2." + repr(ref)) == ref
 

stix2/test/test_malware.py

@@ -126,7 +126,7 @@ def test_parse_malware(data):
 
 
 def test_parse_malware_invalid_labels():
-    data = re.compile('\[.+\]', re.DOTALL).sub('1', EXPECTED_MALWARE)
+    data = re.compile('\\[.+\\]', re.DOTALL).sub('1', EXPECTED_MALWARE)
     with pytest.raises(ValueError) as excinfo:
         stix2.parse(data)
     assert "Invalid value for Malware 'labels'" in str(excinfo.value)

stix2/test/test_pattern_expressions.py

@@ -319,13 +319,13 @@ def test_invalid_binary_constant():
 
 def test_escape_quotes_and_backslashes():
     exp = stix2.MatchesComparisonExpression("file:name",
-                                            "^Final Report.+\.exe$")
+                                            "^Final Report.+\\.exe$")
     assert str(exp) == "file:name MATCHES '^Final Report.+\\\\.exe$'"
 
 
 def test_like():
     exp = stix2.LikeComparisonExpression("directory:path",
-                                         "C:\Windows\%\\foo")
+                                         "C:\\Windows\\%\\foo")
     assert str(exp) == "directory:path LIKE 'C:\\\\Windows\\\\%\\\\foo'"
 
 

stix2/v20/observables.py

@@ -9,8 +9,8 @@ from collections import OrderedDict
 import copy
 
 from ..base import _Extension, _Observable, _STIXBase
-from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError,
+from ..exceptions import (AtLeastOnePropertyError, CustomContentError,
-                          ParseError)
+                          DependentPropertiesError, ParseError)
 from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty,
                           EmbeddedObjectProperty, EnumProperty, FloatProperty,
                           HashesProperty, HexProperty, IntegerProperty,
@@ -76,7 +76,7 @@ class ExtensionsProperty(DictionaryProperty):
                     else:
                         raise ValueError("Cannot determine extension type.")
                 else:
-                    raise ValueError("The key used in the extensions dictionary is not an extension type name")
+                    raise CustomContentError("Can't parse unknown extension type: {}".format(key))
         else:
             raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type)
         return dictified
@@ -937,14 +937,22 @@ def parse_observable(data, _valid_refs=None, allow_custom=False):
     try:
         obj_class = OBJ_MAP_OBSERVABLE[obj['type']]
     except KeyError:
-        raise ParseError("Can't parse unknown observable type '%s'! For custom observables, "
+        if allow_custom:
+            # flag allows for unknown custom objects too, but will not
+            # be parsed into STIX observable object, just returned as is
+            return obj
+        raise CustomContentError("Can't parse unknown observable type '%s'! For custom observables, "
                                  "use the CustomObservable decorator." % obj['type'])
 
     if 'extensions' in obj and obj['type'] in EXT_MAP:
         for name, ext in obj['extensions'].items():
-            if name not in EXT_MAP[obj['type']]:
+            try:
-                raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type']))
                 ext_class = EXT_MAP[obj['type']][name]
+            except KeyError:
+                if not allow_custom:
+                    raise CustomContentError("Can't parse unknown extension type '%s'"
+                                             "for observable type '%s'!" % (name, obj['type']))
+            else:  # extension was found
                 obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name])
 
     return obj_class(allow_custom=allow_custom, **obj)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,py35,py36,pycodestyle,isort-check
+envlist = py27,py34,py35,py36,style,isort-check
 
 [testenv]
 deps =
@@ -10,22 +10,17 @@ deps =
     coverage
     taxii2-client
 commands =
-    py.test --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
+    pytest --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing
-    py.test stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
+    pytest stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append
 
 passenv = CI TRAVIS TRAVIS_*
 
-[testenv:pycodestyle]
+[testenv:style]
 deps =
   flake8
-  pycodestyle
 commands =
-  pycodestyle ./stix2
   flake8
 
-[pycodestyle]
-max-line-length=160
-
 [flake8]
 max-line-length=160
 
@@ -37,7 +32,7 @@ commands =
 
 [travis]
 python =
-  2.7: py27, pycodestyle
+  2.7: py27, style
-  3.4: py34, pycodestyle
+  3.4: py34, style
-  3.5: py35, pycodestyle
+  3.5: py35, style
-  3.6: py36, pycodestyle
+  3.6: py36, style