From cd1851c56b7a974107e2c7852ec3bb810d1f6f4a Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 18 Aug 2017 08:59:27 -0400 Subject: [PATCH 01/14] Fix pattern in code example in README See also #43. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b2a8c9c..03c7cc3 100644 --- a/README.rst +++ b/README.rst @@ -39,8 +39,8 @@ constructor: from stix2 import Indicator indicator = Indicator(name="File hash for malware variant", - labels=['malicious-activity'], - pattern='file:hashes.md5 = "d41d8cd98f00b204e9800998ecf8427e"') + labels=["malicious-activity"], + pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") Certain required attributes of all objects will be set automatically if not provided as keyword arguments: From c64b7de761ecd9d246352fe8aa24db6fac510222 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 18 Aug 2017 12:05:12 -0400 Subject: [PATCH 02/14] Change query dict style for Filter namedtuple. --- stix2/sources/filesystem.py | 8 ++------ stix2/sources/memory.py | 8 ++------ stix2/sources/taxii.py | 14 +++----------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 0613ac0..e16ee0c 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -13,7 +13,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): @@ -78,11 +78,7 @@ class FileSystemSource(DataSource): """ """ query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } + Filter("id", "=", stix_id) ] all_data = self.query(query=query, _composite_filters=_composite_filters) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 24f3c1f..2f45d68 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -22,7 +22,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter from stix2validator import validate_string @@ -205,11 +205,7 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } + Filter("id", "=", stix_id) ] all_data = self.query(query=query, _composite_filters=_composite_filters) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 47ad8ed..4edeeed 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -12,7 +12,7 @@ TODO: Test everything import json -from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources import DataSink, DataSource, DataStore, Filter, make_id TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -89,16 +89,8 @@ class TAXIICollectionSource(DataSource): """ # make query in TAXII query format since 'id' is TAXII field query = [ - { - "field": "match[id]", - "op": "=", - "value": stix_id - }, - { - "field": "match[version]", - "op": "=", - "value": "all" - } + Filter("match[id]", "=", stix_id), + Filter("match[version]", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) From d4edb8b0bc84e8219f2c1121ea6f50feffc80272 Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 18 Aug 2017 14:22:57 -0400 Subject: [PATCH 03/14] Validate patterns when creating Indicators --- setup.py | 1 + stix2/properties.py | 16 ++++++++++++++++ stix2/sdo.py | 6 +++--- stix2/test/test_indicator.py | 20 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index efe754c..5420e04 100644 --- a/setup.py +++ b/setup.py @@ -50,5 +50,6 @@ setup( 'six', 'python-dateutil', 'requests', + 'stix2-patterns', ], ) diff --git a/stix2/properties.py b/stix2/properties.py index f63ec8b..a04294e 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -7,6 +7,8 @@ import uuid from six import string_types, text_type +from stix2patterns.validator import run_validator + from .base import _STIXBase from .exceptions import DictionaryKeyError from .utils import get_dict, parse_into_datetime @@ -370,3 +372,17 @@ class EnumProperty(StringProperty): if value not in self.allowed: raise ValueError("value '%s' is not valid for this enumeration." % value) return self.string_type(value) + + +class PatternProperty(StringProperty): + + def __init__(self, **kwargs): + super(PatternProperty, self).__init__(**kwargs) + + def clean(self, value): + str_value = super(PatternProperty, self).clean(value) + errors = run_validator(str_value) + if errors: + raise ValueError(str(errors[0])) + + return self.string_type(value) diff --git a/stix2/sdo.py b/stix2/sdo.py index 43c8328..86e9f39 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -6,8 +6,8 @@ from .base import _STIXBase from .common import COMMON_PROPERTIES, KillChainPhase from .observables import ObservableProperty from .properties import (IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) + PatternProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW @@ -77,7 +77,7 @@ class Indicator(_STIXBase): 'labels': ListProperty(StringProperty, required=True), 'name': StringProperty(), 'description': StringProperty(), - 'pattern': StringProperty(required=True), + 'pattern': PatternProperty(required=True), 'valid_from': TimestampProperty(default=lambda: NOW), 'valid_until': TimestampProperty(), 'kill_chain_phases': ListProperty(KillChainPhase), diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index 5daa0f6..3a486df 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -174,3 +174,23 @@ def test_parse_indicator(data): assert idctr.valid_from == dt.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc) assert idctr.labels[0] == "malicious-activity" assert idctr.pattern == "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']" + + +def test_invalid_indicator_pattern(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern="file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e'", + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'input is missing square brackets' in excinfo.value.reason + + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern='[file:hashes.MD5 = "d41d8cd98f00b204e9800998ecf8427e"]', + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'mismatched input' in excinfo.value.reason From 4cd99f04eaf3ccbc467b690d2511957c5651c29a Mon Sep 17 00:00:00 2001 From: Greg Back Date: Mon, 21 Aug 2017 22:16:10 +0000 Subject: [PATCH 04/14] Use version of taxii2-client from PyPI. --- .isort.cfg | 2 +- setup.py | 3 ++- stix2/test/test_data_sources.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index e141924..e8d95f8 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests,taxii2_client +known_third_party=dateutil,pytest,pytz,six,requests,taxii2client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/setup.py b/setup.py index c10da46..29d98f6 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ setup( 'pytz', 'requests', 'simplejson', - 'six' + 'six', + 'taxii2-client', ], ) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index bbb7f31..0095802 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,5 +1,5 @@ import pytest -from taxii2_client import Collection +from taxii2client import Collection from stix2.sources import DataSource, Filter, taxii From d060abbed59434dee6ba7ba18ac9c25f0a8d1e6d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 22 Aug 2017 10:47:13 -0400 Subject: [PATCH 05/14] Updated regex expressions. Thanks to @drothenberg for that contribution! --- stix2/test/test_observed_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index 45610f9..dd8f042 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -173,7 +173,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +194,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -215,7 +215,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +358,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" From 3822e39243c3fb5054ec1c179ee7eea206b3032c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 24 Aug 2017 15:10:09 -0400 Subject: [PATCH 06/14] Remove COMMON_PROPERTIES. --- stix2/common.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/stix2/common.py b/stix2/common.py index c126d76..a691b3b 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -6,10 +6,9 @@ except ImportError: from ordereddict import OrderedDict from .base import _STIXBase -from .properties import (BooleanProperty, HashesProperty, IDProperty, - ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) +from .properties import (HashesProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW, get_dict @@ -148,17 +147,3 @@ TLP_RED = MarkingDefinition( definition_type="tlp", definition=TLPMarking(tlp="red") ) - -COMMON_PROPERTIES = OrderedDict() - -COMMON_PROPERTIES.update([ - # 'type' and 'id' should be defined on each individual type - ('created_by_ref', ReferenceProperty(type="identity")), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('revoked', BooleanProperty()), - ('labels', ListProperty(StringProperty)), - ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), - ('granular_markings', ListProperty(GranularMarking)), -]) From d3f1eb86ab5e8730a18a635cfa7bddb4cbf31acd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:29:10 -0400 Subject: [PATCH 07/14] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b8d544..d9b1edc 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'ordereddict', + 'ordereddict ; python_version<"2.7"', 'python-dateutil', 'pytz', 'requests', From 23f36a9a69d103ca1627800f88fc8694ddfbab0c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:30:53 -0400 Subject: [PATCH 08/14] Minor changes to custom object decorators. Add new marking decorator. --- stix2/__init__.py | 5 +++-- stix2/common.py | 49 ++++++++++++++++++++++++++++++++++++++------ stix2/observables.py | 2 +- stix2/sdo.py | 9 +++----- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index b9b6764..76ca696 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -4,8 +4,9 @@ from . import exceptions from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) + CustomMarking, ExternalReference, GranularMarking, + KillChainPhase, MarkingDefinition, StatementMarking, + TLPMarking) from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, diff --git a/stix2/common.py b/stix2/common.py index a691b3b..6242989 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -109,17 +109,54 @@ class MarkingDefinition(_STIXBase): super(MarkingDefinition, self).__init__(**kwargs) -def register_marking(new_marking): - """Register a custom STIX Marking Definition type. - """ - OBJ_MAP_MARKING[new_marking._type] = new_marking - - OBJ_MAP_MARKING = { 'tlp': TLPMarking, 'statement': StatementMarking, } + +def _register_marking(cls): + """Register a custom STIX Marking Definition type. + """ + OBJ_MAP_MARKING[cls._type] = cls + return cls + + +def CustomMarking(type='x-custom-marking', properties=None): + """ + Custom STIX Marking decorator. + + Examples: + + @CustomMarking('x-custom-marking', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) + class MyNewMarkingObjectType(): + pass + + """ + def custom_builder(cls): + + class _Custom(cls, _STIXBase): + _type = type + _properties = OrderedDict() + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update(properties) + + def __init__(self, **kwargs): + _STIXBase.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_marking(_Custom) + return _Custom + + return custom_builder + + TLP_WHITE = MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", diff --git a/stix2/observables.py b/stix2/observables.py index 4a1319c..6c82e99 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -804,7 +804,7 @@ def CustomObservable(type='x-custom-observable', properties=None): ('type', TypeProperty(_type)), ]) - if not properties: + if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") _properties.update(properties) diff --git a/stix2/sdo.py b/stix2/sdo.py index 7eebf56..8c27d11 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -330,13 +330,10 @@ def CustomObject(type='x-custom-type', properties=None): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ]) - if not properties: + if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - normal_properties = [x for x in properties if not x[0].startswith("x_")] - custom_properties = [x for x in properties if x[0].startswith("x_")] - - _properties.update(normal_properties) + _properties.update([x for x in properties if not x[0].startswith("x_")]) # This is to follow the general properties structure. _properties.update([ @@ -348,7 +345,7 @@ def CustomObject(type='x-custom-type', properties=None): ]) # Put all custom properties at the bottom, sorted alphabetically. - _properties.update(sorted(custom_properties, key=lambda x: x[0])) + _properties.update(sorted([x for x in properties if x[0].startswith("x_")], key=lambda x: x[0])) def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) From 07bbe23a98f12dd9f5d8b6012baacd0cc44d4a19 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:32:05 -0400 Subject: [PATCH 09/14] Add some tests for custom objects. --- stix2/test/test_custom.py | 34 ++++++++++++++++++++++ stix2/test/test_markings.py | 56 +++++++++++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 4ce99c6..2b08883 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -128,6 +128,40 @@ def test_custom_observable_object(): NewObservable(property1='something', property2=4) +def test_custom_no_properties_raises_exception(): + with pytest.raises(ValueError): + + @stix2.sdo.CustomObject('x-new-object-type') + class NewObject1(object): + pass + + NewObject1() + + @stix2.sdo.CustomObject('x-new-object-type', ("a", 0)) + class NewObject2(object): + pass + + NewObject2() + + @stix2.observables.CustomObservable('x-new-object-type') + class NewObject3(object): + pass + + NewObject3() + + @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) + class NewObject4(object): + pass + + NewObject4() + + @stix2.common.CustomMarking('x-new-marking-type') + class NewObject5(object): + pass + + NewObject5() + + def test_parse_custom_observable_object(): nt_string = """{ "type": "x-new-observable", diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index f36b032..4e39c54 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -75,7 +75,7 @@ def test_marking_def_example_with_tlp(): assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION -def test_marking_def_example_with_statement(): +def test_marking_def_example_with_statement_positional_argument(): marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", @@ -86,12 +86,13 @@ def test_marking_def_example_with_statement(): assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION -def test_marking_def_example_with_positional_statement(): +def test_marking_def_example_with_kwargs_statement(): + kwargs = dict(statement="Copyright 2016, Example Corp") marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", definition_type="statement", - definition=stix2.StatementMarking("Copyright 2016, Example Corp") + definition=stix2.StatementMarking(**kwargs) ) assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION @@ -180,4 +181,53 @@ def test_parse_marking_definition(data): assert gm.definition_type == "tlp" +@stix2.common.CustomMarking('x-new-marking-type', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), +]) +class NewMarking(object): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + +def test_registered_custom_marking(): + nm = NewMarking(property1='something', property2=55) + + marking_def = stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type", + definition=nm + ) + + assert marking_def.type == "marking-definition" + assert marking_def.id == "marking-definition--00000000-0000-0000-0000-000000000012" + assert marking_def.created == dt.datetime(2017, 1, 22, 0, 0, 0, tzinfo=pytz.utc) + assert marking_def.definition.property1 == "something" + assert marking_def.definition.property2 == 55 + assert marking_def.definition_type == "x-new-marking-type" + + +def test_not_registered_marking_raises_exception(): + with pytest.raises(ValueError): + @stix2.sdo.CustomObject('x-new-marking-type2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ]) + class NewObject2(object): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + no = NewObject2(property1='something', property2=55) + + stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type2", + definition=no + ) + + # TODO: Add other examples From 160d69b03df6e23b6b8c6d7f85df777ef1c010ae Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:32:51 -0400 Subject: [PATCH 10/14] Changes to datastores. --- stix2/sources/__init__.py | 78 ++++++++++++++++++------------------- stix2/sources/filesystem.py | 10 ++--- stix2/sources/memory.py | 10 ++--- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index d8676ca..87ff913 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -72,11 +72,11 @@ class DataStore(object): this abstract class for the specific data store. """ - def __init__(self, name="DataStore"): + def __init__(self, name="DataStore", source=None, sink=None): self.name = name - self.id = make_id() - self.source = None - self.sink = None + self.id_ = make_id() + self.source = source + self.sink = sink def get(self, stix_id): """ @@ -146,14 +146,14 @@ class DataSink(object): different sink components. Attributes: - id (str): A unique UUIDv4 to identify this DataSink. + id_ (str): A unique UUIDv4 to identify this DataSink. name (str): The descriptive name that identifies this DataSink. """ def __init__(self, name="DataSink"): self.name = name - self.id = make_id() + self.id_ = make_id() def add(self, stix_objs): """ @@ -171,17 +171,15 @@ class DataSource(object): different source components. Attributes: - id (str): A unique UUIDv4 to identify this DataSource. + id_ (str): A unique UUIDv4 to identify this DataSource. name (str): The descriptive name that identifies this DataSource. - filters (dict): A collection of filters present in this DataSource. - filter_allowed (dict): A collection of the allowed filters in this - DataSource. + filters (set): A collection of filters present in this DataSource. """ def __init__(self, name="DataSource"): self.name = name - self.id = make_id() + self.id_ = make_id() self.filters = set() def get(self, stix_id, _composite_filters=None): @@ -250,26 +248,26 @@ class DataSource(object): """Add multiple filters to the DataSource. Args: - filter (list): list of filters (dict) to add to the Data Source. + filters (list): list of filters (dict) to add to the Data Source. """ - for filter in filters: - self.add_filter(filter) + for filter_ in filters: + self.add_filter(filter_) - def add_filter(self, filter): + def add_filter(self, filter_): """Add a filter.""" # check filter field is a supported STIX 2.0 common field - if filter.field not in STIX_COMMON_FIELDS: + if filter_.field not in STIX_COMMON_FIELDS: raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") # check filter operator is supported - if filter.op not in FILTER_OPS: + if filter_.op not in FILTER_OPS: raise ValueError("Filter operation(from 'op' field) not supported") # check filter value type is supported - if type(filter.value) not in FILTER_VALUE_TYPES: + if type(filter_.value) not in FILTER_VALUE_TYPES: raise ValueError("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - self.filters.add(filter) + self.filters.add(filter_) # TODO: Do we need a remove_filter function? @@ -398,14 +396,14 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds_id, ds in iteritems(self.data_sources): data = ds.get(stix_id=stix_id, _composite_filters=list(self.filters)) - all_data.extend(data) + all_data.append(data) # remove duplicate versions if len(all_data) > 0: all_data = self.deduplicate(all_data) # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0] return stix_obj @@ -430,10 +428,14 @@ class CompositeDataSource(DataSource): """ all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) # retrieve STIX objects from all configured data sources for ds_id, ds in iteritems(self.data_sources): - data = ds.all_versions(stix_id=stix_id, _composite_filters=list(self.filters)) + data = ds.all_versions(stix_id=stix_id, _composite_filters=list(all_filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -463,11 +465,15 @@ class CompositeDataSource(DataSource): query = [] all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) # federate query to all attached data sources, # pass composite filters to id for ds_id, ds in iteritems(self.data_sources): - data = ds.query(query=query, _composite_filters=list(self.filters)) + data = ds.query(query=query, _composite_filters=list(all_filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 @@ -486,14 +492,14 @@ class CompositeDataSource(DataSource): """ for ds in data_sources: - if issubclass(ds, DataSource): - if self.data_sources[ds['id']] in self.data_sources.keys(): + if issubclass(ds.__class__, DataSource): + if ds.id_ in self.data_sources: # data source already attached to Composite Data Source continue # add data source to Composite Data Source # (its id will be its key identifier) - self.data_sources[ds['id']] = ds + self.data_sources[ds.id_] = ds else: # the Data Source object is not a proper subclass # of DataSource Abstract Class @@ -506,27 +512,21 @@ class CompositeDataSource(DataSource): """Remove/detach Data Source from the Composite Data Source instance Args: - data_source_ids (list): a list of Data Source - id's(which are strings) + data_source_ids (list): a list of Data Source identifiers. """ - for id_ in data_source_ids: - try: - if self.data_sources[id_]: - del self.data_sources[id_] - except KeyError: - # Data Source 'id' was not found in CompositeDataSource's - # list of data sources - pass + if id_ in self.data_sources: + del self.data_sources[id_] + else: + raise ValueError("DataSource 'id' not found in CompositeDataSource collection.") return - @property - def data_sources(self): + def get_all_data_sources(self): """Return all attached Data Sources """ - return copy.deepcopy(self.data_sources.values()) + return self.data_sources.values() class STIXCommonPropertyFilters(object): diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index e16ee0c..61e7c88 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -19,7 +19,7 @@ from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemStore"): + def __init__(self, name="FileSystemStore", stix_dir="stix_data"): super(FileSystemStore, self).__init__(name=name) self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) @@ -28,7 +28,7 @@ class FileSystemStore(DataStore): class FileSystemSink(DataSink): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemSink"): + def __init__(self, name="FileSystemSink", stix_dir="stix_data"): super(FileSystemSink, self).__init__(name=name) self.stix_dir = os.path.abspath(stix_dir) @@ -58,7 +58,7 @@ class FileSystemSink(DataSink): class FileSystemSource(DataSource): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemSource"): + def __init__(self, name="FileSystemSource", stix_dir="stix_data"): super(FileSystemSource, self).__init__(name=name) self.stix_dir = os.path.abspath(stix_dir) @@ -71,8 +71,8 @@ class FileSystemSource(DataSource): return self.stix_dir @stix_dir.setter - def stix_dir(self, dir): - self.stix_dir = dir + def stix_dir(self, dir_): + self.stix_dir = dir_ def get(self, stix_id, _composite_filters=None): """ diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2f45d68..6460ce3 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -29,7 +29,7 @@ from stix2validator import validate_string class MemoryStore(DataStore): """ """ - def __init__(self, stix_data=None, name="MemoryStore"): + def __init__(self, name="MemoryStore", stix_data=None): """ Notes: It doesn't make sense to create a MemoryStore by passing @@ -75,7 +75,7 @@ class MemoryStore(DataStore): class MemorySink(DataSink): """ """ - def __init__(self, stix_data=None, name="MemorySink", _store=False): + def __init__(self, name="MemorySink", stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -150,7 +150,7 @@ class MemorySink(DataSink): class MemorySource(DataSource): - def __init__(self, stix_data=None, name="MemorySource", _store=False): + def __init__(self, name="MemorySource", stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -177,8 +177,8 @@ class MemorySource(DataSource): for stix_obj in stix_data["objects"]: self.data[stix_obj["id"]] = stix_obj else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) + print("Error: json data passed to MemorySource() was found to not be validated by STIX 2 Validator") + print(r.as_dict()) self.data = {} elif type(stix_data) == list: # STIX objects are in a list From aea076103cdd0b65a89e7708a8527c71d3ba2c93 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:33:12 -0400 Subject: [PATCH 11/14] Update tests to improve coverage. --- stix2/test/test_data_sources.py | 241 ++++++++++++++++++++++---------- 1 file changed, 170 insertions(+), 71 deletions(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0095802..df70878 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,19 +1,15 @@ import pytest from taxii2client import Collection -from stix2.sources import DataSource, Filter, taxii +from stix2.sources import CompositeDataSource, DataSink, DataSource, DataStore, Filter, make_id, taxii +from stix2.sources.memory import MemorySource COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' class MockTAXIIClient(object): """Mock for taxii2_client.TAXIIClient""" - - def get(self): - return {} - - def post(self): - return {} + pass @pytest.fixture @@ -21,6 +17,18 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) +def test_ds_smoke(): + ds1 = DataSource() + ds2 = DataSink() + ds3 = DataStore(source=ds1, sink=ds2) + + with pytest.raises(NotImplementedError): + ds3.add(None) + ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + + def test_ds_taxii(collection): ds = taxii.TAXIICollectionSource(collection) assert ds.name == 'TAXIICollectionSource' @@ -104,6 +112,8 @@ def test_add_get_remove_filter(): assert len(ds.filters) == 2 + ds.add_filters(valid_filters) + def test_apply_common_filters(): stix_objs = [ @@ -160,73 +170,112 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] +STIX_OBJS1 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + +STIX_OBJS2 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + def test_deduplicate(): - stix_objs = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } - ] - ds = DataSource() - unique = ds.deduplicate(stix_objs) + unique = ds.deduplicate(STIX_OBJS1) # Only 3 objects are unique # 2 id's vary @@ -243,6 +292,56 @@ def test_deduplicate(): assert "2017-01-27T13:49:53.936Z" in mods +def test_add_remove_composite_datasource(): + cds = CompositeDataSource() + ds1 = DataSource() + ds2 = DataSource() + ds3 = DataSink() + + cds.add_data_source([ds1, ds2, ds1, ds3]) + + assert len(cds.get_all_data_sources()) == 2 + + cds.remove_data_source([ds1.id_, ds2.id_]) + + assert len(cds.get_all_data_sources()) == 0 + + with pytest.raises(ValueError): + cds.remove_data_source([ds3.id_]) + + +def test_composite_datasource_operations(): + BUNDLE1 = dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS1, + spec_version="2.0", + type="bundle") + cds = CompositeDataSource() + ds1 = MemorySource(stix_data=BUNDLE1) + ds2 = MemorySource(stix_data=STIX_OBJS2) + + cds.add_data_source([ds1, ds2]) + + indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + # In STIX_OBJS2 changed the 'modified' property to a later time... + assert len(indicators) == 2 + + indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + query = [ + Filter("type", "=", "indicator") + ] + + results = cds.query(query) + + # STIX_OBJS2 has indicator with later time, one with different id, one with + # original time in STIX_OBJS1 + assert len(results) == 3 + # def test_data_source_file(): # ds = file.FileDataSource() # From 415c53066f99d2d063d5f7d21ec717bd1c223f87 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 15:19:55 -0400 Subject: [PATCH 12/14] Code coverage changes, fix some tests. Add stix2-validator dependency. --- setup.py | 1 + stix2/__init__.py | 7 +++---- stix2/properties.py | 1 - stix2/sources/__init__.py | 1 - stix2/sources/memory.py | 3 ++- stix2/test/test_custom.py | 22 +--------------------- stix2/test/test_data_sources.py | 10 +++++++++- stix2/test/test_markings.py | 13 +++++++++---- 8 files changed, 25 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index d9b1edc..3687e57 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'simplejson', 'six', 'stix2-patterns', + 'stix2-validator', 'taxii2-client', ], ) diff --git a/stix2/__init__.py b/stix2/__init__.py index 76ca696..6e89531 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,10 +3,9 @@ # flake8: noqa from . import exceptions -from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - CustomMarking, ExternalReference, GranularMarking, - KillChainPhase, MarkingDefinition, StatementMarking, - TLPMarking) +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, diff --git a/stix2/properties.py b/stix2/properties.py index a04294e..6406ad4 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -6,7 +6,6 @@ import re import uuid from six import string_types, text_type - from stix2patterns.validator import run_validator from .base import _STIXBase diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 87ff913..424fc3a 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -17,7 +17,6 @@ Notes: """ import collections -import copy import uuid from six import iteritems diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 6460ce3..28d929d 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -21,9 +21,10 @@ Notes: import json import os +from stix2validator import validate_string + from stix2 import Bundle from stix2.sources import DataSink, DataSource, DataStore, Filter -from stix2validator import validate_string class MemoryStore(DataStore): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 2b08883..0ca6fbd 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -135,32 +135,12 @@ def test_custom_no_properties_raises_exception(): class NewObject1(object): pass - NewObject1() - - @stix2.sdo.CustomObject('x-new-object-type', ("a", 0)) - class NewObject2(object): - pass - - NewObject2() - - @stix2.observables.CustomObservable('x-new-object-type') - class NewObject3(object): - pass - - NewObject3() + with pytest.raises(ValueError): @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) class NewObject4(object): pass - NewObject4() - - @stix2.common.CustomMarking('x-new-marking-type') - class NewObject5(object): - pass - - NewObject5() - def test_parse_custom_observable_object(): nt_string = """{ diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index df70878..10e4cc7 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,8 @@ import pytest from taxii2client import Collection -from stix2.sources import CompositeDataSource, DataSink, DataSource, DataStore, Filter, make_id, taxii +from stix2.sources import (CompositeDataSource, DataSink, DataSource, + DataStore, Filter, make_id, taxii) from stix2.sources.memory import MemorySource COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -24,8 +25,14 @@ def test_ds_smoke(): with pytest.raises(NotImplementedError): ds3.add(None) + + with pytest.raises(NotImplementedError): ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) @@ -170,6 +177,7 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] + STIX_OBJS1 = [ { "created": "2017-01-27T13:49:53.935Z", diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 4e39c54..c9d9273 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -187,8 +187,7 @@ def test_parse_marking_definition(data): ]) class NewMarking(object): def __init__(self, property2=None, **kwargs): - if property2 and property2 < 10: - raise ValueError("'property2' is too small.") + return def test_registered_custom_marking(): @@ -217,8 +216,7 @@ def test_not_registered_marking_raises_exception(): ]) class NewObject2(object): def __init__(self, property2=None, **kwargs): - if property2 and property2 < 10: - raise ValueError("'property2' is too small.") + return no = NewObject2(property1='something', property2=55) @@ -230,4 +228,11 @@ def test_not_registered_marking_raises_exception(): ) +def test_bad_marking_construction(): + with pytest.raises(ValueError): + @stix2.sdo.CustomObject('x-new-marking-type2', ("a", "b")) + class NewObject3(object): + pass + + # TODO: Add other examples From b1ac24d46ed2a974e1d7a67175a414dc4210535c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 29 Aug 2017 15:08:26 -0400 Subject: [PATCH 13/14] Minor changes. --- .isort.cfg | 2 +- stix2/base.py | 6 +++--- stix2/exceptions.py | 2 +- stix2/utils.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index e8d95f8..622e7a5 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests,taxii2client +known_third_party=ordereddict,dateutil,pytest,pytz,requests,simplejson,six,stix2patterns,stix2validator,taxii2client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/stix2/base.py b/stix2/base.py index a302eb5..60dd78a 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -38,7 +38,7 @@ def get_required_properties(properties): class _STIXBase(collections.Mapping): """Base class for STIX object types""" - def _object_properties(self): + def object_properties(self): return list(self._properties.keys()) def _check_property(self, prop_name, prop, kwargs): @@ -146,7 +146,7 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - properties = self._object_properties() + properties = self.object_properties() def sort_by(element): return find_property_index(self, properties, element) @@ -157,7 +157,7 @@ class _STIXBase(collections.Mapping): separators=(",", ": ")) def __repr__(self): - props = [(k, self[k]) for k in self._object_properties() if self.get(k)] + props = [(k, self[k]) for k in self.object_properties() if self.get(k)] return "{0}({1})".format(self.__class__.__name__, ", ".join(["{0!s}={1!r}".format(k, v) for k, v in props])) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index ef47dd0..5a9e7b2 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -121,7 +121,7 @@ class DependentPropertiesError(STIXError, TypeError): def __str__(self): msg = "The property dependencies for {0}: ({1}) are not met." return msg.format(self.cls.__name__, - ", ".join(x for x in self.dependencies)) + ", ".join(name for x in self.dependencies for name in x)) class AtLeastOnePropertyError(STIXError, TypeError): diff --git a/stix2/utils.py b/stix2/utils.py index 9e0f33a..de481fc 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -133,7 +133,7 @@ def find_property_index(obj, properties, tuple_to_find): for item in pv: if isinstance(item, _STIXBase): val = find_property_index(item, - item._object_properties(), + item.object_properties(), tuple_to_find) if val is not None: return val @@ -146,7 +146,7 @@ def find_property_index(obj, properties, tuple_to_find): for item in pv.values(): if isinstance(item, _STIXBase): val = find_property_index(item, - item._object_properties(), + item.object_properties(), tuple_to_find) if val is not None: return val From 8ca2c3390b775a99db36f67d8c97bb315140f962 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 29 Aug 2017 15:15:32 -0400 Subject: [PATCH 14/14] Minor changes to DataSource.apply_common_filters(). Improve overall code coverage. --- stix2/sources/__init__.py | 44 +++-- stix2/test/test_bundle.py | 16 ++ stix2/test/test_custom.py | 4 +- stix2/test/test_data_sources.py | 299 ++++++++++++++++++++----------- stix2/test/test_markings.py | 14 +- stix2/test/test_observed_data.py | 16 +- stix2/test/test_properties.py | 17 +- stix2/test/test_versioning.py | 11 ++ 8 files changed, 288 insertions(+), 133 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 424fc3a..6360fde 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -290,28 +290,34 @@ class DataSource(object): for stix_obj in stix_objs: clean = True for filter_ in query: - - # skip filter as filter was identified (when added) as - # not a common filter - if filter_.field not in STIX_COMMON_FIELDS: - continue - - # check filter "field" is in STIX object - if cant be applied - # due to STIX object, STIX object is discarded (i.e. did not - # make it through the filter) - if filter_.field not in stix_obj.keys(): - clean = False - break try: - match = getattr(STIXCommonPropertyFilters, filter_.field)(filter_, stix_obj) + # skip filter as filter was identified (when added) as + # not a common filter + if filter_.field not in STIX_COMMON_FIELDS: + raise Exception("Error, field: {0} is not supported for filtering on.".format(filter_.field)) + + # For properties like granular_markings and external_references + # need to break the first property from the string. + if "." in filter_.field: + field = filter_.field.split(".")[0] + else: + field = filter_.field + + # check filter "field" is in STIX object - if cant be + # applied due to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + if field not in stix_obj.keys(): + clean = False + break + + match = getattr(STIXCommonPropertyFilters, field)(filter_, stix_obj) if not match: clean = False break elif match == -1: - # error, filter operator not supported for specified field: - pass + raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) except Exception as e: - print(e) + raise ValueError(e) # if object unmarked after all filters, add it if clean: @@ -646,11 +652,11 @@ class STIXCommonPropertyFilters(object): @classmethod def modified(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["created"]) + return cls._timestamp(filter_, stix_obj["modified"]) @classmethod - def object_markings_ref(cls, filter_, stix_obj): - for marking_id in stix_obj["object_market_refs"]: + def object_marking_refs(cls, filter_, stix_obj): + for marking_id in stix_obj["object_marking_refs"]: r = cls._id(filter_, marking_id) if r: return r diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b5c1da5..6f56d85 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -128,3 +128,19 @@ def test_parse_bundle(): assert bundle.objects[0].type == 'indicator' assert bundle.objects[1].type == 'malware' assert bundle.objects[2].type == 'relationship' + + +def test_parse_unknown_type(): + unknown = { + "type": "other", + "id": "other--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created": "2016-04-06T20:03:00Z", + "modified": "2016-04-06T20:03:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "name": "Green Group Attacks Against Finance", + } + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse(unknown) + assert str(excinfo.value) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator." diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 0ca6fbd..f25044e 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -135,10 +135,12 @@ def test_custom_no_properties_raises_exception(): class NewObject1(object): pass + +def test_custom_wrong_properties_arg_raises_exception(): with pytest.raises(ValueError): @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) - class NewObject4(object): + class NewObject2(object): pass diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 10e4cc7..ee37825 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -18,6 +18,109 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) +STIX_OBJS1 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + +STIX_OBJS2 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + + def test_ds_smoke(): ds1 = DataSource() ds2 = DataSink() @@ -149,9 +252,21 @@ def test_apply_common_filters(): }, { "created": "2014-05-08T09:00:00.000Z", + "granular_markings": [ + { + "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + "selectors": [ + "relationship_type" + ] + } + ], "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", "modified": "2014-05-08T09:00:00.000Z", + "object_marking_refs": [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + ], "relationship_type": "indicates", + "revoked": True, "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", "type": "relationship" @@ -162,6 +277,13 @@ def test_apply_common_filters(): Filter("type", "!=", "relationship"), Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), Filter("labels", "in", "remote-access-trojan"), + Filter("created", ">", "2015-01-01T01:00:00.000Z"), + Filter("revoked", "=", True), + Filter("revoked", "!=", True), + Filter("revoked", "?", False), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "relationship_type"), + Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), ] ds = DataSource() @@ -177,108 +299,85 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] + resp = ds.apply_common_filters(stix_objs, [filters[3]]) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 1 -STIX_OBJS1 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] + resp = ds.apply_common_filters(stix_objs, [filters[4]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 -STIX_OBJS2 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-31T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] + # Note that if 'revoked' property is not present in object. + # Currently we can't use such an expression to filter for... + resp = ds.apply_common_filters(stix_objs, [filters[5]]) + assert len(resp) == 0 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(stix_objs, [filters[6]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(filters[6].op, + filters[6].field) + + resp = ds.apply_common_filters(stix_objs, [filters[7]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + resp = ds.apply_common_filters(stix_objs, [filters[8], filters[9]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # These are used with STIX_OBJS2 + more_filters = [ + Filter("modified", "<", "2017-01-28T13:49:53.935Z"), + Filter("modified", ">", "2017-01-28T13:49:53.935Z"), + Filter("modified", ">=", "2017-01-27T13:49:53.935Z"), + Filter("modified", "<=", "2017-01-27T13:49:53.935Z"), + Filter("modified", "?", "2017-01-27T13:49:53.935Z"), + Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), + Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), + Filter("notacommonproperty", "=", "bar"), + ] + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[0]]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[1]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[2]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[3]]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[4]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(more_filters[4].op, + more_filters[4].field) + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[5]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[6]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(more_filters[6].op, + more_filters[6].field) + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[7]]) + + assert str(excinfo.value) == ("Error, field: {0} is not supported for " + "filtering on.".format(more_filters[7].field)) def test_deduplicate(): diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index c9d9273..9fe51fb 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -209,7 +209,8 @@ def test_registered_custom_marking(): def test_not_registered_marking_raises_exception(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: + # Used custom object on purpose to demonstrate a not-registered marking @stix2.sdo.CustomObject('x-new-marking-type2', [ ('property1', stix2.properties.StringProperty(required=True)), ('property2', stix2.properties.IntegerProperty()), @@ -227,12 +228,17 @@ def test_not_registered_marking_raises_exception(): definition=no ) + assert str(excinfo.value) == "definition_type must be a valid marking type" -def test_bad_marking_construction(): - with pytest.raises(ValueError): - @stix2.sdo.CustomObject('x-new-marking-type2', ("a", "b")) + +def test_marking_wrong_type_construction(): + with pytest.raises(ValueError) as excinfo: + # Test passing wrong type for properties. + @stix2.CustomMarking('x-new-marking-type2', ("a", "b")) class NewObject3(object): pass + assert str(excinfo.value) == "Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]" + # TODO: Add other examples diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index dd8f042..fdc66ee 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -8,6 +8,8 @@ import stix2 from .constants import OBSERVED_DATA_ID +OBJECTS_REGEX = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL) + EXPECTED = """{ "type": "observed-data", @@ -173,7 +175,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +196,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -204,6 +206,7 @@ def test_artifact_example_dependency_error(): stix2.Artifact(url="http://example.com/sirvizio.exe") assert excinfo.value.dependencies == [("hashes", "url")] + assert str(excinfo.value) == "The property dependencies for Artifact: (hashes, url) are not met." @pytest.mark.parametrize("data", [ @@ -215,7 +218,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +361,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" @@ -555,6 +558,7 @@ def test_artifact_mutual_exclusion_error(): assert excinfo.value.cls == stix2.Artifact assert excinfo.value.properties == ["payload_bin", "url"] + assert str(excinfo.value) == "The (payload_bin, url) properties for Artifact are mutually exclusive." def test_directory_example(): @@ -925,6 +929,10 @@ def test_process_example_empty_error(): properties_of_process = list(stix2.Process._properties.keys()) properties_of_process.remove("type") assert excinfo.value.properties == sorted(properties_of_process) + msg = "At least one of the ({1}) properties for {0} must be populated." + msg = msg.format(stix2.Process.__name__, + ", ".join(sorted(properties_of_process))) + assert str(excinfo.value) == msg def test_process_example_empty_with_extensions(): diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 01daebf..db1c143 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -206,15 +206,22 @@ def test_dictionary_property_valid(d): @pytest.mark.parametrize("d", [ - {'a': 'something'}, - {'a'*300: 'something'}, - {'Hey!': 'something'}, + [{'a': 'something'}, "Invalid dictionary key a: (shorter than 3 characters)."], + [{'a'*300: 'something'}, "Invalid dictionary key aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaa: (longer than 256 characters)."], + [{'Hey!': 'something'}, "Invalid dictionary key Hey!: (contains characters other thanlowercase a-z, " + "uppercase A-Z, numerals 0-9, hyphen (-), or underscore (_))."], ]) def test_dictionary_property_invalid(d): dict_prop = DictionaryProperty() - with pytest.raises(DictionaryKeyError): - dict_prop.clean(d) + with pytest.raises(DictionaryKeyError) as excinfo: + dict_prop.clean(d[0]) + + assert str(excinfo.value) == d[1] @pytest.mark.parametrize("value", [ diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 281ae71..8506d16 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -128,6 +128,11 @@ def test_versioning_error_bad_modified_value(): assert excinfo.value.prop_name == "modified" assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + msg = "Invalid value for {0} '{1}': {2}" + msg = msg.format(stix2.Campaign.__name__, "modified", + "The new modified datetime cannot be before the current modified datatime.") + assert str(excinfo.value) == msg + def test_versioning_error_usetting_required_property(): campaign_v1 = stix2.Campaign( @@ -145,6 +150,10 @@ def test_versioning_error_usetting_required_property(): assert excinfo.value.cls == stix2.Campaign assert excinfo.value.properties == ["name"] + msg = "No values for required properties for {0}: ({1})." + msg = msg.format(stix2.Campaign.__name__, "name") + assert str(excinfo.value) == msg + def test_versioning_error_new_version_of_revoked(): campaign_v1 = stix2.Campaign( @@ -162,6 +171,7 @@ def test_versioning_error_new_version_of_revoked(): campaign_v2.new_version(name="barney") assert excinfo.value.called_by == "new_version" + assert str(excinfo.value) == "Cannot create a new version of a revoked object." def test_versioning_error_revoke_of_revoked(): @@ -180,3 +190,4 @@ def test_versioning_error_revoke_of_revoked(): campaign_v2.revoke() assert excinfo.value.called_by == "revoke" + assert str(excinfo.value) == "Cannot revoke an already revoked object."