From dbe9c3f0875c9b49579580017add733432e01e06 Mon Sep 17 00:00:00 2001 From: Liam Sennitt Date: Fri, 1 Jun 2018 13:49:32 +0100 Subject: [PATCH 01/11] fix custom STIX objects with nested dictionary error #184 --- stix2/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stix2/utils.py b/stix2/utils.py index 619652a..5bbf99f 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -203,6 +203,8 @@ def find_property_index(obj, properties, tuple_to_find): tuple_to_find) if val is not None: return val + else: + return 0 def new_version(data, **kwargs): From 5a9f627669243d4e77b73a457a8ced019b29ad9f Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Wed, 6 Jun 2018 15:30:45 -0400 Subject: [PATCH 02/11] Pickle-proof stix objects --- stix2/base.py | 8 +++++++- stix2/test/test_pickle.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 stix2/test/test_pickle.py diff --git a/stix2/base.py b/stix2/base.py index 2afba16..9b5308f 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -185,7 +185,13 @@ class _STIXBase(collections.Mapping): # Handle attribute access just like key access def __getattr__(self, name): - if name in self: + # Pickle-proofing: pickle invokes this on uninitialized instances (i.e. + # __init__ has not run). So no "self" attributes are set yet. The + # usual behavior of this method reads an __init__-assigned attribute, + # which would cause infinite recursion. So this check disables all + # attribute reads until the instance has been properly initialized. + unpickling = "_inner" not in self.__dict__ + if not unpickling and name in self: return self.__getitem__(name) raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) diff --git a/stix2/test/test_pickle.py b/stix2/test/test_pickle.py new file mode 100644 index 0000000..9e2cc9a --- /dev/null +++ b/stix2/test/test_pickle.py @@ -0,0 +1,17 @@ +import pickle + +import stix2 + + +def test_pickling(): + """ + Ensure a pickle/unpickle cycle works okay. + """ + identity = stix2.Identity( + id="identity--d66cb89d-5228-4983-958c-fa84ef75c88c", + name="alice", + description="this is a pickle test", + identity_class="some_class" + ) + + pickle.loads(pickle.dumps(identity)) From 60da25980580f6c9b04306c0b49a18d48728d2d3 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 8 Jun 2018 15:42:59 -0400 Subject: [PATCH 03/11] Add an example on how to call the serialize() method. --- stix2/base.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/stix2/base.py b/stix2/base.py index 2afba16..0ba6f56 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -236,6 +236,21 @@ class _STIXBase(collections.Mapping): optional properties set to the default value defined in the spec. **kwargs: The arguments for a json.dumps() call. + Examples: + >>> import stix2 + >>> identity = stix2.Identity(name='Example Corp.', identity_class='organization') + >>> print(identity.serialize(sort_keys=True)) + {"created": "2018-06-08T19:03:54.066Z", ... "name": "Example Corp.", "type": "identity"} + >>> print(identity.serialize(sort_keys=True, indent=4)) + { + "created": "2018-06-08T19:03:54.066Z", + "id": "identity--d7f3e25a-ba1c-447a-ab71-6434b092b05e", + "identity_class": "organization", + "modified": "2018-06-08T19:03:54.066Z", + "name": "Example Corp.", + "type": "identity" + } + Returns: str: The serialized JSON object. From 91cae0b5b7667ff22b09561c326fe2203fdf78c4 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 8 Jun 2018 15:43:21 -0400 Subject: [PATCH 04/11] Split `find_property_index` and support multiple levels of nesting. --- stix2/utils.py | 108 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 33 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index 5bbf99f..502e8e1 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -165,46 +165,88 @@ def _get_dict(data): raise ValueError("Cannot convert '%s' to dictionary." % str(data)) +def _iterate_over_values(dict_values, tuple_to_find): + """Loop recursively over dictionary values""" + from .base import _STIXBase + for pv in dict_values: + if isinstance(pv, list): + for item in pv: + if isinstance(item, _STIXBase): + val = find_property_index( + item, + item.object_properties(), + tuple_to_find + ) + if val is not None: + return val + elif isinstance(item, dict): + for idx, val in enumerate(sorted(item)): + if (tuple_to_find[0] == val and + item.get(val) == tuple_to_find[1]): + return idx + elif isinstance(pv, dict): + if pv.get(tuple_to_find[0]) is not None: + try: + return int(tuple_to_find[0]) + except ValueError: + return len(tuple_to_find[0]) + for item in pv.values(): + if isinstance(item, _STIXBase): + index = find_property_index( + item, + item.object_properties(), + tuple_to_find + ) + if index is not None: + return index + elif isinstance(item, dict): + dict_properties = item.keys() + index = find_property_index( + item, + dict_properties, + tuple_to_find + ) + if index is not None: + return index + + def find_property_index(obj, properties, tuple_to_find): """Recursively find the property in the object model, return the index - according to the _properties OrderedDict. If it's a list look for - individual objects. Returns and integer indicating its location + according to the ``properties`` OrderedDict when working with `stix2` + objects. If it's a list look for individual objects. Returns and integer + indicating its location. + + Notes: + This method is intended to pretty print `stix2` properties for better + visual feedback when working with the library. + + Warnings: + This method may not be able to produce the same output if called + multiple times and makes a best effort attempt to print the properties + according to the technical specification. + + See Also: + py:meth:`stix2.base._STIXBase.serialize` for more information. + """ from .base import _STIXBase try: - if tuple_to_find[1] in obj._inner.values(): - return properties.index(tuple_to_find[0]) + if isinstance(obj, _STIXBase): + if tuple_to_find[1] in obj._inner.values(): + return properties.index(tuple_to_find[0]) + elif isinstance(obj, dict): + for idx, val in enumerate(sorted(obj)): + if (tuple_to_find[0] == val and + obj.get(val) == tuple_to_find[1]): + return idx raise ValueError except ValueError: - for pv in obj._inner.values(): - if isinstance(pv, list): - for item in pv: - if isinstance(item, _STIXBase): - val = find_property_index(item, - item.object_properties(), - tuple_to_find) - if val is not None: - return val - elif isinstance(item, dict): - for idx, val in enumerate(sorted(item)): - if (tuple_to_find[0] == val and - item.get(val) == tuple_to_find[1]): - return idx - elif isinstance(pv, dict): - if pv.get(tuple_to_find[0]) is not None: - try: - return int(tuple_to_find[0]) - except ValueError: - return len(tuple_to_find[0]) - for item in pv.values(): - if isinstance(item, _STIXBase): - val = find_property_index(item, - item.object_properties(), - tuple_to_find) - if val is not None: - return val - else: - return 0 + if isinstance(obj, _STIXBase): + index = _iterate_over_values(obj._inner.values(), tuple_to_find) + return index + elif isinstance(obj, dict): + index = _iterate_over_values(obj.values(), tuple_to_find) + return index def new_version(data, **kwargs): From 02894b5be6f9ca0f774b18e875bd10d71976bc83 Mon Sep 17 00:00:00 2001 From: Liam Sennitt Date: Tue, 12 Jun 2018 17:32:46 +0100 Subject: [PATCH 05/11] add test for nested dictionary case --- stix2/test/test_custom.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 7f91d79..68a9e50 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -860,3 +860,30 @@ def test_register_custom_object(): def test_extension_property_location(): assert 'extensions' in stix2.v20.observables.OBJ_MAP_OBSERVABLE['x-new-observable']._properties assert 'extensions' not in stix2.v20.observables.EXT_MAP['domain-name']['x-new-ext']._properties + + +@pytest.mark.parametrize("data", [ + """{ + "type": "x-example", + "id": "x-example--336d8a9f-91f1-46c5-b142-6441bb9f8b8d", + "created": "2018-06-12T16:20:58.059Z", + "modified": "2018-06-12T16:20:58.059Z", + "dictionary": { + "key": { + "key_a": "value", + "key_b": "value" + } + } +}""", +]) +def test_custom_object_nested_dictionary(data): + @stix2.sdo.CustomObject('x-example', [ + ('dictionary', stix2.properties.DictionaryProperty()), + ]) + class Example(object): + def __init__(self, **kwargs): + pass + + example = Example(id='x-example--336d8a9f-91f1-46c5-b142-6441bb9f8b8d', created='2018-06-12T16:20:58.059Z', modified='2018-06-12T16:20:58.059Z', dictionary={'key':{'key_b':'value', 'key_a':'value'}}) + + assert data == str(example) From bdec14937d1c67cff0e70b24d15dda95c1caad55 Mon Sep 17 00:00:00 2001 From: Liam Sennitt Date: Tue, 12 Jun 2018 17:45:48 +0100 Subject: [PATCH 06/11] fix flake8 errors in new test --- stix2/test/test_custom.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 68a9e50..6fb24d2 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -884,6 +884,9 @@ def test_custom_object_nested_dictionary(data): def __init__(self, **kwargs): pass - example = Example(id='x-example--336d8a9f-91f1-46c5-b142-6441bb9f8b8d', created='2018-06-12T16:20:58.059Z', modified='2018-06-12T16:20:58.059Z', dictionary={'key':{'key_b':'value', 'key_a':'value'}}) + example = Example(id='x-example--336d8a9f-91f1-46c5-b142-6441bb9f8b8d', + created='2018-06-12T16:20:58.059Z', + modified='2018-06-12T16:20:58.059Z', + dictionary={'key': {'key_b': 'value', 'key_a': 'value'}}) assert data == str(example) From 9be819ea6a4729bfb5a29c56b0ee7cce58aa183b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 12 Jun 2018 12:57:25 -0400 Subject: [PATCH 07/11] Minor tweaks to return immediately after recursive call. Ensure dicts are matched alphabetically --- stix2/test/test_utils.py | 74 +++++++++++++++++++++++++++++++++++++--- stix2/utils.py | 32 +++++++---------- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py index 655cd61..49bfe86 100644 --- a/stix2/test/test_utils.py +++ b/stix2/test/test_utils.py @@ -10,7 +10,7 @@ amsterdam = pytz.timezone('Europe/Amsterdam') eastern = pytz.timezone('US/Eastern') -@pytest.mark.parametrize('dttm,timestamp', [ +@pytest.mark.parametrize('dttm, timestamp', [ (dt.datetime(2017, 1, 1, tzinfo=pytz.utc), '2017-01-01T00:00:00Z'), (amsterdam.localize(dt.datetime(2017, 1, 1)), '2016-12-31T23:00:00Z'), (eastern.localize(dt.datetime(2017, 1, 1, 12, 34, 56)), '2017-01-01T17:34:56Z'), @@ -76,12 +76,12 @@ def test_get_dict_invalid(data): stix2.utils._get_dict(data) -@pytest.mark.parametrize('stix_id, typ', [ +@pytest.mark.parametrize('stix_id, type', [ ('malware--d69c8146-ab35-4d50-8382-6fc80e641d43', 'malware'), ('intrusion-set--899ce53f-13a0-479b-a0e4-67d46e241542', 'intrusion-set') ]) -def test_get_type_from_id(stix_id, typ): - assert stix2.utils.get_type_from_id(stix_id) == typ +def test_get_type_from_id(stix_id, type): + assert stix2.utils.get_type_from_id(stix_id) == type def test_deduplicate(stix_objs1): @@ -100,3 +100,69 @@ def test_deduplicate(stix_objs1): assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids assert "2017-01-27T13:49:53.935Z" in mods assert "2017-01-27T13:49:53.936Z" in mods + + +@pytest.mark.parametrize('object, tuple_to_find, expected_index', [ + (stix2.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects={ + "0": { + "name": "foo.exe", + "type": "file" + }, + "1": { + "type": "ipv4-addr", + "value": "198.51.100.3" + }, + "2": { + "type": "network-traffic", + "src_ref": "1", + "protocols": [ + "tcp", + "http" + ], + "extensions": { + "http-request-ext": { + "request_method": "get", + "request_value": "/download.html", + "request_version": "http/1.1", + "request_header": { + "Accept-Encoding": "gzip,deflate", + "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113", + "Host": "www.example.com" + } + } + } + } + }, + ), ('1', {"type": "ipv4-addr", "value": "198.51.100.3"}), 1), + ({ + "type": "x-example", + "id": "x-example--d5413db2-c26c-42e0-b0e0-ec800a310bfb", + "created": "2018-06-11T01:25:22.063Z", + "modified": "2018-06-11T01:25:22.063Z", + "dictionary": { + "key": { + "key_one": "value", + "key_two": "value" + } + } + }, ('key', {'key_one': 'value', 'key_two': 'value'}), 0) +]) +def test_find_property_index(object, tuple_to_find, expected_index): + assert stix2.utils.find_property_index( + object, + [], + tuple_to_find + ) == expected_index + + + +def test_iterate_over_values(): + pass diff --git a/stix2/utils.py b/stix2/utils.py index 502e8e1..cdbb88e 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -172,42 +172,36 @@ def _iterate_over_values(dict_values, tuple_to_find): if isinstance(pv, list): for item in pv: if isinstance(item, _STIXBase): - val = find_property_index( + return find_property_index( item, item.object_properties(), tuple_to_find ) - if val is not None: - return val elif isinstance(item, dict): for idx, val in enumerate(sorted(item)): if (tuple_to_find[0] == val and item.get(val) == tuple_to_find[1]): return idx elif isinstance(pv, dict): - if pv.get(tuple_to_find[0]) is not None: - try: + for idx, item in enumerate(sorted(pv.keys())): + if ((item == tuple_to_find[0] and str.isdecimal(item)) and + (pv[item] == tuple_to_find[1])): return int(tuple_to_find[0]) - except ValueError: - return len(tuple_to_find[0]) + elif pv[item] == tuple_to_find[1]: + return idx for item in pv.values(): if isinstance(item, _STIXBase): - index = find_property_index( + return find_property_index( item, item.object_properties(), tuple_to_find ) - if index is not None: - return index elif isinstance(item, dict): - dict_properties = item.keys() - index = find_property_index( + return find_property_index( item, - dict_properties, + item.keys(), tuple_to_find ) - if index is not None: - return index def find_property_index(obj, properties, tuple_to_find): @@ -223,7 +217,7 @@ def find_property_index(obj, properties, tuple_to_find): Warnings: This method may not be able to produce the same output if called multiple times and makes a best effort attempt to print the properties - according to the technical specification. + according to the STIX technical specification. See Also: py:meth:`stix2.base._STIXBase.serialize` for more information. @@ -242,11 +236,9 @@ def find_property_index(obj, properties, tuple_to_find): raise ValueError except ValueError: if isinstance(obj, _STIXBase): - index = _iterate_over_values(obj._inner.values(), tuple_to_find) - return index + return _iterate_over_values(obj._inner.values(), tuple_to_find) elif isinstance(obj, dict): - index = _iterate_over_values(obj.values(), tuple_to_find) - return index + return _iterate_over_values(obj.values(), tuple_to_find) def new_version(data, **kwargs): From fcffb165ad0a2c3e9cf6fe8012ed4942a49aa549 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 12 Jun 2018 14:38:35 -0400 Subject: [PATCH 08/11] Add tests for `find_property_index` and `iterate_over_values` --- stix2/test/test_utils.py | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py index 49bfe86..fb63ff7 100644 --- a/stix2/test/test_utils.py +++ b/stix2/test/test_utils.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import datetime as dt from io import StringIO @@ -153,7 +155,25 @@ def test_deduplicate(stix_objs1): "key_two": "value" } } - }, ('key', {'key_one': 'value', 'key_two': 'value'}), 0) + }, ('key', {'key_one': 'value', 'key_two': 'value'}), 0), + ({ + "type": "language-content", + "id": "language-content--b86bd89f-98bb-4fa9-8cb2-9ad421da981d", + "created": "2017-02-08T21:31:22.007Z", + "modified": "2017-02-08T21:31:22.007Z", + "object_ref": "campaign--12a111f0-b824-4baf-a224-83b80237a094", + "object_modified": "2017-02-08T21:31:22.007Z", + "contents": { + "de": { + "name": "Bank Angriff 1", + "description": "Weitere Informationen über Banküberfall" + }, + "fr": { + "name": "Attaque Bank 1", + "description": "Plus d'informations sur la crise bancaire" + } + } + }, ('fr', {"name": "Attaque Bank 1", "description": "Plus d'informations sur la crise bancaire"}), 1) ]) def test_find_property_index(object, tuple_to_find, expected_index): assert stix2.utils.find_property_index( @@ -163,6 +183,29 @@ def test_find_property_index(object, tuple_to_find, expected_index): ) == expected_index - -def test_iterate_over_values(): - pass +@pytest.mark.parametrize('dict_value, tuple_to_find, expected_index', [ + ({ + "contents": { + "de": { + "name": "Bank Angriff 1", + "description": "Weitere Informationen über Banküberfall" + }, + "fr": { + "name": "Attaque Bank 1", + "description": "Plus d'informations sur la crise bancaire" + }, + "es": { + "name": "Ataque al Banco", + "description": "Mas informacion sobre el ataque al banco" + } + } + }, ('es', {"name": "Ataque al Banco", "description": "Mas informacion sobre el ataque al banco"}), 1), # Sorted alphabetically + ({ + 'my_list': [ + {"key_one": 1}, + {"key_two": 2} + ] + }, ('key_one', 1), 0) +]) +def test_iterate_over_values(dict_value, tuple_to_find, expected_index): + assert stix2.utils._iterate_over_values(dict_value.values(), tuple_to_find) == expected_index From 78a480aa0881e1e9bde80eac7b97ed167cacce99 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 12 Jun 2018 14:45:15 -0400 Subject: [PATCH 09/11] Change str.isdecimal to isdigit. On lists keep iterating if unless index is found --- stix2/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index cdbb88e..b1a7eda 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -172,11 +172,13 @@ def _iterate_over_values(dict_values, tuple_to_find): if isinstance(pv, list): for item in pv: if isinstance(item, _STIXBase): - return find_property_index( + index = find_property_index( item, item.object_properties(), tuple_to_find ) + if index is not None: + return index elif isinstance(item, dict): for idx, val in enumerate(sorted(item)): if (tuple_to_find[0] == val and @@ -184,7 +186,7 @@ def _iterate_over_values(dict_values, tuple_to_find): return idx elif isinstance(pv, dict): for idx, item in enumerate(sorted(pv.keys())): - if ((item == tuple_to_find[0] and str.isdecimal(item)) and + if ((item == tuple_to_find[0] and str.isdigit(item)) and (pv[item] == tuple_to_find[1])): return int(tuple_to_find[0]) elif pv[item] == tuple_to_find[1]: From c12336b55a9071af2f3aa6f330a844e5e416ed13 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 12 Jun 2018 15:03:25 -0400 Subject: [PATCH 10/11] Revert changes to iterables. --- stix2/utils.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/stix2/utils.py b/stix2/utils.py index b1a7eda..137ea01 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -185,25 +185,30 @@ def _iterate_over_values(dict_values, tuple_to_find): item.get(val) == tuple_to_find[1]): return idx elif isinstance(pv, dict): - for idx, item in enumerate(sorted(pv.keys())): - if ((item == tuple_to_find[0] and str.isdigit(item)) and - (pv[item] == tuple_to_find[1])): - return int(tuple_to_find[0]) - elif pv[item] == tuple_to_find[1]: - return idx + if pv.get(tuple_to_find[0]) is not None: + for idx, item in enumerate(sorted(pv.keys())): + if ((item == tuple_to_find[0] and str.isdigit(item)) and + (pv[item] == tuple_to_find[1])): + return int(tuple_to_find[0]) + elif pv[item] == tuple_to_find[1]: + return idx for item in pv.values(): if isinstance(item, _STIXBase): - return find_property_index( + index = find_property_index( item, item.object_properties(), tuple_to_find ) + if index is not None: + return index elif isinstance(item, dict): - return find_property_index( + index = find_property_index( item, item.keys(), tuple_to_find ) + if index is not None: + return index def find_property_index(obj, properties, tuple_to_find): From 2928824ec5654766357a0b481d81081339879e48 Mon Sep 17 00:00:00 2001 From: stmtstk Date: Wed, 13 Jun 2018 14:34:27 +0900 Subject: [PATCH 11/11] Cannot execute sample code. I tried to execute the sample code of parse() on python 2.7 and 3.0. But It occured JSONDecodeError because the "pattern" value of this sample JSON are described as multi-line. To ensure that the sample code works, this value shuld be single-line. --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0212eed..79b8e7e 100644 --- a/README.rst +++ b/README.rst @@ -60,8 +60,7 @@ To parse a STIX JSON string into a Python STIX object, use "malicious-activity" ], "name": "File hash for malware variant", - "pattern": "[file:hashes.md5 = - 'd41d8cd98f00b204e9800998ecf8427e']", + "pattern": "[file:hashes.md5 ='d41d8cd98f00b204e9800998ecf8427e']", "valid_from": "2017-09-26T23:33:39.829952Z" }""") print(indicator)