From c1ce2bd3f1a2586ec0f0c42b610a82a5b4149b5d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 8 Jun 2017 10:44:23 -0400 Subject: [PATCH 01/82] Add markings package structure. --- stix2/markings/__init__.py | 0 stix2/markings/granular_markings.py | 0 stix2/markings/object_markings.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 stix2/markings/__init__.py create mode 100644 stix2/markings/granular_markings.py create mode 100644 stix2/markings/object_markings.py diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py new file mode 100644 index 0000000..e69de29 From 069c82abf1c6f1ee4bd2a5773b5b3eedf40c5a87 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 9 Jun 2017 14:20:16 -0400 Subject: [PATCH 02/82] Add markings.utils.py --- stix2/markings/utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stix2/markings/utils.py diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py new file mode 100644 index 0000000..e69de29 From 1b7695c4f6623c6fced4a6790653c54181f623b3 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 9 Jun 2017 14:21:42 -0400 Subject: [PATCH 03/82] Initial marking code. --- stix2/markings/__init__.py | 197 +++++++++++++++++++++++ stix2/markings/granular_markings.py | 237 ++++++++++++++++++++++++++++ stix2/markings/object_markings.py | 140 ++++++++++++++++ stix2/markings/utils.py | 228 ++++++++++++++++++++++++++ 4 files changed, 802 insertions(+) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index e69de29..8301131 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -0,0 +1,197 @@ +""" +Python STIX 2.0 Data Markings API. + +These high level functions will operate on both object level markings and +granular markings unless otherwise noted in each of the functions. +""" + +from stix2.markings import utils +from stix2.markings import granular_markings +from stix2.markings import object_markings + + +def get_markings(obj, selectors, inherited=False, descendants=False): + """ + Get all markings associated to the field(s). + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + inherited: If True, include object level markings and granular markings + inherited relative to the field(s). + descendants: If True, include granular markings applied to any children + relative to the field(s). + + Returns: + list: Marking IDs that matched the selectors expression. + + Note: + If ``selectors`` is None, operation will be performed only on object + level markings. + + """ + if selectors is None: + return object_markings.get_markings(obj) + + results = granular_markings.get_markings( + obj, + selectors, + inherited, + descendants + ) + + if inherited: + results.extend(object_markings.get_markings(obj)) + + return list(set(results)) + + +def set_markings(obj, selectors, marking): + """ + Removes all markings associated with selectors and appends a new granular + marking. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + object_markings.set_markings(obj, marking) + else: + granular_markings.set_markings(obj, selectors, marking) + + +def remove_markings(obj, selectors, marking): + """ + Removes granular_marking from the granular_markings collection. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + Raises: + AssertionError: If `selectors` or `marking` fail data validation. Also + if markings to remove are not found on the provided TLO. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + object_markings.remove_markings(obj, marking) + else: + granular_markings.remove_markings(obj, selectors, marking) + + +def add_markings(obj, selectors, marking): + """ + Appends a granular_marking to the granular_markings collection. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + Raises: + AssertionError: If `selectors` or `marking` fail data validation. + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + object_markings.add_markings(obj, marking) + else: + granular_markings.add_markings(obj, selectors, marking) + + +def clear_markings(obj, selectors): + """ + Removes all granular_marking associated with the selectors. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + + Note: + If ``selectors`` is None, operations will be performed on object level + markings. Otherwise on granular markings. + + """ + if selectors is None: + object_markings.clear_markings(obj) + else: + granular_markings.clear_markings(obj, selectors) + + +def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): + """ + Checks if field(s) is marked by any marking or by specific marking(s). + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + inherited: If True, include object level markings and granular markings + inherited to determine if the field(s) is/are marked. + descendants: If True, include granular markings applied to any children + of the given selector to determine if the field(s) is/are marked. + + Returns: + bool: True if ``selectors`` is found on internal TLO collection. + False otherwise. + + Note: + When a list of marking IDs is provided, if ANY of the provided marking + IDs matches, True is returned. + + If ``selectors`` is None, operation will be performed only on object + level markings. + + """ + if selectors is None: + return object_markings.is_marked(obj, marking) + + result = granular_markings.is_marked( + obj, + selectors, + marking, + inherited, + descendants + ) + + if inherited: + granular_marks = granular_markings.get_markings(obj, selectors) + object_marks = object_markings.get_markings(obj) + + if granular_marks: + result = granular_markings.is_marked( + obj, + selectors, + granular_marks, + inherited, + descendants + ) + + result = result or object_markings.is_marked(obj, object_marks) + + return result diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index e69de29..6d69f16 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -0,0 +1,237 @@ + +from stix2.markings import utils + + +def get_markings(obj, selectors, inherited=False, descendants=False): + """ + Get all markings associated to the field(s). + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + inherited: If True, include markings inherited relative to the field(s). + descendants: If True, include granular markings applied to any children + relative to the field(s). + + Returns: + list: Marking IDs that matched the selectors expression. + + """ + selectors = utils.fix_value(selectors) + utils.validate(obj, selectors) + + granular_markings = obj.get("granular_markings", []) + + if not granular_markings: + return [] + + results = set() + + for marking in granular_markings: + for user_selector in selectors: + for marking_selector in marking.get("selectors", []): + if any([(user_selector == marking_selector), # Catch explicit selectors. + (user_selector.startswith(marking_selector) and inherited), # Catch inherited selectors. + (marking_selector.startswith(user_selector) and descendants)]): # Catch descendants selectors + refs = marking.get("marking_ref", []) + results.update([refs]) + + return list(results) + + +def set_markings(obj, selectors, marking): + """ + Removes all markings associated with selectors and appends a new granular + marking. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + """ + clear_markings(obj, selectors) + add_markings(obj, selectors, marking) + + +def remove_markings(obj, selectors, marking): + """ + Removes granular_marking from the granular_markings collection. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + Raises: + AssertionError: If `selectors` or `marking` fail data validation. Also + if markings to remove are not found on the provided TLO. + + """ + selectors = utils.fix_value(selectors) + utils.validate(obj, selectors, marking) + + utils.expand_markings(obj) + + granular_markings = obj.get("granular_markings") + + if not granular_markings: + return + + tlo = utils.build_granular_marking( + {"selectors": selectors, "marking_ref": marking} + ) + + remove = tlo.get("granular_markings", []) + + if not any(marking in granular_markings for marking in remove): + raise AssertionError("Unable to remove Granular Marking(s) from" + " internal collection. Marking(s) not found...") + + obj["granular_markings"] = [ + m for m in granular_markings if m not in remove + ] + + utils.compress_markings(obj) + + if not obj.get("granular_markings"): + obj.pop("granular_markings") + + +def add_markings(obj, selectors, marking): + """ + Appends a granular_marking to the granular_markings collection. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + + Raises: + AssertionError: If `selectors` or `marking` fail data validation. + + """ + selectors = utils.fix_value(selectors) + utils.validate(obj, selectors, marking) + + granular_marking = {"selectors": sorted(selectors), "marking_ref": marking} + + if not obj.get("granular_markings"): + obj["granular_markings"] = list() + + obj["granular_markings"].append(granular_marking) + + utils.expand_markings(obj) + utils.compress_markings(obj) + + +def clear_markings(obj, selectors): + """ + Removes all granular_marking associated with the selectors. + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + + Raises: + AssertionError: If `selectors` or `marking` fail data validation. Also + if markings to remove are not found on the provided TLO. + + """ + selectors = utils.fix_value(selectors) + utils.validate(obj, selectors) + + utils.expand_markings(obj) + + granular_markings = obj.get("granular_markings") + + if not granular_markings: + return + + tlo = utils.build_granular_marking( + {"selectors": selectors, "marking_ref": ["N/A"]} + ) + + clear = tlo.get("granular_markings", []) + + if not any(clear_selector in tlo_selectors.get("selectors", []) + for tlo_selectors in granular_markings + for clear_marking in clear + for clear_selector in clear_marking.get("selectors", []) + ): + raise AssertionError("Unable to clear Granular Marking(s) from" + " internal collection. Selector(s) not found...") + + for granular_marking in granular_markings: + for s in selectors: + if s in granular_marking.get("selectors", []): + marking_refs = granular_marking.get("marking_ref") + + if marking_refs: + granular_marking["marking_ref"] = list() + + utils.compress_markings(obj) + + if not obj.get("granular_markings"): + obj.pop("granular_markings") + + +def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): + """ + Checks if field is marked by any marking or by specific marking(s). + + Args: + obj: A TLO object. + selectors: string or list of selectors strings relative to the TLO in + which the field(s) appear(s). + marking: identifier or list of marking identifiers that apply to the + field(s) selected by `selectors`. + inherited: If True, return markings inherited from the given selector. + descendants: If True, return granular markings applied to any children + of the given selector. + + Returns: + bool: True if ``selectors`` is found on internal TLO collection. + False otherwise. + + Note: + When a list of marking IDs is provided, if ANY of the provided marking + IDs matches, True is returned. + + """ + selectors = utils.fix_value(selectors) + marking = utils.fix_value(marking) + utils.validate(obj, selectors, marking) + + granular_markings = obj.get("granular_markings", []) + + marked = False + markings = set() + + for granular_marking in granular_markings: + for user_selector in selectors: + for marking_selector in granular_marking.get("selectors", []): + + if any([(user_selector == marking_selector), # Catch explicit selectors. + (user_selector.startswith(marking_selector) and inherited), # Catch inherited selectors. + (marking_selector.startswith(user_selector) and descendants)]): # Catch descendants selectors + marking_ref = granular_marking.get("marking_ref", "") + + if marking and any(x == marking_ref for x in marking): + markings.update([marking_ref]) + + marked = True + + if marking: + # All user-provided markings must be found. + return markings.issuperset(set(marking)) + + return marked diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index e69de29..33a8d2a 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -0,0 +1,140 @@ + +import six + +from stix2.markings import utils + + +def get_markings(obj): + """ + Get all object level markings from the given TLO object. + + Args: + obj: A TLO object. + + Returns: + list: Marking IDs contained in the TLO. + + """ + object_markings = obj.get("object_marking_refs", []) + + if not object_markings: + return [] + elif isinstance(object_markings, six.string_types): + return [object_markings] + else: + return object_markings + + +def add_markings(obj, marking): + """ + Appends an object level marking to the object_marking_refs collection. + + Args: + obj: A TLO object. + marking: identifier or list of marking identifiers that apply to the + TLO object. + + Raises: + AssertionError: If `marking` fail data validation. + + """ + marking = utils.convert_to_list(marking) + utils.validate(obj, marking=marking) + + if not obj.get("object_marking_refs"): + obj["object_marking_refs"] = list() + + object_markings = set(obj.get("object_marking_refs") + marking) + + obj["object_marking_refs"] = list(object_markings) + + +def remove_markings(obj, marking): + """ + Removes object level marking from the object_marking_refs collection. + + Args: + obj: A TLO object. + marking: identifier or list of marking identifiers that apply to the + TLO object. + + Raises: + AssertionError: If `marking` fail data validation. Also + if markings to remove are not found on the provided TLO. + + """ + marking = utils.convert_to_list(marking) + utils.validate(obj, marking=marking) + + object_markings = obj.get("object_marking_refs", []) + + if not object_markings: + return [] + + if any(x not in obj["object_marking_refs"] for x in marking): + raise AssertionError("Unable to remove Object Level Marking(s) from " + "internal collection. Marking(s) not found...") + + obj["object_marking_refs"] = [x for x in object_markings + if x not in marking] + + if not obj.get("object_marking_refs"): + obj.pop("object_marking_refs") + + +def set_markings(obj, marking): + """ + Removes all object level markings and appends new object level markings to + the collection. Refer to `clear_markings` and `add_markings` for details. + + Args: + obj: A TLO object. + marking: identifier or list of marking identifiers that apply to the + TLO object. + + """ + utils.validate(obj, marking=marking) + + clear_markings(obj) + add_markings(obj, marking) + + +def clear_markings(obj): + """ + Removes all object level markings from the object_marking_refs collection. + + Args: + obj: A TLO object. + + """ + try: + del obj["object_marking_refs"] + except KeyError: + raise AssertionError("Unable to clear Object Marking(s) from internal" + " collection. No Markings in object...") + + +def is_marked(obj, marking=None): + """ + Checks if TLO is marked by any marking or by specific marking(s). + + Args: + obj: A TLO object. + marking: identifier or list of marking identifiers that apply to the + TLO object. + + Returns: + bool: True if TLO has object level markings. False otherwise. + + Note: + When a list of marking IDs is provided, if ANY of the provided marking + IDs matches, True is returned. + + """ + marking = utils.convert_to_list(marking) + object_markings = obj.get("object_marking_refs", []) + + if marking: + return any(x in object_markings for x in marking) + else: + return bool(object_markings) diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index e69de29..9286695 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -0,0 +1,228 @@ + +import collections + +import six + + +def evaluate_expression(obj, selector): + + for items, value in iterpath(obj): + path = ".".join(items) + + if path == selector and value: + return [value] + + return [] + + +def validate_selector(obj, selector): + results = list(evaluate_expression(obj, selector)) + + if len(results) >= 1: + return True + + +def validate_markings(marking): + if isinstance(marking, six.string_types): + if not marking: + return False + else: + return True + + elif isinstance(marking, list) and len(marking) >= 1: + for m in marking: + if not m: + return False + elif not isinstance(m, six.string_types): + return False + + return True + else: + return False + + +def validate(obj, selectors=None, marking=None): + + if selectors is not None: + assert selectors + + for s in selectors: + assert validate_selector(obj, s) + + if marking is not None: + assert validate_markings(marking) + + +def convert_to_list(data): + if data is not None: + if isinstance(data, list): + return data + else: + return [data] + + +def fix_value(data): + data = convert_to_list(data) + + return data + + +def _fix_markings(markings): + + for granular_marking in markings: + refs = granular_marking.get("marking_ref", []) + selectors = granular_marking.get("selectors", []) + + if not isinstance(refs, list): + granular_marking["marking_ref"] = [refs] + + if not isinstance(selectors, list): + granular_marking["selectors"] = [selectors] + + +def _group_by(markings): + + key = "marking_ref" + retrieve = "selectors" + + map_ = collections.defaultdict(set) + + for granular_marking in markings: + for data in granular_marking.get(key, []): + map_[data].update(granular_marking.get(retrieve)) + + granular_markings = \ + [ + {"selectors": sorted(selectors), "marking_ref": ref} + for ref, selectors in six.iteritems(map_) + ] + + return granular_markings + + +def compress_markings(tlo): + + if not tlo.get("granular_markings"): + return + + granular_markings = tlo.get("granular_markings") + + _fix_markings(granular_markings) + + tlo["granular_markings"] = _group_by(granular_markings) + + +def expand_markings(tlo): + + if not tlo.get("granular_markings"): + return + + granular_markings = tlo.get("granular_markings") + + _fix_markings(granular_markings) + + expanded = list() + + for marking in granular_markings: + selectors = marking.get("selectors", []) + marking_ref = marking.get("marking_ref", []) + + expanded.extend( + [ + {"selectors": [sel], "marking_ref": ref} + for sel in selectors + for ref in marking_ref + ] + ) + + tlo["granular_markings"] = expanded + + +def build_granular_marking(granular_marking): + tlo = {"granular_markings": [granular_marking]} + + expand_markings(tlo) + + return tlo + + +def iterpath(obj, path=None): + """ + Generator which walks the input ``obj`` model. Each iteration yields a + tuple containing a list of ancestors and the property value. + + Args: + obj: A TLO object. + path: None, used recursively to store ancestors. + + Example: + >>> for item in iterpath(tlo): + >>> print(item) + (['type'], 'campaign') + ... + (['cybox', 'objects', '[0]', 'hashes', 'sha1'], 'cac35ec206d868b7d7cb0b55f31d9425b075082b') + + Returns: + tuple: Containing two items: a list of ancestors and the property value. + + """ + if path is None: + path = [] + + for varname, varobj in iter(sorted(six.iteritems(obj))): + path.append(varname) + yield (path, varobj) + + if isinstance(varobj, dict): + + for item in iterpath(varobj, path): + yield item + + elif isinstance(varobj, list): + + for item in varobj: + index = "[{0}]".format(varobj.index(item)) + path.append(index) + + yield (path, item) + + if isinstance(item, dict): + for descendant in iterpath(item, path): + yield descendant + + path.pop() + + path.pop() + + +def get_selector(obj, prop): + """ + Function that creates a selector based on ``prop``. + + Args: + obj: A TLO object. + prop: A property of the TLO object. + + Note: + Must supply the actual value inside the structure. Since some + limitations exist with Python interning methods, checking for object + location is for now the option to assert the data. + + Example: + >>> selector = get_selector(tlo, tlo["cybox"]["objects"][0]["file_name"]) + >>> print(selector) + ["cybox.objects.[0].file_name"] + + Returns: + list: A list with one selector that asserts the supplied property. + Empty list if it was unable to find the property. + + """ + selector = [] + + for ancestors, value in iterpath(obj): + if value is prop: + path = ".".join(ancestors) + selector.append(path) + + return selector From 1f258551e1bd1aeaeb51edd0a7fe96b76d73811a Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 9 Jun 2017 14:22:56 -0400 Subject: [PATCH 04/82] Add and update tests for markings API. --- stix2/test/test_granular_markings.py | 968 +++++++++++++++++++++++++++ stix2/test/test_object_markings.py | 487 ++++++++++++++ 2 files changed, 1455 insertions(+) create mode 100644 stix2/test/test_granular_markings.py create mode 100644 stix2/test/test_object_markings.py diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py new file mode 100644 index 0000000..da53ce1 --- /dev/null +++ b/stix2/test/test_granular_markings.py @@ -0,0 +1,968 @@ + +from stix2 import markings + +import pytest + +"""Tests for the Data Markings API.""" + + +def test_add_marking_mark_one_selector_multiple_refs(): + before = { + "description": "test description", + "title": "foo", + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.add_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_add_marking_mark_multiple_selector_one_refs(): + before = { + "description": "test description", + "title": "foo", + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + ] + } + markings.add_markings(before, ["description", "title"], ["marking-definition--1"]) + assert before == after + + +def test_add_marking_mark_multiple_selector_multiple_refs(): + before = { + "description": "test description", + "title": "foo", + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.add_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_add_marking_mark_another_property_same_marking(): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + ] + } + markings.add_markings(before, ["title"], ["marking-definition--1"]) + assert before == after + + +def test_add_marking_mark_same_property_same_marking(): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + markings.add_markings(before, ["description"], ["marking-definition--1"]) + assert before == after + + +@pytest.mark.parametrize("data,marking", [ + ({"description": "test description"}, + [["title"], ["marking-definition--1", "marking-definition--2"], + "", ["marking-definition--1", "marking-definition--2"], + [], ["marking-definition--1", "marking-definition--2"], + [""], ["marking-definition--1", "marking-definition--2"], + ["description"], [""], + ["description"], [], + ["description"], ["marking-definition--1", 456] + ]) +]) +def test_add_marking_bad_selector(data, marking): + with pytest.raises(AssertionError): + markings.add_markings(data, marking[0], marking[1]) + + +GET_MARKINGS_TEST_DATA = \ +{ + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] +} + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_smoke(data): + """Test get_markings does not fail.""" + assert len(markings.get_markings(data, "a")) >= 1 + assert markings.get_markings(data, "a") == ["1"] + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_not_marked(data): + """Test selector that is not marked returns empty list.""" + results = markings.get_markings(data, "b") + assert len(results) == 0 + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_multiple_selectors(data): + """Test multiple selectors return combination of markings.""" + total = markings.get_markings(data, ["x.y", "x.z"]) + xy_markings = markings.get_markings(data, ["x.y"]) + xz_markings = markings.get_markings(data, ["x.z"]) + + assert set(xy_markings).issubset(total) + assert set(xz_markings).issubset(total) + assert set(xy_markings).union(xz_markings).issuperset(total) + + +@pytest.mark.parametrize("data,selector", [ + (GET_MARKINGS_TEST_DATA, "foo"), + (GET_MARKINGS_TEST_DATA, ""), + (GET_MARKINGS_TEST_DATA, []), + (GET_MARKINGS_TEST_DATA, [""]), + (GET_MARKINGS_TEST_DATA, "x.z.[-2]"), + (GET_MARKINGS_TEST_DATA, "c.f"), + (GET_MARKINGS_TEST_DATA, "c.[2].i"), + (GET_MARKINGS_TEST_DATA, "c.[3]"), + (GET_MARKINGS_TEST_DATA, "d"), + (GET_MARKINGS_TEST_DATA, "x.[0]"), + (GET_MARKINGS_TEST_DATA, "z.y.w"), + (GET_MARKINGS_TEST_DATA, "x.z.[1]"), + (GET_MARKINGS_TEST_DATA, "x.z.foo3") +]) +def test_get_markings_bad_selector(data, selector): + """Test bad selectors raise exception""" + with pytest.raises(AssertionError): + markings.get_markings(data, selector) + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_positional_arguments_combinations(data): + """Test multiple combinations for inherited and descendant markings.""" + assert set(markings.get_markings(data, "a", False, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, True)) == set(["1"]) + assert set(markings.get_markings(data, "a", False, True)) == set(["1"]) + + assert set(markings.get_markings(data, "b", False, False)) == set([]) + assert set(markings.get_markings(data, "b", True, False)) == set([]) + assert set(markings.get_markings(data, "b", True, True)) == set([]) + assert set(markings.get_markings(data, "b", False, True)) == set([]) + + assert set(markings.get_markings(data, "c", False, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, True)) == set(["2", "3", "4", "5"]) + assert set(markings.get_markings(data, "c", False, True)) == set(["2", "3", "4", "5"]) + + assert set(markings.get_markings(data, "c.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "c.[0]", True, False)) == set(["2"]) + assert set(markings.get_markings(data, "c.[0]", True, True)) == set(["2"]) + assert set(markings.get_markings(data, "c.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "c.[1]", False, False)) == set(["3"]) + assert set(markings.get_markings(data, "c.[1]", True, False)) == set(["2", "3"]) + assert set(markings.get_markings(data, "c.[1]", True, True)) == set(["2", "3"]) + assert set(markings.get_markings(data, "c.[1]", False, True)) == set(["3"]) + + assert set(markings.get_markings(data, "c.[2]", False, False)) == set(["4"]) + assert set(markings.get_markings(data, "c.[2]", True, False)) == set(["2", "4"]) + assert set(markings.get_markings(data, "c.[2]", True, True)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2]", False, True)) == set(["4", "5"]) + + assert set(markings.get_markings(data, "c.[2].g", False, False)) == set(["5"]) + assert set(markings.get_markings(data, "c.[2].g", True, False)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2].g", True, True)) == set(["2", "4", "5"]) + assert set(markings.get_markings(data, "c.[2].g", False, True)) == set(["5"]) + + assert set(markings.get_markings(data, "x", False, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, True)) == set(["6", "7", "8", "9", "10"]) + assert set(markings.get_markings(data, "x", False, True)) == set(["6", "7", "8", "9", "10"]) + + assert set(markings.get_markings(data, "x.y", False, False)) == set(["7"]) + assert set(markings.get_markings(data, "x.y", True, False)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y", True, True)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y", False, True)) == set(["7", "8"]) + + assert set(markings.get_markings(data, "x.y.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "x.y.[0]", True, False)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y.[0]", True, True)) == set(["6", "7"]) + assert set(markings.get_markings(data, "x.y.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.y.[1]", False, False)) == set(["8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, False)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, True)) == set(["6", "7", "8"]) + assert set(markings.get_markings(data, "x.y.[1]", False, True)) == set(["8"]) + + assert set(markings.get_markings(data, "x.z", False, False)) == set(["9"]) + assert set(markings.get_markings(data, "x.z", True, False)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z", True, True)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z", False, True)) == set(["9", "10"]) + + assert set(markings.get_markings(data, "x.z.foo1", False, False)) == set([]) + assert set(markings.get_markings(data, "x.z.foo1", True, False)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z.foo1", True, True)) == set(["6", "9"]) + assert set(markings.get_markings(data, "x.z.foo1", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.z.foo2", False, False)) == set(["10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, False)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, True)) == set(["6", "9", "10"]) + assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) + + +def test_remove_marking_remove_one_selector_with_multiple_refs(): + after = { + "description": "test description", + "title": "foo", + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.remove_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) + assert before == after + + +def test_remove_marking_remove_multiple_selector_one_ref(): + after = { + "description": "test description", + "title": "foo", + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + ] + } + markings.remove_markings(before, ["description", "title"], ["marking-definition--1"]) + assert before == after + + +def test_remove_marking_mark_one_selector_from_multiple_ones(): + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + ] + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + ] + } + markings.remove_markings(before, ["title"], ["marking-definition--1"]) + assert before == after + + +def test_remove_marking_mark_one_selector_markings_from_multiple_ones(): + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--2" + }, + ] + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.remove_markings(before, ["title"], ["marking-definition--1"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_remove_marking_mark_mutilple_selector_multiple_refs(): + after = { + "description": "test description", + "title": "foo", + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.remove_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) + assert before == after + + +def test_remove_marking_mark_another_property_same_marking(): + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + ] + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["title"], + "marking_ref": "marking-definition--1" + } + ] + } + markings.remove_markings(before, ["title"], ["marking-definition--1"]) + assert before == after + + +def test_remove_marking_mark_same_property_same_marking(): + after = { + "description": "test description", + "title": "foo", + } + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + markings.remove_markings(before, ["description"], ["marking-definition--1"]) + assert before == after + + +def test_remove_marking_bad_selector(): + before = { + "description": "test description", + } + with pytest.raises(AssertionError): + markings.remove_markings(before, ["title"], ["marking-definition--1", "marking-definition--2"]) + + +IS_MARKED_TEST_DATA = \ +{ + "title": "test title", + "description": "test description", + "revision": 2, + "type": "test", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["revision", "description"], + "marking_ref": "marking-definition--2" + }, + { + "selectors": ["revision", "description"], + "marking_ref": "marking-definition--3" + }, + ] +} + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_smoke(data): + """Smoke test is_marked call does not fail.""" + assert markings.is_marked(data, ["description"]) + assert markings.is_marked(data, ["title"]) is False + + +@pytest.mark.parametrize("data,selector", [ + (IS_MARKED_TEST_DATA, "foo"), + (IS_MARKED_TEST_DATA, ""), + (IS_MARKED_TEST_DATA, []), + (IS_MARKED_TEST_DATA, [""]), + (IS_MARKED_TEST_DATA, "x.z.[-2]"), + (IS_MARKED_TEST_DATA, "c.f"), + (IS_MARKED_TEST_DATA, "c.[2].i"), + (IS_MARKED_TEST_DATA, "c.[3]"), + (IS_MARKED_TEST_DATA, "d"), + (IS_MARKED_TEST_DATA, "x.[0]"), + (IS_MARKED_TEST_DATA, "z.y.w"), + (IS_MARKED_TEST_DATA, "x.z.[1]"), + (IS_MARKED_TEST_DATA, "x.z.foo3") +]) +def test_is_marked_invalid_selector(data, selector): + """Test invalid selector raises an error.""" + with pytest.raises(AssertionError): + markings.is_marked(data, selector) + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_mix_selector(data): + """Test valid selector, one marked and one not marked returns True.""" + assert markings.is_marked(data, ["description", "revision"]) + assert markings.is_marked(data, ["description"]) + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_valid_selector_no_refs(data): + """Test that a valid selector return True when it has marking refs and False when not.""" + assert markings.is_marked(data, ["description"]) + assert markings.is_marked(data, ["description"], ["marking-definition--2", "marking-definition--3"]) + assert markings.is_marked(data, ["description"], ["marking-definition--2"]) + assert markings.is_marked(data, ["description"], ["marking-definition--2", "marking-definition--8"]) is False + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_valid_selector_and_refs(data): + """Test that a valid selector returns True when marking_refs match.""" + assert markings.is_marked(data, ["description"], ["marking-definition--1"]) + assert markings.is_marked(data, ["title"], ["marking-definition--1"]) is False + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_valid_selector_multiple_refs(data): + """Test that a valid selector returns True if aall marking_refs match. + Otherwise False.""" + assert markings.is_marked(data, ["revision"], ["marking-definition--2", "marking-definition--3"]) + assert markings.is_marked(data, ["revision"], ["marking-definition--2", "marking-definition--1"]) is False + assert markings.is_marked(data, ["revision"], "marking-definition--2") + assert markings.is_marked(data, ["revision"], ["marking-definition--1234"]) is False + + +@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +def test_is_marked_no_marking_refs(data): + """Test that a valid content selector with no marking_refs returns True + if there is a granular_marking that asserts that field, False + otherwise.""" + assert markings.is_marked(data, ["type"]) is False + assert markings.is_marked(data, ["revision"]) + + +def test_is_marked_positional_arguments_combinations(): + """Test multiple combinations for inherited and descendant markings.""" + test_tlo = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + assert markings.is_marked(test_tlo, "a", ["1"], False, False) + assert markings.is_marked(test_tlo, "a", ["1"], True, False) + assert markings.is_marked(test_tlo, "a", ["1"], True, True) + assert markings.is_marked(test_tlo, "a", ["1"], False, True) + + assert markings.is_marked(test_tlo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "b", inherited=True, descendants=False) is False + assert markings.is_marked(test_tlo, "b", inherited=True, descendants=True) is False + assert markings.is_marked(test_tlo, "b", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "c", ["2"], False, False) + assert markings.is_marked(test_tlo, "c", ["2"], True, False) + assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], True, True) + assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], False, True) + + assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "c.[0]", ["2"], True, False) + assert markings.is_marked(test_tlo, "c.[0]", ["2"], True, True) + assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, False) + assert markings.is_marked(test_tlo, "c.[1]", ["2", "3"], True, False) + assert markings.is_marked(test_tlo, "c.[1]", ["2", "3"], True, True) + assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, True) + + assert markings.is_marked(test_tlo, "c.[2]", ["4"], False, False) + assert markings.is_marked(test_tlo, "c.[2]", ["2", "4"], True, False) + assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "5"], True, True) + assert markings.is_marked(test_tlo, "c.[2]", ["4", "5"], False, True) + + assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, False) + assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5"], True, False) + assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5"], True, True) + assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, True) + + assert markings.is_marked(test_tlo, "x", ["6"], False, False) + assert markings.is_marked(test_tlo, "x", ["6"], True, False) + assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], True, True) + assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], False, True) + + assert markings.is_marked(test_tlo, "x.y", ["7"], False, False) + assert markings.is_marked(test_tlo, "x.y", ["6", "7"], True, False) + assert markings.is_marked(test_tlo, "x.y", ["6", "7", "8"], True, True) + assert markings.is_marked(test_tlo, "x.y", ["7", "8"], False, True) + + assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7"], True, False) + assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7"], True, True) + assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, False) + assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8"], True, False) + assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8"], True, True) + assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, True) + + assert markings.is_marked(test_tlo, "x.z", ["9"], False, False) + assert markings.is_marked(test_tlo, "x.z", ["6", "9"], True, False) + assert markings.is_marked(test_tlo, "x.z", ["6", "9", "10"], True, True) + assert markings.is_marked(test_tlo, "x.z", ["9", "10"], False, True) + + assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9"], True, False) + assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9"], True, True) + assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, False) + assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10"], True, False) + assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10"], True, True) + assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, True) + + +def test_set_marking_mark_one_selector_multiple_refs(): + before = { + "description": "test description", + "title": "foo", + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.set_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_set_marking_mark_multiple_selector_one_refs(): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--3" + }, + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + ] + } + markings.set_markings(before, ["description", "title"], ["marking-definition--1"]) + assert before == after + + +def test_set_marking_mark_multiple_selector_multiple_refs_from_none(): + before = { + "description": "test description", + "title": "foo", + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["description", "title"], + "marking_ref": "marking-definition--2" + }, + ] + } + markings.set_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +def test_set_marking_mark_another_property_same_marking(): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--7" + }, + { + "selectors": ["description"], + "marking_ref": "marking-definition--8" + }, + ] + } + markings.set_markings(before, ["description"], ["marking-definition--7", "marking-definition--8"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + +@pytest.mark.parametrize("marking", [ + (["foo"], ["marking-definition--7", "marking-definition--8"]), + ("", ["marking-definition--7", "marking-definition--8"]), + ([], ["marking-definition--7", "marking-definition--8"]), + ([""], ["marking-definition--7", "marking-definition--8"]) +]) +def test_set_marking_bad_selector(marking): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + ] + } + + with pytest.raises(AssertionError): + markings.set_markings(before, marking[0], marking[1]) + + assert before == after + + +def test_set_marking_mark_same_property_same_marking(): + before = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + after = { + "description": "test description", + "title": "foo", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + } + ] + } + markings.set_markings(before, ["description"], ["marking-definition--1"]) + assert before == after + + +CLEAR_MARKINGS_TEST_DATA = \ +{ + "title": "test title", + "description": "test description", + "revision": 2, + "type": "test", + "granular_markings": [ + { + "selectors": ["description"], + "marking_ref": "marking-definition--1" + }, + { + "selectors": ["revision", "description"], + "marking_ref": "marking-definition--2" + }, + { + "selectors": ["revision", "description", "type"], + "marking_ref": "marking-definition--3" + }, + ] +} + + +@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +def test_clear_marking_smoke(data): + """Test clear_marking call does not fail.""" + markings.clear_markings(data, "revision") + assert markings.is_marked(data, "revision") is False + + +@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +def test_clear_marking_multiple_selectors(data): + """Test clearing markings for multiple selectors effectively removes associated markings.""" + markings.clear_markings(data, ["type", "description"]) + assert markings.is_marked(data, ["type", "description"]) is False + + +@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +def test_clear_marking_one_selector(data): + """Test markings associated with one selector were removed.""" + markings.clear_markings(data, "description") + assert markings.is_marked(data, "description") is False + + +@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +def test_clear_marking_all_selectors(data): + markings.clear_markings(data, ["description", "type", "revision"]) + assert markings.is_marked(data, "description") is False + assert "granular_markings" not in data + + +@pytest.mark.parametrize("data,selector", [ + (CLEAR_MARKINGS_TEST_DATA, "foo"), + (CLEAR_MARKINGS_TEST_DATA, ""), + (CLEAR_MARKINGS_TEST_DATA, []), + (CLEAR_MARKINGS_TEST_DATA, [""]), +]) +def test_clear_marking_bad_selector(data, selector): + """Test bad selector raises exception.""" + with pytest.raises(AssertionError): + markings.clear_markings(data, selector) diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py new file mode 100644 index 0000000..5205ad7 --- /dev/null +++ b/stix2/test/test_object_markings.py @@ -0,0 +1,487 @@ + +from stix2 import markings + +import pytest + +"""Tests for the Data Markings API.""" + + +def test_add_markings_one_marking(): + before = { + "title": "test title", + "description": "test description" + } + + after = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1"] + } + + markings.add_markings(before, None, "marking-definition--1") + + assert before == after + + +def test_add_markings_multiple_marking(): + before = { + "title": "test title", + "description": "test description" + } + + after = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2"] + } + + markings.add_markings(before, None, ["marking-definition--1", "marking-definition--2"]) + + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] + + +def test_add_markings_combination(): + before = { + "title": "test title", + "description": "test description" + } + + after = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2"], + "granular_markings": [ + { + "selectors": ["title"], + "marking_ref": "marking-definition--3" + }, + { + "selectors": ["description"], + "marking_ref": "marking-definition--4" + }, + ] + } + + markings.add_markings(before, None, "marking-definition--1") + markings.add_markings(before, None, "marking-definition--2") + markings.add_markings(before, "title", "marking-definition--3") + markings.add_markings(before, "description", "marking-definition--4") + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] + + +@pytest.mark.parametrize("data", [ + ([""]), + (""), + ([]), + (["marking-definition--1", 456]) +]) +def test_add_markings_bad_markings(data): + before = { + "title": "test title", + "description": "test description" + } + with pytest.raises(AssertionError): + markings.add_markings(before, None, data) + + assert "object_marking_refs" not in before + + +GET_MARKINGS_TEST_DATA = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "object_marking_refs": ["11"], + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_object_marking(data): + assert set(markings.get_markings(data, None)) == set(["11"]) + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +def test_get_markings_object_and_granular_combinations(data): + """Test multiple combinations for inherited and descendant markings.""" + assert set(markings.get_markings(data, "a", False, False)) == set(["1"]) + assert set(markings.get_markings(data, "a", True, False)) == set(["1", "11"]) + assert set(markings.get_markings(data, "a", True, True)) == set(["1", "11"]) + assert set(markings.get_markings(data, "a", False, True)) == set(["1"]) + + assert set(markings.get_markings(data, "b", False, False)) == set([]) + assert set(markings.get_markings(data, "b", True, False)) == set(["11"]) + assert set(markings.get_markings(data, "b", True, True)) == set(["11"]) + assert set(markings.get_markings(data, "b", False, True)) == set([]) + + assert set(markings.get_markings(data, "c", False, False)) == set(["2"]) + assert set(markings.get_markings(data, "c", True, False)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c", True, True)) == set(["2", "3", "4", "5", "11"]) + assert set(markings.get_markings(data, "c", False, True)) == set(["2", "3", "4", "5"]) + + assert set(markings.get_markings(data, "c.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "c.[0]", True, False)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c.[0]", True, True)) == set(["2", "11"]) + assert set(markings.get_markings(data, "c.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "c.[1]", False, False)) == set(["3"]) + assert set(markings.get_markings(data, "c.[1]", True, False)) == set(["2", "3", "11"]) + assert set(markings.get_markings(data, "c.[1]", True, True)) == set(["2", "3", "11"]) + assert set(markings.get_markings(data, "c.[1]", False, True)) == set(["3"]) + + assert set(markings.get_markings(data, "c.[2]", False, False)) == set(["4"]) + assert set(markings.get_markings(data, "c.[2]", True, False)) == set(["2", "4", "11"]) + assert set(markings.get_markings(data, "c.[2]", True, True)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2]", False, True)) == set(["4", "5"]) + + assert set(markings.get_markings(data, "c.[2].g", False, False)) == set(["5"]) + assert set(markings.get_markings(data, "c.[2].g", True, False)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2].g", True, True)) == set(["2", "4", "5", "11"]) + assert set(markings.get_markings(data, "c.[2].g", False, True)) == set(["5"]) + + assert set(markings.get_markings(data, "x", False, False)) == set(["6"]) + assert set(markings.get_markings(data, "x", True, False)) == set(["6", "11"]) + assert set(markings.get_markings(data, "x", True, True)) == set(["6", "7", "8", "9", "10", "11"]) + assert set(markings.get_markings(data, "x", False, True)) == set(["6", "7", "8", "9", "10"]) + + assert set(markings.get_markings(data, "x.y", False, False)) == set(["7"]) + assert set(markings.get_markings(data, "x.y", True, False)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y", True, True)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y", False, True)) == set(["7", "8"]) + + assert set(markings.get_markings(data, "x.y.[0]", False, False)) == set([]) + assert set(markings.get_markings(data, "x.y.[0]", True, False)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y.[0]", True, True)) == set(["6", "7", "11"]) + assert set(markings.get_markings(data, "x.y.[0]", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.y.[1]", False, False)) == set(["8"]) + assert set(markings.get_markings(data, "x.y.[1]", True, False)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y.[1]", True, True)) == set(["6", "7", "8", "11"]) + assert set(markings.get_markings(data, "x.y.[1]", False, True)) == set(["8"]) + + assert set(markings.get_markings(data, "x.z", False, False)) == set(["9"]) + assert set(markings.get_markings(data, "x.z", True, False)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z", True, True)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z", False, True)) == set(["9", "10"]) + + assert set(markings.get_markings(data, "x.z.foo1", False, False)) == set([]) + assert set(markings.get_markings(data, "x.z.foo1", True, False)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z.foo1", True, True)) == set(["6", "9", "11"]) + assert set(markings.get_markings(data, "x.z.foo1", False, True)) == set([]) + + assert set(markings.get_markings(data, "x.z.foo2", False, False)) == set(["10"]) + assert set(markings.get_markings(data, "x.z.foo2", True, False)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z.foo2", True, True)) == set(["6", "9", "10", "11"]) + assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) + + +def test_remove_markings_object_level(): + after = { + "title": "test title", + "description": "test description" + } + + before = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1"] + } + + markings.remove_markings(before, None, "marking-definition--1") + + assert before == after + + +def test_remove_markings_multiple(): + after = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--2"] + } + + before = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] + } + + markings.remove_markings(before, None, ["marking-definition--1", "marking-definition--3"]) + + assert before == after + + +def test_remove_markings_bad_markings(): + before = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] + } + with pytest.raises(AssertionError): + markings.remove_markings(before, None, ["marking-definition--5"]) + + +def test_clear_markings(): + after = { + "title": "test title", + "description": "test description" + } + + before = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] + } + + markings.clear_markings(before, None) + + assert before == after + + +def test_is_marked_object_and_granular_combinations(): + """Test multiple combinations for inherited and descendant markings.""" + test_tlo = \ + { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45 + } + ], + "x": { + "y": [ + "hello", + 88 + ], + "z": { + "foo1": "bar", + "foo2": 65 + } + }, + "object_marking_refs": "11", + "granular_markings": [ + { + "marking_ref": "1", + "selectors": ["a"] + }, + { + "marking_ref": "2", + "selectors": ["c"] + }, + { + "marking_ref": "3", + "selectors": ["c.[1]"] + }, + { + "marking_ref": "4", + "selectors": ["c.[2]"] + }, + { + "marking_ref": "5", + "selectors": ["c.[2].g"] + }, + { + "marking_ref": "6", + "selectors": ["x"] + }, + { + "marking_ref": "7", + "selectors": ["x.y"] + }, + { + "marking_ref": "8", + "selectors": ["x.y.[1]"] + }, + { + "marking_ref": "9", + "selectors": ["x.z"] + }, + { + "marking_ref": "10", + "selectors": ["x.z.foo2"] + }, + ] + } + + assert markings.is_marked(test_tlo, "a", ["1"], False, False) + assert markings.is_marked(test_tlo, "a", ["1", "11"], True, False) + assert markings.is_marked(test_tlo, "a", ["1", "11"], True, True) + assert markings.is_marked(test_tlo, "a", ["1"], False, True) + + assert markings.is_marked(test_tlo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "b", ["11"], True, False) + assert markings.is_marked(test_tlo, "b", ["11"], True, True) + assert markings.is_marked(test_tlo, "b", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "c", ["2"], False, False) + assert markings.is_marked(test_tlo, "c", ["2", "11"], True, False) + assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5", "11"], True, True) + assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], False, True) + + assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "c.[0]", ["2", "11"], True, False) + assert markings.is_marked(test_tlo, "c.[0]", ["2", "11"], True, True) + assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, False) + assert markings.is_marked(test_tlo, "c.[1]", ["2", "3", "11"], True, False) + assert markings.is_marked(test_tlo, "c.[1]", ["2", "3", "11"], True, True) + assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, True) + + assert markings.is_marked(test_tlo, "c.[2]", ["4"], False, False) + assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "11"], True, False) + assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "5", "11"], True, True) + assert markings.is_marked(test_tlo, "c.[2]", ["4", "5"], False, True) + + assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, False) + assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5", "11"], True, False) + assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5", "11"], True, True) + assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, True) + + assert markings.is_marked(test_tlo, "x", ["6"], False, False) + assert markings.is_marked(test_tlo, "x", ["6", "11"], True, False) + assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10", "11"], True, True) + assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], False, True) + + assert markings.is_marked(test_tlo, "x.y", ["7"], False, False) + assert markings.is_marked(test_tlo, "x.y", ["6", "7", "11"], True, False) + assert markings.is_marked(test_tlo, "x.y", ["6", "7", "8", "11"], True, True) + assert markings.is_marked(test_tlo, "x.y", ["7", "8"], False, True) + + assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7", "11"], True, False) + assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7", "11"], True, True) + assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, False) + assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8", "11"], True, False) + assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8", "11"], True, True) + assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, True) + + assert markings.is_marked(test_tlo, "x.z", ["9"], False, False) + assert markings.is_marked(test_tlo, "x.z", ["6", "9", "11"], True, False) + assert markings.is_marked(test_tlo, "x.z", ["6", "9", "10", "11"], True, True) + assert markings.is_marked(test_tlo, "x.z", ["9", "10"], False, True) + + assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9", "11"], True, False) + assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9", "11"], True, True) + assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=True) is False + + assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, False) + assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10", "11"], True, False) + assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10", "11"], True, True) + assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, True) + + +def test_set_marking(): + before = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] + } + after = { + "title": "test title", + "description": "test description", + "object_marking_refs": ["marking-definition--7", "marking-definition--9"] + } + + markings.set_markings(before, None, ["marking-definition--7", "marking-definition--9"]) + + for m in before["object_marking_refs"]: + assert m in ["marking-definition--7", "marking-definition--9"] + + assert ["marking-definition--1", "marking-definition--2", "marking-definition--3"] not in before["object_marking_refs"] + + for x in before["object_marking_refs"]: + assert x in after["object_marking_refs"] + + +@pytest.mark.parametrize("data", [ + ([]), + ([""]), + (""), + (["marking-definition--7", 687]) +]) +def test_set_marking_bad_input(data): + before = { + "description": "test description", + "title": "foo", + "object_marking_refs": ["marking-definition--1"] + } + after = { + "description": "test description", + "title": "foo", + "object_marking_refs": ["marking-definition--1"] + } + with pytest.raises(AssertionError): + markings.set_markings(before, None, data) + + assert before == after From ee4618f6c831d1a53420726e08925e1cd1661269 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 12 Jun 2017 08:06:13 -0400 Subject: [PATCH 05/82] Add new marking errors. --- stix2/exceptions.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 3043047..7fa76c4 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -150,3 +150,29 @@ class RevokeError(STIXError, ValueError): return "Cannot revoke an already revoked object." else: return "Cannot create a new version of a revoked object." + + +class InvalidSelectorError(STIXError, ValueError): + """Granular Marking selector violation. The selector must resolve into an existing STIX object property.""" + + def __init__(self, cls, key): + super(InvalidSelectorError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Selector '{0}' in '{1}' is not valid!" + return msg.format(self.key, self.__class__.__name__) + + +class InvalidMarkingError(STIXError, ValueError): + """Marking violation. The marking reference must be a valid identifier.""" + + def __init__(self, cls, key): + super(InvalidMarkingError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Marking '{0}' in '{1}' is not a valid marking reference." + return msg.format(self.key, self.__class__.__name__) From bf740b21eb1b91bd268515f52d7d779e007e1db2 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 12 Jun 2017 08:06:37 -0400 Subject: [PATCH 06/82] Initial selector and marking_ref validation. --- stix2/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index c40c9e6..d78c801 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -11,6 +11,7 @@ from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, MissingPropertiesError, MutuallyExclusivePropertiesError, RevokeError, UnmodifiablePropertyError) +from .markings.utils import validate from .utils import NOW, format_datetime, get_timestamp, parse_into_datetime __all__ = ['STIXJSONEncoder', '_STIXBase'] @@ -80,8 +81,7 @@ class _STIXBase(collections.Mapping): def _check_object_constraints(self): for m in self.get("granular_markings", []): - # TODO: check selectors - pass + validate(self, m.get("selectors"), m.get("marking_ref")) def __init__(self, **kwargs): cls = self.__class__ From 7abcce7635c99e7d581fd05f51b1a9961ef19851 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 13 Jul 2017 07:55:52 -0400 Subject: [PATCH 07/82] Add new duplicate marking exception. --- stix2/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 7fa76c4..7e56117 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -165,11 +165,11 @@ class InvalidSelectorError(STIXError, ValueError): return msg.format(self.key, self.__class__.__name__) -class InvalidMarkingError(STIXError, ValueError): +class DuplicateMarkingError(STIXError, ValueError): """Marking violation. The marking reference must be a valid identifier.""" def __init__(self, cls, key): - super(InvalidMarkingError, self).__init__() + super(DuplicateMarkingError, self).__init__() self.cls = cls self.key = key From 747f0307a799885f9384aefcb8fab4a0a6d6107b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 13 Jul 2017 07:57:01 -0400 Subject: [PATCH 08/82] [WIP] Changes to align python-stix2 with marking-prototype project. --- stix2/markings/__init__.py | 4 +- stix2/markings/object_markings.py | 82 +++++++++++-------------------- 2 files changed, 32 insertions(+), 54 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 8301131..14d21bb 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -116,9 +116,9 @@ def add_markings(obj, selectors, marking): """ if selectors is None: - object_markings.add_markings(obj, marking) + return object_markings.add_markings(obj, marking) else: - granular_markings.add_markings(obj, selectors, marking) + return granular_markings.add_markings(obj, selectors, marking) def clear_markings(obj, selectors): diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index 33a8d2a..9d2b2bc 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -1,28 +1,20 @@ -import six - from stix2.markings import utils def get_markings(obj): """ - Get all object level markings from the given TLO object. + Get all object level markings from the given SDO or SRO object. Args: - obj: A TLO object. + obj: A SDO or SRO object. Returns: - list: Marking IDs contained in the TLO. + list: Marking IDs contained in the SDO or SRO. Empty list if no + markings are present in `object_marking_refs`. """ - object_markings = obj.get("object_marking_refs", []) - - if not object_markings: - return [] - elif isinstance(object_markings, six.string_types): - return [object_markings] - else: - return object_markings + return obj.get("object_marking_refs", []) def add_markings(obj, marking): @@ -30,23 +22,19 @@ def add_markings(obj, marking): Appends an object level marking to the object_marking_refs collection. Args: - obj: A TLO object. - marking: identifier or list of marking identifiers that apply to the - TLO object. + obj: A SDO or SRO object. + marking: identifier or list of identifiers to apply SDO or SRO object. Raises: AssertionError: If `marking` fail data validation. """ marking = utils.convert_to_list(marking) - utils.validate(obj, marking=marking) - if not obj.get("object_marking_refs"): - obj["object_marking_refs"] = list() + # TODO: Remove set for comparison and raise DuplicateMarkingException. + object_markings = set(obj.get("object_marking_refs", []) + marking) - object_markings = set(obj.get("object_marking_refs") + marking) - - obj["object_marking_refs"] = list(object_markings) + return obj.new_version(object_marking_refs=list(object_markings)) def remove_markings(obj, marking): @@ -54,13 +42,13 @@ def remove_markings(obj, marking): Removes object level marking from the object_marking_refs collection. Args: - obj: A TLO object. - marking: identifier or list of marking identifiers that apply to the - TLO object. + obj: A SDO or SRO object. + marking: identifier or list of identifiers that apply to the + SDO or SRO object. Raises: - AssertionError: If `marking` fail data validation. Also - if markings to remove are not found on the provided TLO. + AssertionError: If markings to remove are not found on the provided + SDO or SRO. """ marking = utils.convert_to_list(marking) @@ -69,17 +57,14 @@ def remove_markings(obj, marking): object_markings = obj.get("object_marking_refs", []) if not object_markings: - return [] + return obj if any(x not in obj["object_marking_refs"] for x in marking): raise AssertionError("Unable to remove Object Level Marking(s) from " "internal collection. Marking(s) not found...") - obj["object_marking_refs"] = [x for x in object_markings - if x not in marking] - - if not obj.get("object_marking_refs"): - obj.pop("object_marking_refs") + return obj.new_version(object_marking_refs=[x for x in object_markings + if x not in marking]) def set_markings(obj, marking): @@ -88,15 +73,12 @@ def set_markings(obj, marking): the collection. Refer to `clear_markings` and `add_markings` for details. Args: - obj: A TLO object. - marking: identifier or list of marking identifiers that apply to the - TLO object. + obj: A SDO or SRO object. + marking: identifier or list of identifiers to apply in the + SDO or SRO object. """ - utils.validate(obj, marking=marking) - - clear_markings(obj) - add_markings(obj, marking) + return add_markings(clear_markings(obj), marking) def clear_markings(obj): @@ -104,31 +86,27 @@ def clear_markings(obj): Removes all object level markings from the object_marking_refs collection. Args: - obj: A TLO object. + obj: A SDO or SRO object. """ - try: - del obj["object_marking_refs"] - except KeyError: - raise AssertionError("Unable to clear Object Marking(s) from internal" - " collection. No Markings in object...") + return obj.new_version(object_marking_refs=None) def is_marked(obj, marking=None): """ - Checks if TLO is marked by any marking or by specific marking(s). + Checks if SDO or SRO is marked by any marking or by specific marking(s). Args: - obj: A TLO object. + obj: A SDO or SRO object. marking: identifier or list of marking identifiers that apply to the - TLO object. + SDO or SRO object. Returns: - bool: True if TLO has object level markings. False otherwise. + bool: True if SDO or SRO has object level markings. False otherwise. Note: - When a list of marking IDs is provided, if ANY of the provided marking - IDs matches, True is returned. + When an identifier or list of identifiers is provided, if ANY of the + provided marking refs match, True is returned. """ marking = utils.convert_to_list(marking) From 6d2cfcdedf81741be64505227c8b7965a481e429 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 13 Jul 2017 07:57:33 -0400 Subject: [PATCH 09/82] [WIP] Update tests. --- stix2/test/test_markings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index f1f07db..58a0864 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -5,10 +5,10 @@ import pytz import stix2 from stix2.other import TLP_WHITE +from stix2 import markings from .constants import MARKING_DEFINITION_ID - EXPECTED_TLP_MARKING_DEFINITION = """{ "created": "2017-01-20T00:00:00Z", "definition": { @@ -119,7 +119,8 @@ def test_campaign_with_granular_markings_example(): marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", selectors=["description"]) ]) - print(str(campaign)) + print (markings.get_markings(campaign, None)) + print (markings.add_markings(campaign, None, "marking-definition--00000000-0000-0000-0000-000000000000")) assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS From 20958b908a5c103bf8d691cefe3102986737d7c1 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 31 May 2017 11:44:24 -0400 Subject: [PATCH 10/82] more tests for TAXII data source --- stix2/test/test_data_sources.py | 174 +++++++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 1 deletion(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0b57729..8efc868 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -61,8 +61,8 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params - def test_add_get_remove_filter(): + class dummy(object): x = 4 @@ -130,6 +130,178 @@ def test_add_get_remove_filter(): assert statuses[5]['errors'][0] == expected_errors[2] assert statuses[6]['errors'][0] == expected_errors[3] + #get + ds_filters = ds.get_filters() + + for idx,flt in enumerate(filters): + assert flt['value'] == filters[idx]['value'] + + #remove + ds.remove_filter([ids[3]]) + ds.remove_filter([ids[4]]) + ds.remove_filter([ids[5]]) + ds.remove_filter([ids[6]]) + + rem_filters = ds.get_filters() + + assert len(rem_filters) == 3 + + #check remaining filters + rem_ids = [f['id'] for f in rem_filters] + + #check remaining + for id_ in rem_ids: + assert id_ in ids[:3] + +def test_apply_common_filters(): + stix_objs = [ + { + "created": "2017-01-27T13:49:53.997Z", + "description": "\n\nTITLE:\n\tPoison Ivy", + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "labels": [ + "remote-access-trojan" + ], + "modified": "2017-01-27T13:49:53.997Z", + "name": "Poison Ivy", + "type": "malware" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "labels": [ + "file-hash-watchlist" + ], + "modified": "2014-05-08T09:00:00.000Z", + "name": "File hash for Poison Ivy variant", + "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "type": "indicator", + "valid_from": "2014-05-08T09:00:00.000000Z" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", + "modified": "2014-05-08T09:00:00.000Z", + "relationship_type": "indicates", + "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "type": "relationship" + } + ] + + filters = [ + { + "field": "type", + "op": "!=", + "value": "relationship" + }, + { + "field": "id", + "op": "=", + "value": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" + }, + { + "field": "labels", + "op": "in", + "value": "trojan" + } + ] + + ds = taxii.TAXIIDataSource() + + resp = ds.apply_common_filters(stix_objs, [filters[0]]) + ids = [r['id'] for r in resp] + assert stix_objs[0]['id'] in ids + assert stix_objs[1]['id'] in ids + + resp = ds.apply_common_filters(stix_objs, [filters[1]]) + assert resp[0]['id'] == stix_objs[2]['id'] + + resp = ds.apply_common_filters(stix_objs, [filters[2]]) + assert resp[0]['id'] == stix_objs[0]['id'] + +def test_deduplicate(): + stix_objs = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ] + + ds = taxii.TAXIIDataSource() + unique = ds.deduplicate(stix_objs) + + #Only 3 objects are unique + #2 id's vary + #2 modified times vary for a particular id + + assert len(unique) == 3 + + ids = [obj['id'] for obj in unique] + mods = [obj['modified'] for obj in unique] + + assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids + assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids + assert "2017-01-27T13:49:53.935Z" in mods + assert "2017-01-27T13:49:53.936Z" in mods + + # def test_data_source_file(): # ds = file.FileDataSource() From 9d72d6070627eb3b1bc81366f5938cdde4b1171d Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Wed, 12 Jul 2017 10:58:31 -0400 Subject: [PATCH 11/82] Data* suite reorg, fixing bugs --- stix2/sources/__init__.py | 1028 +++++++++++++++++++++-------------- stix2/sources/filesystem.py | 256 +++++++++ stix2/sources/memory.py | 268 +++++++++ stix2/sources/taxii.py | 430 ++++++++------- 4 files changed, 1380 insertions(+), 602 deletions(-) create mode 100644 stix2/sources/filesystem.py create mode 100644 stix2/sources/memory.py diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index f30e815..8f252f7 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -1,18 +1,19 @@ """ -Python STIX 2.0 Composite Data Source and Data Source (classes) +Python STIX 2.0 Sources +Classes: + DataStore + DataSink + DataSource + STIXCommonPropertyFilters ----TODO/Questions--- - - -Test everything - - -add_filter(), remove_filter(), deduplicate() - if these functions remain - the exact same for both CompositeDataSource and DataSource, they just - inherit/have module access to +TODO:Test everything +NOTE: add_filter(), remove_filter(), deduplicate() - if these functions remain + the exact same for DataSource, DataSink, CompositeDataSource etc... -> just + make those functions an interface to inherit? """ -import abc import copy import uuid @@ -29,18 +30,22 @@ STIX_VERSION_FIELDS = ['id', 'modified'] # Currently, only STIX 2.0 common SDO fields (that are not compex objects) # are supported for filtering on STIX_COMMON_FIELDS = [ - 'type', - 'id', - 'created_by_ref', - 'created', - 'modified', - 'revoked', - 'labels', - # 'external_references', # list of external references object type - not supported for filtering - 'object_references', - 'object_marking_refs', - 'granular_marking_refs', - # 'granular_markings' # list of granular-marking type - not supported for filtering + "created", + "created_by_ref", + "external_references.source_name", + "external_references.description", + "external_references.url", + "external_references.hashes", + "external_references.external_id", + "granular_markings.marking_ref", + "granular_markings.selectors", + "id", + "labels", + "modified", + "object_marking_refs", + "revoked", + "type", + "granular_markings" ] @@ -54,350 +59,139 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] -class CompositeDataSource(object): - """Composite Data Source +class DataStore(object): + """ + An implementer will create a concrete subclass from + this abstract class for the specific data store. + """ + def __init__(self, name="DataStore", source=None, sink=None): + self.name = name + self.id = make_id() + if source: + self.source = source + else: + self.source = DataSource() + if sink: + self.sink = sink + else: + self.sink = DataSink() - Acts as a controller for all the defined/configured STIX Data Sources - e.g. a user can defined n Data Sources - creating Data Source (objects) - for each. There is only one instance of this for any python STIX 2.0 - application + @property + def source(self): + return self.source + @source.setter + def source(self, source): + self.source = source + + @property + def sink(self): + return self.sink + + @sink.setter + def sink(self, sink): + self.sink = sink + + def get(self, stix_id): + """ + Implement: + -translate API get() call to the appropriate DataSource call + + Args: + + stix_id (str): the id of the STIX 2.0 object to retrieve. Should + return a single object, the most recent version of the object + specified by the "id". + + _composite_filters (list): list of filters passed along from + the Composite Data Filter. + + Returns: + stix_obj (dictionary): the STIX object to be returned + """ + return self.source.get(stix_id=stix_id) + + def all_versions(self, stix_id): + """ + Implement: + -translate all_versions() call to the appropriate DataSource call + + Args: + + stix_id (str): the id of the STIX 2.0 object to retrieve. Should + return a single object, the most recent version of the object + specified by the "id". + + _composite_filters (list): list of filters passed along from + the Composite Data Filter. + + Returns: + stix_objs (list): a list of STIX objects (where each object is a + STIX object) + """ + return self.source.all_versions(stix_id=stix_id) + + def query(self, query): + """ + Fill: + -implement the specific data source API calls, processing, + functionality required for retrieving query from the data source + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on + + _composite_filters (list): a list of filters passed from the + Composite Data Source + + Returns: + stix_objs (list): a list of STIX objects (where each object is a + STIX object) + + """ + return self.source.query(query=query) + + def add(self, stix_objs): + """ + Fill: + -translate add() to the appropriate DataSink call() + + """ + return self.sink.add(stix_objs=stix_objs) + + +class DataSink(object): + """ + An implementer will create a concrete subclass from this + abstract class for the specific data sink. """ - def __init__(self, name="CompositeDataSource"): - """ - Creates a new STIX Data Source. - - Args: - 'data_sources' (dict): a dict of DataSource objects; to be - controlled and used by the Data Source Controller object - - filters : - name : - """ - self.id_ = make_id() + def __init__(self, name="DataSink"): self.name = name - self.data_sources = {} - self.filters = {} - self.filter_allowed = {} - - def get(self, id_): - """Retrieve STIX object by 'id' - - federated retrieve method-iterates through all STIX data sources - defined in the "data_sources" parameter. Each data source has a - specific API retrieve-like function and associated parameters. This - function does a federated retrieval and consolidation of the data - returned from all the STIX data sources. - - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle - - Args: - id_ (str): the id of the STIX object to retrieve - - Returns: - stix_obj (dict): the STIX object to be returned + self.id = make_id() + def add(self, stix_objs): """ - - all_data = [] - - # for every configured Data Source, call its retrieve handler - for ds_id, ds in iteritems(self.data_sources): - data = ds.get(id_=id_, _composite_filters=self.filters.values()) - all_data += data - - # remove duplicate versions - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - - return stix_obj - - def all_versions(self, id_): - """Retrieve STIX objects by 'id' - - Federated all_versions retrieve method - iterates through all STIX data - sources defined in "data_sources" - - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle - - Args: - id_ (str): id of the STIX objects to retrieve - - Returns: - all_data (list): list of STIX objects that have the specified id + Fill: + -implement the specific data sink API calls, processing, + functionality required for adding data to the sink """ - all_data = [] - - # retrieve STIX objects from all configured data sources - for ds_id, ds in iteritems(self.data_sources): - data = ds.all_versions(id_=id_, _composite_filters=self.filters.values()) - all_data += data - - # remove exact duplicates (where duplicates are STIX 2.0 objects - # with the same 'id' and 'modified' values) - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - return all_data - - def query(self, query=None): - """composite data source query - - Federate the query to all Data Sources attached - to the Composite Data Source - - Args: - query (list): list of filters to search on - - Returns: - all_data (list): list of STIX objects to be returned - - """ - if not query: - query = [] - - all_data = [] - - # federate query to all attached data sources, - # pass composite filters to them - for ds_id, ds in iteritems(self.data_sources): - data = ds.query(query=query, _composite_filters=self.filters.values()) - all_data += data - - # remove exact duplicates (where duplicates are STIX 2.0 - # objects with the same 'id' and 'modified' values) - if len(all_data) > 0: - all_data = self.deduplicate(all_data) - - return all_data - - def add_data_source(self, data_sources): - """add/attach Data Source to the Composite Data Source instance - - Args: - data_sources (list): a list of Data Source objects to attach - to the Composite Data Source - - Returns: - - """ - - for ds in data_sources: - if issubclass(ds, DataSource): - if self.data_sources[ds['id']] in self.data_sources.keys(): - # data source already attached to Composite Data Source - continue - - # add data source to Composite Data Source - # (its id will be its key identifier) - self.data_sources[ds['id']] = ds - else: - # the Data Source object is not a proper subclass - # of DataSource Abstract Class - # TODO: maybe log error? - continue - - return - - def remove_data_source(self, data_source_ids): - """remove/detach Data Source from the Composite Data Source instance - - Args: - data_source_ids (list): a list of Data Source - id's(which are strings) - - Returns: - - """ - - for id_ in data_source_ids: - try: - if self.data_sources[id_]: - del self.data_sources[id_] - except KeyError: - # Data Source 'id' was not found in CompositeDataSource's - # list of data sources - pass - return - - def get_data_sources(self): - """return all attached Data Sources - - TODO: Make this a property? - - Args: - - Returns: - - """ - return copy.deepcopy(self.data_sources.values()) - - def add_filter(self, filters): - """add/attach a filter to the Composite Data Source instance - - Args: - filters (list): list of filters (dict) to add to the Data Source - - Returns: - status (list): list of status/error messages - - """ - - status = [] - errors = [] - ids = [] - allowed = True - - for filter_ in filters: - # check required filter components ("field", "op", "value") exist - for field in FILTER_FIELDS: - if field not in filter_.keys(): - allowed = False - errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") - break - - if allowed: - # no need for further checks if filter is missing parameters - - # check filter field is a supported STIX 2.0 common field - if filter_['field'] not in STIX_COMMON_FIELDS: - allowed = False - errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - - # check filter operator is supported - if filter_['op'] not in FILTER_OPS: - allowed = False - errors.append("Filter operation(from 'op' field) not supported") - - # check filter value type is supported - if type(filter_['value']) not in FILTER_VALUE_TYPES: - allowed = False - errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - - # Filter is added regardless of whether it fits requirements - # to be a common filter. This is done because some filters - # may be added and used by third party Data Sources, where the - # filtering may be conducted within those plugins, just not here - - id_ = make_id() - filter_['id'] = id_ - self.filters['id_'] = filter_ - ids.append(id_) - - if allowed: - self.filter_allowed[id_] = True - status.append({ - "status": "added as a common filter", - "filter": filter_, - "data_source_name": self.name, - "data_source_id": self.id_ - }) - else: - self.filter_allowed[id_] = False - status.append({ - "status": "added but is not a common filter", - "filter": filter_, - "errors": errors, - "data_source_name": self.name, - "data_source_id": self.id_ - }) - del errors[:] - - allowed = True - - return ids, status - - def remove_filter(self, filter_ids): - """Remove/detach a filter from the Data Source instance - - Args: - filter_ids (list): list of filter id's (which are strings) - dettach from the Composite Data Source - - Returns: - - """ - - for filter_id in filter_ids: - try: - if filter_id in self.filters: - del self.filters[filter_id] - del self.filter_allowed[filter_id] - except KeyError: - # filter id not found in list of filters - # attached to the Composite Data Source - pass - - return - - def get_filters(self): - """return filters attached to Composite Data Source - - Args: - - Returns: - (list): the list of filters currently attached to the Data Source - - """ - return copy.deepcopy(list(self.filters.values())) - - def deduplicate(self, stix_obj_list): - """deduplicate a list fo STIX objects to a unique set - - Reduces a set of STIX objects to unique set by looking - at 'id' and 'modified' fields - as a unique object version - is determined by the combination of those fields - - Args: - stix_obj_list (list): list of STIX objects (dicts) - - Returns: - (list): unique set of the passed list of STIX objects - """ - - unique = [] - dont_have = False - for i in stix_obj_list: - dont_have = False - for j in unique: - for field in STIX_VERSION_FIELDS: - if not i[field] == j[field]: - dont_have = True - break - if dont_have: - unique.append(i) - return unique + raise NotImplementedError() class DataSource(object): """ - Abstract Data Source class for STIX 2.0 - An implementer will create a concrete subclass from this abstract class for the specific data source. - - The purpose of the concrete subclasses is to then - supply them to a Composite Data Source which calls - the subclass methods when conducting STIX 2.0 - data retrievals. """ - __metaclass__ = abc.ABCMeta - def __init__(self, name="DataSource"): self.name = name - self.id_ = make_id() + self.id = make_id() self.filters = {} self.filter_allowed = {} - @abc.abstractmethod - def get(self, id_, _composite_filters=None): + def get(self, stix_id, _composite_filters=None): """ Fill: -implement the specific data source API calls, processing, @@ -405,7 +199,7 @@ class DataSource(object): Args: - id_ (str): the id of the STIX 2.0 object to retrieve. Should + stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". @@ -418,8 +212,7 @@ class DataSource(object): """ raise NotImplementedError() - @abc.abstractmethod - def all_versions(self, id_, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None): """ Fill: -Similar to get() except returns list of all object versions of @@ -429,7 +222,7 @@ class DataSource(object): functionality required for retrieving data from the data source Args: - id_ (str): The id of the STIX 2.0 object to retrieve. Should + id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". @@ -440,11 +233,9 @@ class DataSource(object): stix_objs (list): a list of STIX objects (where each object is a STIX object) """ - stix_objs = [] - return stix_objs + raise NotImplementedError() - @abc.abstractmethod def query(self, query, _composite_filters=None): """ Fill: @@ -462,24 +253,8 @@ class DataSource(object): """ - stix_objs = [] - return stix_objs - - @abc.abstractmethod - def close(self): - """ - Fill: - Close, release, shutdown any objects, contexts, variables - Args: - - Returns: - (list): list of status/error messages - """ - - status = [] - - return status + raise NotImplementedError() def add_filter(self, filters): """add/attach a filter to the Data Source instance @@ -506,7 +281,7 @@ class DataSource(object): break if allowed: - # no reason for further checks if missing filter parameters + # no need for further checks if filter is missing parameters # check filter field is a supported STIX 2.0 common field if filter_['field'] not in STIX_COMMON_FIELDS: @@ -621,61 +396,23 @@ class DataSource(object): # due to STIX object, STIX object is discarded (i.e. did not # make it through the filter) if filter_['field'] not in stix_obj.keys(): + clean = False break - try: + match = getattr(STIXCommonPropertyFilters, filter_['field'])(filter_, stix_obj) + if not match: + clean = False + break + elif match == -1: + # error, filter operator not supported for specified field: + pass + except Exception as e: + print(e) - if filter_['op'] == '=': - if not stix_obj[filter_['field']] == filter_['value']: - clean = False - break - elif filter_['op'] == "!=": - if not stix_obj[filter_['field']] != filter_['value']: - clean = False - break - elif filter_['op'] == "in": - if not stix_obj[filter_['field']] in filter_['value']: - clean = False - break - else: - # filter operation not supported - continue - - # TODO: I think the rest of the operations only - # apply to timestamps, in which case I don't think - # simple operator usage (like below) works - - # elif filter_['op'] == ">": - # if not stix_obj[filter_['field']] > filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == "<": - # if not stix_obj[filter_['field']] < filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == ">=": - # if not stix_obj[filter_['field']] >= filter_['value']: - # clean = False - # break - # - # elif filter_['op'] == "<=": - # if not stix_obj[filter_['field']] <= filter_['value']: - # clean = False - # break - - except TypeError: - # type mismatch of comparison operands - ignore filter, - # no error raised for now - pass - - # if object unmarked after all filter, add it + # if object unmarked after all filters, add it if clean: filtered_stix_objs.append(stix_obj) - clean = True - return filtered_stix_objs def deduplicate(self, stix_obj_list): @@ -704,3 +441,458 @@ class DataSource(object): unique.append(i) have = False return unique + + +class CompositeDataSource(object): + """Composite Data Source + + Acts as a controller for all the defined/configured STIX Data Sources + e.g. a user can defined n Data Sources - creating Data Source (objects) + for each. There is only one instance of this for any python STIX 2.0 + application + + """ + def __init__(self, name="CompositeDataSource"): + """ + Creates a new STIX Data Source. + + Args: + 'data_sources' (dict): a dict of DataSource objects; to be + controlled and used by the Data Source Controller object + + filters : + name : + """ + self.id = make_id() + self.name = name + self.data_sources = {} + self.filters = {} + self.filter_allowed = {} + + def get(self, stix_id): + """Retrieve STIX object by 'id' + + federated retrieve method-iterates through all STIX data sources + defined in the "data_sources" parameter. Each data source has a + specific API retrieve-like function and associated parameters. This + function does a federated retrieval and consolidation of the data + returned from all the STIX data sources. + + note: a composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle + + Args: + id (str): the id of the STIX object to retrieve + + Returns: + stix_obj (dict): the STIX object to be returned + + """ + + all_data = [] + + # for every configured Data Source, call its retrieve handler + for ds_id, ds in iteritems(self.data_sources): + data = ds.get(stix_id=stix_id, _composite_filters=self.filters.values()) + all_data.extend(data) + + # remove duplicate versions + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + + return stix_obj + + def all_versions(self, stix_id): + """Retrieve STIX objects by 'id' + + Federated all_versions retrieve method - iterates through all STIX data + sources defined in "data_sources" + + note: a composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle + + Args: + id_ (str): id of the STIX objects to retrieve + + Returns: + all_data (list): list of STIX objects that have the specified id + """ + all_data = [] + + # retrieve STIX objects from all configured data sources + for ds_id, ds in iteritems(self.data_sources): + data = ds.all_versions(stix_id=stix_id, _composite_filters=self.filters.values()) + all_data.extend(data) + + # remove exact duplicates (where duplicates are STIX 2.0 objects + # with the same 'id' and 'modified' values) + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + return all_data + + def query(self, query=None): + """composite data source query + + Federate the query to all Data Sources attached + to the Composite Data Source + + Args: + query (list): list of filters to search on + + Returns: + all_data (list): list of STIX objects to be returned + + """ + if not query: + query = [] + + all_data = [] + + # federate query to all attached data sources, + # pass composite filters to id + for ds_id, ds in iteritems(self.data_sources): + data = ds.query(query=query, _composite_filters=self.filters.values()) + all_data.extend(data) + + # remove exact duplicates (where duplicates are STIX 2.0 + # objects with the same 'id' and 'modified' values) + if len(all_data) > 0: + all_data = self.deduplicate(all_data) + + return all_data + + def add_data_source(self, data_sources): + """add/attach Data Source to the Composite Data Source instance + + Args: + data_sources (list): a list of Data Source objects to attach + to the Composite Data Source + + Returns: + + """ + + for ds in data_sources: + if issubclass(ds, DataSource): + if self.data_sources[ds['id']] in self.data_sources.keys(): + # data source already attached to Composite Data Source + continue + + # add data source to Composite Data Source + # (its id will be its key identifier) + self.data_sources[ds['id']] = ds + else: + # the Data Source object is not a proper subclass + # of DataSource Abstract Class + # TODO: maybe log error? + continue + + return + + def remove_data_source(self, data_source_ids): + """remove/detach Data Source from the Composite Data Source instance + + Args: + data_source_ids (list): a list of Data Source + id's(which are strings) + + Returns: + + """ + + for id_ in data_source_ids: + try: + if self.data_sources[id_]: + del self.data_sources[id_] + except KeyError: + # Data Source 'id' was not found in CompositeDataSource's + # list of data sources + pass + return + + @property + def data_sources(self): + """return all attached Data Sources + + Args: + + Returns: + + """ + return copy.deepcopy(self.data_sources.values()) + + def add_filter(self, filters): + """add/attach a filter to the Composite Data Source instance + + Args: + filters (list): list of filters (dict) to add to the Data Source + + Returns: + status (list): list of status/error messages + + """ + + status = [] + errors = [] + ids = [] + allowed = True + + for filter_ in filters: + # check required filter components ("field", "op", "value") exist + for field in FILTER_FIELDS: + if field not in filter_.keys(): + allowed = False + errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") + break + + if allowed: + # no need for further checks if filter is missing parameters + + # check filter field is a supported STIX 2.0 common field + if filter_['field'] not in STIX_COMMON_FIELDS: + allowed = False + errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") + + # check filter operator is supported + if filter_['op'] not in FILTER_OPS: + allowed = False + errors.append("Filter operation(from 'op' field) not supported") + + # check filter value type is supported + if type(filter_['value']) not in FILTER_VALUE_TYPES: + allowed = False + errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") + + # Filter is added regardless of whether it fits requirements + # to be a common filter. This is done because some filters + # may be added and used by third party Data Sources, where the + # filtering may be conducted within those plugins, just not here + + id_ = make_id() + filter_['id'] = id_ + self.filters['id_'] = filter_ + ids.append(id_) + + if allowed: + self.filter_allowed[id_] = True + status.append({ + "status": "added as a common filter", + "filter": filter_, + "data_source_name": self.name, + "data_source_id": self.id_ + }) + else: + self.filter_allowed[id_] = False + status.append({ + "status": "added but is not a common filter", + "filter": filter_, + "errors": errors, + "data_source_name": self.name, + "data_source_id": self.id_ + }) + del errors[:] + + allowed = True + + return ids, status + + def remove_filter(self, filter_ids): + """Remove/detach a filter from the Data Source instance + + Args: + filter_ids (list): list of filter id's (which are strings) + dettach from the Composite Data Source + + Returns: + + """ + + for filter_id in filter_ids: + try: + if filter_id in self.filters: + del self.filters[filter_id] + del self.filter_allowed[filter_id] + except KeyError: + # filter id not found in list of filters + # attached to the Composite Data Source + pass + + return + + @property + def filters(self): + """return filters attached to Composite Data Source + + Args: + + Returns: + (list): the list of filters currently attached to the Data Source + + """ + return copy.deepcopy(list(self.filters.values())) + + def deduplicate(self, stix_obj_list): + """deduplicate a list fo STIX objects to a unique set + + Reduces a set of STIX objects to unique set by looking + at 'id' and 'modified' fields - as a unique object version + is determined by the combination of those fields + + Args: + stix_obj_list (list): list of STIX objects (dicts) + + Returns: + (list): unique set of the passed list of STIX objects + """ + + unique = [] + dont_have = False + for i in stix_obj_list: + dont_have = False + for j in unique: + for field in STIX_VERSION_FIELDS: + if not i[field] == j[field]: + dont_have = True + break + if dont_have: + unique.append(i) + return unique + + +class STIXCommonPropertyFilters(): + """ + """ + @classmethod + def _all(cls, filter_, stix_obj_field): + """all filter operations (for filters whose value type can be applied to any operation type)""" + if filter_["op"] == '=': + return stix_obj_field == filter_["value"] + elif filter_["op"] == "!=": + return stix_obj_field != filter_["value"] + elif filter_["op"] == "in": + return stix_obj_field in filter_["value"] + elif filter_["op"] == ">": + return stix_obj_field > filter_["value"] + elif filter_["op"] == "<": + return stix_obj_field < filter_["value"] + elif filter_["op"] == ">=": + return stix_obj_field >= filter_["value"] + elif filter_["op"] == "<=": + return stix_obj_field <= filter_["value"] + else: + return -1 + + @classmethod + def _id(cls, filter_, stix_obj_id): + """base filter types""" + if filter_["op"] == "=": + return stix_obj_id == filter_["value"] + elif filter_["op"] == "!=": + return stix_obj_id != filter_["value"] + else: + return -1 + + @classmethod + def _boolean(filter_, stix_obj_field): + if filter_["op"] == "=": + return stix_obj_field == filter_["value"] + elif filter_["op"] == "!=": + return stix_obj_field != filter_["value"] + else: + return -1 + + @classmethod + def _string(cls, filter_, stix_obj_field): + return cls._all(filter_, stix_obj_field) + + @classmethod + def _timestamp(cls, filter_, stix_obj_timestamp): + return cls._all(filter_, stix_obj_timestamp) + + # STIX 2.0 Common Property filters + @classmethod + def created(cls, filter_, stix_obj): + return cls._timestamp(filter_, stix_obj["created"]) + + @classmethod + def created_by_ref(cls, filter_, stix_obj): + return cls._id(filter_, stix_obj["created_by_ref"]) + + @classmethod + def external_references(cls, filter_, stix_obj): + ''' + stix object's can have a list of external references + + external-reference properties: + external_reference.source_name (string) + external_reference.description (string) + external_reference.url (string) + external_reference.hashes (hash, but for filtering purposes , a string) + external_reference.external_id (string) + ''' + for er in stix_obj["external_references"]: + # grab er property name from filter field + filter_field = filter_["field"].split(".")[1] + r = cls._string(filter_, er[filter_field]) + if r: + return r + return False + + @classmethod + def granular_markings(cls, filter_, stix_obj): + ''' + stix object's can have a list of granular marking references + + granular-marking properties: + granular-marking.marking_ref (id) + granular-marking.selectors (string) + ''' + for gm in stix_obj["granular_markings"]: + # grab gm property name from filter field + filter_field = filter_["field"].split(".")[1] + + if filter_field == "marking_ref": + return cls._id(filter_, gm[filter_field]) + + elif filter_field == "selectors": + for selector in gm[filter_field]: + r = cls._string(filter_, selector) + if r: + return r + return False + + @classmethod + def id(cls, filter_, stix_obj): + return cls._id(filter_, stix_obj["id"]) + + @classmethod + def labels(cls, filter_, stix_obj): + for label in stix_obj["labels"]: + r = cls._string(filter_, label) + if r: + return r + return False + + @classmethod + def modified(cls, filter_, stix_obj): + return cls._timestamp(filter_, stix_obj["created"]) + + @classmethod + def object_markings_ref(cls, filter_, stix_obj): + for marking_id in stix_obj["object_market_refs"]: + r = cls._id(filter_, marking_id) + if r: + return r + return False + + @classmethod + def revoked(cls, filter_, stix_obj): + return cls._boolean(filter_, stix_obj["revoked"]) + + @classmethod + def type(cls, filter_, stix_obj): + return cls._string(filter_, stix_obj["type"]) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py new file mode 100644 index 0000000..17cb9b8 --- /dev/null +++ b/stix2/sources/filesystem.py @@ -0,0 +1,256 @@ +""" +Python STIX 2.0 FileSystem Source/Sink + +Classes: + FileSystemStore + FileSystemSink + FileSystemSource + +TODO: Test everything +""" + +import json +import os + +from sources import DataSink, DataSource, DataStore, make_id +from stix2 import Bundle + + +class FileSystemStore(DataStore): + """ + + """ + def __init__(self, stix_dir="stix_data", source=None, sink=None, name="FileSystemStore"): + self.name = name + self.id = make_id() + + if source: + self.source = source + else: + self.source = FileSystemSource(stix_dir=stix_dir) + + if sink: + self.sink = sink + else: + self.sink = FileSystemSink(stix_dir=stix_dir) + + @property + def source(self): + return self.source + + @source.setter + def source(self, source): + self.source = source + + @property + def sink(self): + return self.sink + + @sink.setter + def sink(self, sink): + self.sink = sink + + # file system sink API calls + + def add(self, stix_objs): + return self.sink.add(stix_objs=stix_objs) + + def remove(self, stix_ids): + return self.sink.remove(stix_ids=stix_ids) + + # file sytem source API calls + + def get(self, stix_id): + return self.source.get(stix_id=stix_id) + + def all_versions(self, stix_id): + return self.source.all_versions(stix_id=stix_id) + + def query(self, query): + return self.source.query(query=query) + + +class FileSystemSink(DataSink): + """ + """ + def __init__(self, stix_dir="stix_data", name="FileSystemSink"): + super(FileSystemSink, self).__init__(name=name) + self.stix_dir = os.path.abspath(stix_dir) + + # check directory path exists + if not os.path.exists(self.stix_dir): + print("Error: directory path for STIX data does not exist") + + @property + def stix_dir(self): + return self.stix_dir + + @stix_dir.setter + def stix_dir(self, dir): + self.stix_dir = dir + + def add(self, stix_objs=None): + """ + Q: bundlify or no? + """ + if not stix_objs: + stix_objs = [] + for stix_obj in stix_objs: + path = os.path.join(self.stix_dir, stix_obj["type"], stix_obj["id"]) + json.dump(Bundle([stix_obj]), open(path, 'w+', indent=4)) + + def remove(self, stix_ids=None): + if not stix_ids: + stix_ids = [] + for stix_id in stix_ids: + stix_type = stix_id.split("--")[0] + try: + os.remove(os.path.join(self.stix_dir, stix_type, stix_id)) + except OSError: + # log error? nonexistent object in data with directory + continue + + +class FileSystemSource(DataSource): + """ + """ + def __init__(self, stix_dir="stix_data", name="FileSystemSource"): + super(FileSystemSource, self).__init__(name=name) + self.stix_dir = os.path.abspath(stix_dir) + + # check directory path exists + if not os.path.exists(self.stix_dir): + print("Error: directory path for STIX data does not exist") + + @property + def stix_dir(self): + return self.stix_dir + + @stix_dir.setter + def stix_dir(self, dir): + self.stix_dir = dir + + def get(self, stix_id, _composite_filters=None): + """ + """ + query = [ + { + "field": "id", + "op": "=", + "value": stix_id + } + ] + + all_data = self.query(query=query, _composite_filters=_composite_filters) + + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + + return stix_obj + + def all_versions(self, stix_id, _composite_filters=None): + """ + NOTE: since FileSystem sources/sinks dont handle mutliple verions of a STIX object, + this operation is futile. Pass call to get(). (Appoved by G.B.) + """ + + # query = [ + # { + # "field": "id", + # "op": "=", + # "value": stix_id + # } + # ] + + # all_data = self.query(query=query, _composite_filters=_composite_filters) + + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + + def query(self, query=None, _composite_filters=None): + """ + """ + all_data = [] + + if query is None: + query = [] + + # combine all query filters + if self.filters: + query.extend(self.filters.values()) + if _composite_filters: + query.extend(_composite_filters) + + # extract any filters that are for "type" or "id" , as we can then do + # filtering before reading in the STIX objects. A STIX 'type' filter + # can reduce the query to a single sub-directory. A STIX 'id' filter + # allows for the fast checking of the file names versus loading it. + file_filters = self._parse_file_filters(query) + + # establish which subdirectories can be avoided in query + # by decluding as many as possible. A filter with "type" as the field + # means that certain STIX object types can be ruled out, and thus + # the corresponding subdirectories as well + include_paths = [] + declude_paths = [] + if "type" in [filter_['field'] for filter_ in file_filters]: + for filter_ in file_filters: + if filter_['field'] == "type": + if filter_['op'] == '=': + include_paths.append(os.path.join(self.stix_dir, filter_['value'])) + elif filter_['op'] == "!=": + declude_paths.append(os.path.join(self.stix_dir, filter_['value'])) + else: + # have to walk entire STIX directory + include_paths.append(self.stix_dir) + + # if a user specifies a "type" filter like "type = ", + # the filter is reducing the search space to single stix object types + # (and thus single directories). This makes such a filter more powerful + # than "type != " bc the latter is substracting + # only one type of stix object type (and thus only one directory), + # As such the former type of filters are given preference over the latter; + # i.e. if both exist in a query, that latter type will be ignored + + if not include_paths: + # user has specified types that are not wanted (i.e. "!=") + # so query will look in all STIX directories that are not + # the specified type. Compile correct dir paths + for dir_ in os.listdir(self.stix_dir): + if os.path.abspath(dir_) not in declude_paths: + include_paths.append(os.path.abspath(dir_)) + + # grab stix object ID as well - if present in filters, as + # may forgo the loading of STIX content into memory + if "id" in [filter_['field'] for filter_ in file_filters]: + for filter_ in file_filters: + if filter_['field'] == 'id' and filter_['field'] == '=': + id_ = filter_['value'] + else: + id_ = None + + # now iterate through all STIX objs + for path in include_paths: + for root, dirs, files in os.walk(path): + for file in files: + if id_: + if id_ == file.split(".")[0]: + # since ID is specified in one of filters, can evaluate against filename first without loading + stix_obj = json.load(file)['objects'] + # check against other filters, add if match + all_data.extend(self.apply_common_filters([stix_obj], query)) + else: + # have to load into memory regardless to evaluate other filters + stix_obj = json.load(file)['objects'] + all_data.extend(self.apply_common_filters([stix_obj], query)) + + all_data = self.deduplicate(all_data) + + return all_data + + def _parse_file_filters(self, query): + """ + """ + file_filters = [] + for filter_ in query: + if filter_['field'] == "id" or filter_['field'] == "type": + file_filters.append(filter_) + return file_filters diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py new file mode 100644 index 0000000..ff231e4 --- /dev/null +++ b/stix2/sources/memory.py @@ -0,0 +1,268 @@ +""" +Python STIX 2.0 Memory Source/Sink + +Classes: + MemoryStore + MemorySink + MemorySource + +TODO: Test everything. + +TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) + can save a lot of time for successive queries + +NOTE: Not worrying about STIX versioning. The in memory STIX data at anytime + will only hold one version of a STIX object. As such, when save() is called, + the single versions of all the STIX objects are what is written to file. + +""" + +import json +import os + +from stix2 import Bundle +from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2validator import validate_string + + +class MemoryStore(DataStore): + """ + """ + def __init__(self, stix_data=None, source=None, sink=None, name="MemoryStore"): + self.name = name + self.id = make_id() + + if source: + self.source = source + else: + self.source = MemorySource(stix_data=stix_data) + + if sink: + self.sink = sink + else: + self.sink = MemorySink(stix_data=stix_data) + + @property + def source(self): + return self.source + + @source.setter + def source(self, source): + self.source = source + + @property + def sink(self): + return self.sink + + @sink.setter + def sink(self, sink): + self.sink = sink + + # memory sink API calls + + def add(self, stix_data): + return self.sink.add(stix_data=stix_data) + + def remove(self, stix_ids): + return self.sink.remove(stix_ids=stix_ids) + + def save(self): + return self.sink.save() + + # memory source API calls + + def get(self, stix_id): + return self.source.get(stix_id=stix_id) + + def all_versions(self, stix_id): + return self.source.all_versions(stix_id=stix_id) + + def query(self, query): + return self.source.query(query=query) + + +class MemorySink(DataSink): + """ + + """ + def __init__(self, stix_data=None, name="MemorySink"): + """ + Args: + + data (dictionary OR list): valid STIX 2.0 content in bundle or a list + name (string): optional name tag of the data source + + """ + super(MemorySink, self).__init__(name=name) + self.data = {} + if stix_data: + if type(stix_data) == dict: + # stix objects are in a bundle + # verify STIX json data + r = validate_string(json.dumps(stix_data)) + # make dictionary of the objects for easy lookup + if r.is_valid: + for stix_obj in stix_data["objects"]: + + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") + print(r) + self.data = {} + elif type(stix_data) == list: + # stix objects are in a list + for stix_obj in stix_data: + r = validate_string(json.dumps(stix_obj)) + if r.is_valid: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print(r) + else: + raise ValueError("stix_data must be in bundle format or raw list") + + def add(self, stix_data): + """ + """ + if type(stix_data) == dict: + # stix data is in bundle + r = validate_string(json.dumps(stix_data)) + if r.is_valid: + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") + print(r) + elif type(stix_data) == list: + # stix data is in list + for stix_obj in stix_data: + r = validate_string(json.dumps(stix_obj)) + if r.is_valid: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print(r) + else: + raise ValueError("stix_data must be in bundle format or raw list") + + def remove(self, stix_ids): + """ + """ + for stix_id in stix_ids: + try: + del self.data[stix_id] + except KeyError: + pass + + def save(self, file_path=None): + """ + """ + if not file_path: + file_path = os.path.dirname(os.path.realpath(__file__)) + json.dump(Bundle(self.data.values()), file_path, indent=4) + + +class MemorySource(DataSource): + + def __init__(self, stix_data=None, name="MemorySource"): + """ + Args: + + data (dictionary OR list): valid STIX 2.0 content in bundle or list + name (string): optional name tag of the data source + + """ + super(MemorySource, self).__init__(name=name) + self.data = {} + + if stix_data: + if type(stix_data) == dict: + # stix objects are in a bundle + # verify STIX json data + r = validate_string(json.dumps(stix_data)) + # make dictionary of the objects for easy lookup + if r.is_valid: + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") + print(r) + self.data = {} + elif type(stix_data) == list: + # stix objects are in a list + for stix_obj in stix_data: + r = validate_string(json.dumps(stix_obj)) + if r.is_valid: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print(r) + else: + raise ValueError("stix_data must be in bundle format or raw list") + + def get(self, stix_id, _composite_filters=None): + """ + """ + if _composite_filters is None: + # if get call is only based on 'id', no need to search, just retrieve from dict + try: + stix_obj = self.data[stix_id] + except KeyError: + stix_obj = None + return stix_obj + + # if there are filters from the composite level, process full query + query = [ + { + "field": "id", + "op": "=", + "value": stix_id + } + ] + + all_data = self.query(query=query, _composite_filters=_composite_filters) + + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + + return stix_obj + + def all_versions(self, stix_id, _composite_filters=None): + """ + NOTE: since Memory sources/sinks dont handle mutliple verions of a STIX object, + this operation is futile. Translate call to get(). (Appoved by G.B.) + """ + + # query = [ + # { + # "field": "id", + # "op": "=", + # "value": stix_id + # } + # ] + + # all_data = self.query(query=query, _composite_filters=_composite_filters) + + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + + def query(self, query=None, _composite_filters=None): + """ + + """ + + if query is None: + query = [] + + # combine all query filters + if self.filters: + query.extend(self.filters.values()) + if _composite_filters: + query.extend(_composite_filters) + + # deduplicate data before filtering -> Deduplication is not required as Memory only ever holds one version of an object + # all_data = self.depuplicate(all_data) + + # apply STIX common property filters + all_data = self.apply_common_filters(self.data.values(), query) + + return all_data diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index c83d220..d2669f3 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,131 +1,213 @@ -import requests -from requests.auth import HTTPBasicAuth +""" +Python STIX 2.0 TAXII Source/Sink -from stix2.sources import DataSource +Classes: + TAXIICollectionStore + TAXIICollectionSink + TAXIICollectionSource -# TODO: -Should we make properties for the TAXIIDataSource address and other -# possible variables that are found in "self.taxii_info" +TODO: Test everything +""" + +import json +import uuid + +from stix2.sources import DataSink, DataSource, DataStore, make_id +from taxii2_client import TAXII2Client TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] -test = True + +class TAXIICollectionStore(DataStore): + """ + """ + def __init__(self, + source=None, + sink=None, + server_uri=None, + api_root_name=None, + collection_id=None, + user=None, + password=None, + name="TAXIICollectionStore"): + + self.name = name + self.id = make_id() + + if source: + self.source = source + else: + self.source = TAXIICollectionSource(server_uri, api_root_name, collection_id, user, password) + + if sink: + self.sink = sink + else: + self.TAXIICollectionSink(server_uri, api_root_name, collection_id, user, password) + + @property + def source(self): + return self.source + + @source.setter + def source(self, source): + self.source = source + + @property + def sink(self): + return self.sink + + @sink.setter + def sink(self, sink): + self.sink = sink + + # file system sink API calls + + def add(self, stix_objs): + return self.sink.add(stix_objs=stix_objs) + + # file sytem source API calls + + def get(self, stix_id): + return self.source.get(stix_id=stix_id) + + def all_versions(self, stix_id): + return self.source.all_versions(stix_id=stix_id) + + def query(self, query): + return self.source.query(query=query) -class TAXIIDataSource(DataSource): - """STIX 2.0 Data Source - TAXII 2.0 module""" +class TAXIICollectionSink(DataSink): + """ + """ - def __init__(self, api_root=None, auth=None, name="TAXII"): - super(TAXIIDataSource, self).__init__(name=name) + def __init__(self, server_uri=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSink"): + super(TAXIICollectionSink, self).__init__(name=name) - if not api_root: - api_root = "http://localhost:5000" - if not auth: - auth = {"user": "admin", "pass": "taxii"} + self.taxii_client = TAXII2Client(server_uri, user, password) + self.taxii_client.populate_available_information() - self.taxii_info = { - "api_root": { - "url": api_root - }, - "auth": auth - } - - if test: - return - - try: - # check api-root is reachable/exists and grab api collections - coll_url = self.taxii_info['api_root']['url'] + "/collections/" - headers = {} - - resp = requests.get(coll_url, - headers=headers, - auth=HTTPBasicAuth(self.taxii_info['auth']['user'], - self.taxii_info['auth']['pass'])) - # TESTING - # print("\n-------__init__() ----\n") - # print(resp.text) - # print("\n") - # print(resp.status_code) - # END TESTING - - # raise http error if request returned error code - resp.raise_for_status() - - resp_json = resp.json() - - try: - self.taxii_info['api_root']['collections'] = resp_json['collections'] - except KeyError as e: - if e == "collections": - raise - # raise type(e), type(e)(e.message + - # "To connect to the TAXII collections, the API root - # resource must contain a collection endpoint URL. - # This was not found in the API root resource received - # from the API root" ), sys.exc_info()[2] - - except requests.ConnectionError as e: - raise - # raise type(e), type(e)(e.message + - # "Attempting to connect to %s" % coll_url) - - def get(self, id_, _composite_filters=None): - """Get STIX 2.0 object from TAXII source by specified 'id' - - Notes: - Just pass _composite_filters to the query() as they are applied - there. de-duplication of results is also done within query() - - Args: - id_ (str): id of STIX object to retrieve - - _composite_filters (list): filters passed from a Composite Data - Source (if this data source is attached to one) - - Returns: + if not api_root_name: + raise ValueError("No api_root specified.") + else: + self.api_root = None + for a_r in self.taxii_client.api_roots: + if api_root_name == a_r.name: + self.api_root = a_r + break + if not self.api_root: + raise ValueError("The api_root %s is not found on this taxii server" % api_root_name) + if not collection_id: + raise ValueError("No collection specified.") + else: + self.collection = None + for c in self.api_root.collections: + if c.id_ == collection_id: + self.collection = c + break + if not self.collection: + raise ValueError("The collection %s is not found on the api_root %s of this taxii server" % + (collection_id, api_root_name)) + def save(self, stix_obj): """ + """ + self.collection.add_objects(self.create_bundle([json.loads(str(stix_obj))])) - # make query in TAXII query format since 'id' is TAXii field - query = [ - { - "field": "match[id]", - "op": "=", - "value": id_ - } - ] + @staticmethod + def create_bundle(objects): + return dict(id="bundle--" + str(uuid.uuid4()), + objects=objects, + spec_version="2.0", + type="bundle") - all_data = self.query(query=query, _composite_filters=_composite_filters) + # utility functions for the current set collection and api root + def get_api_root_info(self): + """ + """ + return self.api_root.get_information() - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + def get_api_root_collections(self): + """ + """ + return self.api_root.get_collections() + + def get_collection_manifest(self): + """ + """ + return self.collection.get_collection_manifest() + + +class TAXIICollectionSource(DataSource): + """ + """ + def __init__(self, server_uri=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSourc"): + super(TAXIICollectionSource, self).__init__(name=name) + + self.taxii_client = TAXII2Client(server_uri, user, password) + self.taxii_client.populate_available_information() + + if not api_root_name: + raise ValueError("No api_root specified.") + else: + self.api_root = None + for a_r in self.taxii_client.api_roots: + if api_root_name == a_r.name: + self.api_root = a_r + break + if not self.api_root: + raise ValueError("The api_root %s is not found on this taxii server" % api_root_name) + if not collection_id: + raise ValueError("No collection specified.") + else: + self.collection = None + for c in self.api_root.collections: + if c.id_ == collection_id: + self.collection = c + break + if not self.collection: + raise ValueError("The collection %s is not found on the api_root %s of this taxii server" % + (collection_id, api_root_name)) + + def get(self, stix_id, _composite_filters=None): + """ + """ + # combine all query filters + query = [] + if self.filters: + query.extend(self.filters.values()) + if _composite_filters: + query.extend(_composite_filters) + + # separate taxii query terms (can be done remotely) + taxii_filters = self._parse_taxii_filters(query) + + stix_objs = self.collection.get_object(stix_id, taxii_filters)["objects"] + + stix_obj = self.apply_common_filters(stix_objs, query) + + if len(stix_obj) > 0: + stix_obj = stix_obj[0] + else: + stix_obj = None return stix_obj - def all_versions(self, id_, _composite_filters=None): - """Get all versions of STIX 2.0 object from TAXII source by - specified 'id' - - Notes: - Just passes _composite_filters to the query() as they are applied - there. de-duplication of results is also done within query() - - Args: - id_ (str): id of STIX objects to retrieve - _composite_filters (list): filters passed from a Composite Data - Source (if this data source is attached to one) - - Returns: - The query results with filters applied. + def all_versions(self, stix_id, _composite_filters=None): + """ """ - # make query in TAXII query format since 'id' is TAXII field query = [ { "field": "match[id]", "op": "=", - "value": id_ + "value": stix_id + }, + { + "field": "match[version]", + "op": "=", + "value": "all" } ] @@ -134,84 +216,22 @@ class TAXIIDataSource(DataSource): return all_data def query(self, query=None, _composite_filters=None): - """Query the TAXII data source for STIX objects matching the query - - The final full query could contain filters from: - -the current API call - -Composite Data source filters (that are passed in via - '_composite_filters') - -TAXII data source filters that are attached - - TAXII filters ['added_after', 'match[<>]'] are extracted and sent - to TAXII if they are present - - TODO: Authentication for TAXII - - Args: - - query(list): list of filters (dicts) to search on - - _composite_filters (list): filters passed from a - Composite Data Source (if this data source is attached to one) - - Returns: - - """ - - all_data = [] - + """ if query is None: query = [] # combine all query filters if self.filters: - query += self.filters.values() + query.extend(self.filters.values()) if _composite_filters: - query += _composite_filters + query.extend(_composite_filters) # separate taxii query terms (can be done remotely) taxii_filters = self._parse_taxii_filters(query) - # for each collection endpoint - send query request - for collection in self.taxii_info['api_root']['collections']: - - coll_obj_url = "/".join([self.taxii_info['api_root']['url'], - "collections", str(collection['id']), - "objects"]) - headers = {} - try: - resp = requests.get(coll_obj_url, - params=taxii_filters, - headers=headers, - auth=HTTPBasicAuth(self.taxii_info['auth']['user'], - self.taxii_info['auth']['pass'])) - # TESTING - # print("\n-------query() ----\n") - # print("Request that was sent: \n") - # print(resp.url) - # print("Response: \n") - # print(json.dumps(resp.json(),indent=4)) - # print("\n") - # print(resp.status_code) - # print("------------------") - # END TESTING - - # raise http error if request returned error code - resp.raise_for_status() - resp_json = resp.json() - - # grab all STIX 2.0 objects in json response - for stix_obj in resp_json['objects']: - all_data.append(stix_obj) - - except requests.exceptions.RequestException as e: - raise e - # raise type(e), type(e)(e.message + - # "Attempting to connect to %s" % coll_url) - - # TODO: Is there a way to collect exceptions while carrying - # on then raise all of them at the end? + # query TAXII collection + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = self.deduplicate(all_data) @@ -224,11 +244,10 @@ class TAXIIDataSource(DataSource): def _parse_taxii_filters(self, query): """Parse out TAXII filters that the TAXII server can filter on - TAXII filters should be analgous to how they are supplied - in the url to the TAXII endpoint. For instance + For instance "?match[type]=indicator,sighting" should be in a query dict as follows { - "field": "match[type]" + "field": "type" "op": "=", "value": "indicator,sighting" } @@ -244,19 +263,62 @@ class TAXIIDataSource(DataSource): params = {} - for q in query: - if q['field'] in TAXII_FILTERS: - if q['field'] == 'added_after': - params[q['field']] = q['value'] + for filter_ in query: + if filter_["field"] in TAXII_FILTERS: + if filter_["field"] == "added_after": + params[filter_["field"]] = filter_["value"] else: - taxii_field = 'match[' + q['field'] + ']' - params[taxii_field] = q['value'] + taxii_field = "match[" + filter_["field"] + ']' + params[taxii_field] = filter_["value"] return params - def close(self): - """Close down the Data Source - if any clean up is required. - + # utility functions for the current attached collection and api root + def get_api_root_info(self): """ - pass + """ + return self.api_root.get_information() - # TODO: - getters/setters (properties) for TAXII config info + def get_api_root_collections(self): + """ + """ + return self.api_root.get_collections() + + def get_collection_manifest(self): + """ + """ + return self.collection.get_collection_manifest() + + +def get_server_api_roots(taxii_client): + """ + """ + api_root_info = [] + taxii_client.populate_available_information() + + for api_root in taxii_client.api_roots: + api_root_info.append(api_root.information()) + + return api_root_info + + +def get_server_collections(taxii_client): + """ + """ + server_collections = [] + + taxii_client.populate_available_information() + + for api_root in taxii_client.api_roots: + server_collections.extend(api_root.get_collections()) + + return server_collections + + +def get_api_root_collections(taxii_client, api_root_name): + """ + """ + taxii_client.populate_available_information() + + for api_root in taxii_client.api_roots: + if api_root == api_root_name: + return api_root.get_collections() From 9f659f9ecbf407a98d5c0cefad49346f91b2b07e Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Wed, 12 Jul 2017 11:26:50 -0400 Subject: [PATCH 12/82] formatting --- stix2/sources/filesystem.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 17cb9b8..141f432 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -191,13 +191,13 @@ class FileSystemSource(DataSource): # the corresponding subdirectories as well include_paths = [] declude_paths = [] - if "type" in [filter_['field'] for filter_ in file_filters]: + if "type" in [filter_["field"] for filter_ in file_filters]: for filter_ in file_filters: - if filter_['field'] == "type": - if filter_['op'] == '=': - include_paths.append(os.path.join(self.stix_dir, filter_['value'])) - elif filter_['op'] == "!=": - declude_paths.append(os.path.join(self.stix_dir, filter_['value'])) + if filter_["field"] == "type": + if filter_["op"] == '=': + include_paths.append(os.path.join(self.stix_dir, filter_["value"])) + elif filter_["op"] == "!=": + declude_paths.append(os.path.join(self.stix_dir, filter_["value"])) else: # have to walk entire STIX directory include_paths.append(self.stix_dir) @@ -220,26 +220,26 @@ class FileSystemSource(DataSource): # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory - if "id" in [filter_['field'] for filter_ in file_filters]: + if "id" in [filter_["field"] for filter_ in file_filters]: for filter_ in file_filters: - if filter_['field'] == 'id' and filter_['field'] == '=': - id_ = filter_['value'] + if filter_["field"] == "id" and filter_["field"] == '=': + id_ = filter_["value"] else: id_ = None # now iterate through all STIX objs for path in include_paths: for root, dirs, files in os.walk(path): - for file in files: + for file_ in files: if id_: - if id_ == file.split(".")[0]: + if id_ == file_.split(".")[0]: # since ID is specified in one of filters, can evaluate against filename first without loading - stix_obj = json.load(file)['objects'] + stix_obj = json.load(file_)["objects"] # check against other filters, add if match all_data.extend(self.apply_common_filters([stix_obj], query)) else: # have to load into memory regardless to evaluate other filters - stix_obj = json.load(file)['objects'] + stix_obj = json.load(file_)["objects"] all_data.extend(self.apply_common_filters([stix_obj], query)) all_data = self.deduplicate(all_data) @@ -251,6 +251,6 @@ class FileSystemSource(DataSource): """ file_filters = [] for filter_ in query: - if filter_['field'] == "id" or filter_['field'] == "type": + if filter_["field"] == "id" or filter_["field"] == "type": file_filters.append(filter_) return file_filters From 611045528f6537bcd31327e0ec1f15866437645d Mon Sep 17 00:00:00 2001 From: mbastian1135 Date: Wed, 12 Jul 2017 18:03:59 -0400 Subject: [PATCH 13/82] taxii client and source/sink seperated; memory store common data dict (bug); all *Store classes have their sources and sinks tethered to one backend target --- stix2/sources/__init__.py | 20 +--- stix2/sources/filesystem.py | 36 +----- stix2/sources/memory.py | 233 ++++++++++++++++++++---------------- stix2/sources/taxii.py | 35 ++---- 4 files changed, 145 insertions(+), 179 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 8f252f7..e25aeb0 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -64,34 +64,20 @@ class DataStore(object): An implementer will create a concrete subclass from this abstract class for the specific data store. """ - def __init__(self, name="DataStore", source=None, sink=None): + def __init__(self, name="DataStore"): self.name = name self.id = make_id() - if source: - self.source = source - else: - self.source = DataSource() - if sink: - self.sink = sink - else: - self.sink = DataSink() + self.source = DataSource() + self.sink = DataSink() @property def source(self): return self.source - @source.setter - def source(self, source): - self.source = source - @property def sink(self): return self.sink - @sink.setter - def sink(self, sink): - self.sink = sink - def get(self, stix_id): """ Implement: diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 141f432..a8b6de0 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -20,44 +20,25 @@ class FileSystemStore(DataStore): """ """ - def __init__(self, stix_dir="stix_data", source=None, sink=None, name="FileSystemStore"): + def __init__(self, stix_dir="stix_data", name="FileSystemStore"): self.name = name self.id = make_id() - - if source: - self.source = source - else: - self.source = FileSystemSource(stix_dir=stix_dir) - - if sink: - self.sink = sink - else: - self.sink = FileSystemSink(stix_dir=stix_dir) + self.source = FileSystemSource(stix_dir=stix_dir) + self.sink = FileSystemSink(stix_dir=stix_dir) @property def source(self): return self.source - @source.setter - def source(self, source): - self.source = source - @property def sink(self): return self.sink - @sink.setter - def sink(self, sink): - self.sink = sink - # file system sink API calls def add(self, stix_objs): return self.sink.add(stix_objs=stix_objs) - def remove(self, stix_ids): - return self.sink.remove(stix_ids=stix_ids) - # file sytem source API calls def get(self, stix_id): @@ -99,17 +80,6 @@ class FileSystemSink(DataSink): path = os.path.join(self.stix_dir, stix_obj["type"], stix_obj["id"]) json.dump(Bundle([stix_obj]), open(path, 'w+', indent=4)) - def remove(self, stix_ids=None): - if not stix_ids: - stix_ids = [] - for stix_id in stix_ids: - stix_type = stix_id.split("--")[0] - try: - os.remove(os.path.join(self.stix_dir, stix_type, stix_id)) - except OSError: - # log error? nonexistent object in data with directory - continue - class FileSystemSource(DataSource): """ diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index ff231e4..b14e683 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -28,73 +28,16 @@ from stix2validator import validate_string class MemoryStore(DataStore): """ """ - def __init__(self, stix_data=None, source=None, sink=None, name="MemoryStore"): + def __init__(self, stix_data=None, name="MemoryStore"): + """ + Note: It doesnt make sense to create a MemoryStore by passing + in existing MemorySource and MemorySink because there could + be data concurrency issues. Just as easy to create new MemoryStore. + """ self.name = name self.id = make_id() - - if source: - self.source = source - else: - self.source = MemorySource(stix_data=stix_data) - - if sink: - self.sink = sink - else: - self.sink = MemorySink(stix_data=stix_data) - - @property - def source(self): - return self.source - - @source.setter - def source(self, source): - self.source = source - - @property - def sink(self): - return self.sink - - @sink.setter - def sink(self, sink): - self.sink = sink - - # memory sink API calls - - def add(self, stix_data): - return self.sink.add(stix_data=stix_data) - - def remove(self, stix_ids): - return self.sink.remove(stix_ids=stix_ids) - - def save(self): - return self.sink.save() - - # memory source API calls - - def get(self, stix_id): - return self.source.get(stix_id=stix_id) - - def all_versions(self, stix_id): - return self.source.all_versions(stix_id=stix_id) - - def query(self, query): - return self.source.query(query=query) - - -class MemorySink(DataSink): - """ - - """ - def __init__(self, stix_data=None, name="MemorySink"): - """ - Args: - - data (dictionary OR list): valid STIX 2.0 content in bundle or a list - name (string): optional name tag of the data source - - """ - super(MemorySink, self).__init__(name=name) self.data = {} + if stix_data: if type(stix_data) == dict: # stix objects are in a bundle @@ -108,7 +51,6 @@ class MemorySink(DataSink): else: print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") print(r) - self.data = {} elif type(stix_data) == list: # stix objects are in a list for stix_obj in stix_data: @@ -118,8 +60,86 @@ class MemorySink(DataSink): else: print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] print(r) - else: - raise ValueError("stix_data must be in bundle format or raw list") + + self.source = MemorySource(stix_data=self.data, _store=True) + self.sink = MemorySink(stix_data=self.data, _store=True) + + @property + def source(self): + return self.source + + @property + def sink(self): + return self.sink + + # memory sink API calls + + def add(self, stix_data): + return self.sink.add(stix_data=stix_data) + + def save_to_file(self, file_path): + return self.sink.save(file_path=file_path) + + # memory source API calls + + def get(self, stix_id): + return self.source.get(stix_id=stix_id) + + def all_versions(self, stix_id): + return self.source.all_versions(stix_id=stix_id) + + def query(self, query): + return self.source.query(query=query) + + def load_from_file(self, file_path): + return self.source.load_from_file(file_path=file_path) + + +class MemorySink(DataSink): + """ + + """ + def __init__(self, stix_data=None, name="MemorySink", _store=False): + """ + Args: + + data (dictionary OR list): valid STIX 2.0 content in bundle or a list + name (string): optional name tag of the data source + _store (bool): if the MemorySink is a part of a DataStore, in which case + "stix_data" is a direct reference to shared memory with DataSource + + """ + super(MemorySink, self).__init__(name=name) + + if _store: + self.data = stix_data + else: + self.data = {} + if stix_data: + if type(stix_data) == dict: + # stix objects are in a bundle + # verify STIX json data + r = validate_string(json.dumps(stix_data)) + # make dictionary of the objects for easy lookup + if r.is_valid: + for stix_obj in stix_data["objects"]: + + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") + print(r) + self.data = {} + elif type(stix_data) == list: + # stix objects are in a list + for stix_obj in stix_data: + r = validate_string(json.dumps(stix_obj)) + if r.is_valid: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print(r) + else: + raise ValueError("stix_data must be in bundle format or raw list") def add(self, stix_data): """ @@ -145,60 +165,54 @@ class MemorySink(DataSink): else: raise ValueError("stix_data must be in bundle format or raw list") - def remove(self, stix_ids): + def save_to_file(self, file_path): """ """ - for stix_id in stix_ids: - try: - del self.data[stix_id] - except KeyError: - pass - - def save(self, file_path=None): - """ - """ - if not file_path: - file_path = os.path.dirname(os.path.realpath(__file__)) json.dump(Bundle(self.data.values()), file_path, indent=4) class MemorySource(DataSource): - def __init__(self, stix_data=None, name="MemorySource"): + def __init__(self, stix_data=None, name="MemorySource", _store=False): """ Args: data (dictionary OR list): valid STIX 2.0 content in bundle or list name (string): optional name tag of the data source + _store (bool): if the MemorySource is a part of a DataStore, in which case + "stix_data" is a direct reference to shared memory with DataSink """ super(MemorySource, self).__init__(name=name) - self.data = {} - if stix_data: - if type(stix_data) == dict: - # stix objects are in a bundle - # verify STIX json data - r = validate_string(json.dumps(stix_data)) - # make dictionary of the objects for easy lookup - if r.is_valid: - for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) - self.data = {} - elif type(stix_data) == list: - # stix objects are in a list - for stix_obj in stix_data: - r = validate_string(json.dumps(stix_obj)) + if _store: + self.data = stix_data + else: + self.data = {} + if stix_data: + if type(stix_data) == dict: + # stix objects are in a bundle + # verify STIX json data + r = validate_string(json.dumps(stix_data)) + # make dictionary of the objects for easy lookup if r.is_valid: - self.data[stix_obj["id"]] = stix_obj + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") print(r) - else: - raise ValueError("stix_data must be in bundle format or raw list") + self.data = {} + elif type(stix_data) == list: + # stix objects are in a list + for stix_obj in stix_data: + r = validate_string(json.dumps(stix_obj)) + if r.is_valid: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print(r) + else: + raise ValueError("stix_data must be in bundle format or raw list") def get(self, stix_id, _composite_filters=None): """ @@ -266,3 +280,18 @@ class MemorySource(DataSource): all_data = self.apply_common_filters(self.data.values(), query) return all_data + + def load_from_file(self, file_path): + """ + """ + file_path = os.path.abspath(file_path) + stix_data = json.load(open(file_path, "r")) + + r = validate_string(json.dumps(stix_data)) + + if r.is_valid: + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj + else: + print("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator") % file_path + print(r) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index d2669f3..b6aea31 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -14,7 +14,6 @@ import json import uuid from stix2.sources import DataSink, DataSource, DataStore, make_id -from taxii2_client import TAXII2Client TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -23,9 +22,7 @@ class TAXIICollectionStore(DataStore): """ """ def __init__(self, - source=None, - sink=None, - server_uri=None, + taxii_client=None, api_root_name=None, collection_id=None, user=None, @@ -34,33 +31,17 @@ class TAXIICollectionStore(DataStore): self.name = name self.id = make_id() - - if source: - self.source = source - else: - self.source = TAXIICollectionSource(server_uri, api_root_name, collection_id, user, password) - - if sink: - self.sink = sink - else: - self.TAXIICollectionSink(server_uri, api_root_name, collection_id, user, password) + self.source = TAXIICollectionSource(taxii_client, api_root_name, collection_id, user, password) + self.sink = self.TAXIICollectionSink(taxii_client, api_root_name, collection_id, user, password) @property def source(self): return self.source - @source.setter - def source(self, source): - self.source = source - @property def sink(self): return self.sink - @sink.setter - def sink(self, sink): - self.sink = sink - # file system sink API calls def add(self, stix_objs): @@ -82,10 +63,10 @@ class TAXIICollectionSink(DataSink): """ """ - def __init__(self, server_uri=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSink"): + def __init__(self, taxii_client=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSink"): super(TAXIICollectionSink, self).__init__(name=name) - self.taxii_client = TAXII2Client(server_uri, user, password) + self.taxii_client = taxii_client self.taxii_client.populate_available_information() if not api_root_name: @@ -110,7 +91,7 @@ class TAXIICollectionSink(DataSink): raise ValueError("The collection %s is not found on the api_root %s of this taxii server" % (collection_id, api_root_name)) - def save(self, stix_obj): + def add(self, stix_obj): """ """ self.collection.add_objects(self.create_bundle([json.loads(str(stix_obj))])) @@ -142,10 +123,10 @@ class TAXIICollectionSink(DataSink): class TAXIICollectionSource(DataSource): """ """ - def __init__(self, server_uri=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSourc"): + def __init__(self, taxii_client=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSourc"): super(TAXIICollectionSource, self).__init__(name=name) - self.taxii_client = TAXII2Client(server_uri, user, password) + self.taxii_client = taxii_client self.taxii_client.populate_available_information() if not api_root_name: From f4335f43ad280903afa392c880098206d71fdd35 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Wed, 19 Jul 2017 21:12:56 +0000 Subject: [PATCH 14/82] Clean up redundant code. --- stix2/sources/__init__.py | 12 ++---------- stix2/sources/filesystem.py | 8 -------- stix2/sources/memory.py | 8 -------- stix2/sources/taxii.py | 10 +--------- 4 files changed, 3 insertions(+), 35 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index e25aeb0..38c8e41 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -67,16 +67,8 @@ class DataStore(object): def __init__(self, name="DataStore"): self.name = name self.id = make_id() - self.source = DataSource() - self.sink = DataSink() - - @property - def source(self): - return self.source - - @property - def sink(self): - return self.sink + self.source = None + self.sink = None def get(self, stix_id): """ diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index a8b6de0..374bdde 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -26,14 +26,6 @@ class FileSystemStore(DataStore): self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) - @property - def source(self): - return self.source - - @property - def sink(self): - return self.sink - # file system sink API calls def add(self, stix_objs): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index b14e683..ecfc790 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -64,14 +64,6 @@ class MemoryStore(DataStore): self.source = MemorySource(stix_data=self.data, _store=True) self.sink = MemorySink(stix_data=self.data, _store=True) - @property - def source(self): - return self.source - - @property - def sink(self): - return self.sink - # memory sink API calls def add(self, stix_data): diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index b6aea31..e892fcc 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -32,15 +32,7 @@ class TAXIICollectionStore(DataStore): self.name = name self.id = make_id() self.source = TAXIICollectionSource(taxii_client, api_root_name, collection_id, user, password) - self.sink = self.TAXIICollectionSink(taxii_client, api_root_name, collection_id, user, password) - - @property - def source(self): - return self.source - - @property - def sink(self): - return self.sink + self.sink = TAXIICollectionSink(taxii_client, api_root_name, collection_id, user, password) # file system sink API calls From 55e92bc237ea5dded1cc4e8357fc84d7d6b8c8b8 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Wed, 19 Jul 2017 21:30:29 +0000 Subject: [PATCH 15/82] Update with correct class name Also PEP-8 fixes --- stix2/test/test_data_sources.py | 37 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 8efc868..24adcf9 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -2,19 +2,19 @@ from stix2.sources import taxii def test_ds_taxii(): - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource() assert ds.name == 'TAXII' def test_ds_taxii_name(): - ds = taxii.TAXIIDataSource(name='My Data Source Name') + ds = taxii.TAXIICollectionSource(name='My Data Source Name') assert ds.name == "My Data Source Name" def test_ds_params(): url = "http://taxii_url.com:5000" creds = {"username": "Wade", "password": "Wilson"} - ds = taxii.TAXIIDataSource(api_root=url, auth=creds) + ds = taxii.TAXIICollectionSource(api_root=url, auth=creds) assert ds.taxii_info['api_root']['url'] == url assert ds.taxii_info['auth'] == creds @@ -55,12 +55,13 @@ def test_parse_taxii_filters(): "match[version]": "first" } - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource() taxii_filters = ds._parse_taxii_filters(query) assert taxii_filters == expected_params + def test_add_get_remove_filter(): class dummy(object): @@ -113,7 +114,7 @@ def test_add_get_remove_filter(): "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" ] - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource() # add ids, statuses = ds.add_filter(filters) @@ -130,13 +131,14 @@ def test_add_get_remove_filter(): assert statuses[5]['errors'][0] == expected_errors[2] assert statuses[6]['errors'][0] == expected_errors[3] - #get + # get ds_filters = ds.get_filters() - for idx,flt in enumerate(filters): - assert flt['value'] == filters[idx]['value'] + # TODO: what are we trying to test here? + for idx, flt in enumerate(filters): + assert flt['value'] == ds_filters[idx]['value'] - #remove + # remove ds.remove_filter([ids[3]]) ds.remove_filter([ids[4]]) ds.remove_filter([ids[5]]) @@ -146,13 +148,14 @@ def test_add_get_remove_filter(): assert len(rem_filters) == 3 - #check remaining filters + # check remaining filters rem_ids = [f['id'] for f in rem_filters] - #check remaining + # check remaining for id_ in rem_ids: assert id_ in ids[:3] + def test_apply_common_filters(): stix_objs = [ { @@ -207,7 +210,7 @@ def test_apply_common_filters(): } ] - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource() resp = ds.apply_common_filters(stix_objs, [filters[0]]) ids = [r['id'] for r in resp] @@ -220,6 +223,7 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] + def test_deduplicate(): stix_objs = [ { @@ -284,12 +288,12 @@ def test_deduplicate(): } ] - ds = taxii.TAXIIDataSource() + ds = taxii.TAXIICollectionSource() unique = ds.deduplicate(stix_objs) - #Only 3 objects are unique - #2 id's vary - #2 modified times vary for a particular id + # Only 3 objects are unique + # 2 id's vary + # 2 modified times vary for a particular id assert len(unique) == 3 @@ -302,7 +306,6 @@ def test_deduplicate(): assert "2017-01-27T13:49:53.936Z" in mods - # def test_data_source_file(): # ds = file.FileDataSource() # From f4833c05f60be2e696f61543f18f53faefe033b2 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 20 Jul 2017 19:34:09 +0000 Subject: [PATCH 16/82] Remove duplicate code. --- stix2/sources/filesystem.py | 16 ---------------- stix2/sources/memory.py | 18 +----------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 374bdde..77d5229 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -26,22 +26,6 @@ class FileSystemStore(DataStore): self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) - # file system sink API calls - - def add(self, stix_objs): - return self.sink.add(stix_objs=stix_objs) - - # file sytem source API calls - - def get(self, stix_id): - return self.source.get(stix_id=stix_id) - - def all_versions(self, stix_id): - return self.source.all_versions(stix_id=stix_id) - - def query(self, query): - return self.source.query(query=query) - class FileSystemSink(DataSink): """ diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index ecfc790..82bcf2a 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -64,24 +64,8 @@ class MemoryStore(DataStore): self.source = MemorySource(stix_data=self.data, _store=True) self.sink = MemorySink(stix_data=self.data, _store=True) - # memory sink API calls - - def add(self, stix_data): - return self.sink.add(stix_data=stix_data) - def save_to_file(self, file_path): - return self.sink.save(file_path=file_path) - - # memory source API calls - - def get(self, stix_id): - return self.source.get(stix_id=stix_id) - - def all_versions(self, stix_id): - return self.source.all_versions(stix_id=stix_id) - - def query(self, query): - return self.source.query(query=query) + return self.sink.save_to_file(file_path=file_path) def load_from_file(self, file_path): return self.source.load_from_file(file_path=file_path) From b82606ba3889e1d300b072b4bd4ca6f59efa1490 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 20 Jul 2017 19:36:54 +0000 Subject: [PATCH 17/82] Clean up TAXII Datastore to use latest TAXII client --- .isort.cfg | 2 +- stix2/sources/taxii.py | 155 +++----------------------------- stix2/test/test_data_sources.py | 56 +++++++----- 3 files changed, 48 insertions(+), 165 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 63f5b73..e141924 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests +known_third_party=dateutil,pytest,pytz,six,requests,taxii2_client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index e892fcc..581a3d5 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -21,67 +21,28 @@ TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] class TAXIICollectionStore(DataStore): """ """ - def __init__(self, - taxii_client=None, - api_root_name=None, - collection_id=None, - user=None, - password=None, - name="TAXIICollectionStore"): + def __init__(self, collection, name="TAXIICollectionStore"): + """ + Create a new TAXII Collection Data store + + Args: + collection (taxii2.Collection): Collection instance + """ self.name = name self.id = make_id() - self.source = TAXIICollectionSource(taxii_client, api_root_name, collection_id, user, password) - self.sink = TAXIICollectionSink(taxii_client, api_root_name, collection_id, user, password) - - # file system sink API calls - - def add(self, stix_objs): - return self.sink.add(stix_objs=stix_objs) - - # file sytem source API calls - - def get(self, stix_id): - return self.source.get(stix_id=stix_id) - - def all_versions(self, stix_id): - return self.source.all_versions(stix_id=stix_id) - - def query(self, query): - return self.source.query(query=query) + self.source = TAXIICollectionSource(collection) + self.sink = TAXIICollectionSink(collection) class TAXIICollectionSink(DataSink): """ """ - def __init__(self, taxii_client=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSink"): + def __init__(self, collection, name="TAXIICollectionSink"): super(TAXIICollectionSink, self).__init__(name=name) - self.taxii_client = taxii_client - self.taxii_client.populate_available_information() - - if not api_root_name: - raise ValueError("No api_root specified.") - else: - self.api_root = None - for a_r in self.taxii_client.api_roots: - if api_root_name == a_r.name: - self.api_root = a_r - break - if not self.api_root: - raise ValueError("The api_root %s is not found on this taxii server" % api_root_name) - if not collection_id: - raise ValueError("No collection specified.") - else: - self.collection = None - for c in self.api_root.collections: - if c.id_ == collection_id: - self.collection = c - break - if not self.collection: - raise ValueError("The collection %s is not found on the api_root %s of this taxii server" % - (collection_id, api_root_name)) + self.collection = collection def add(self, stix_obj): """ @@ -95,53 +56,14 @@ class TAXIICollectionSink(DataSink): spec_version="2.0", type="bundle") - # utility functions for the current set collection and api root - def get_api_root_info(self): - """ - """ - return self.api_root.get_information() - - def get_api_root_collections(self): - """ - """ - return self.api_root.get_collections() - - def get_collection_manifest(self): - """ - """ - return self.collection.get_collection_manifest() - class TAXIICollectionSource(DataSource): """ """ - def __init__(self, taxii_client=None, api_root_name=None, collection_id=None, user=None, password=None, name="TAXIICollectionSourc"): + def __init__(self, collection, name="TAXIICollectionSource"): super(TAXIICollectionSource, self).__init__(name=name) - self.taxii_client = taxii_client - self.taxii_client.populate_available_information() - - if not api_root_name: - raise ValueError("No api_root specified.") - else: - self.api_root = None - for a_r in self.taxii_client.api_roots: - if api_root_name == a_r.name: - self.api_root = a_r - break - if not self.api_root: - raise ValueError("The api_root %s is not found on this taxii server" % api_root_name) - if not collection_id: - raise ValueError("No collection specified.") - else: - self.collection = None - for c in self.api_root.collections: - if c.id_ == collection_id: - self.collection = c - break - if not self.collection: - raise ValueError("The collection %s is not found on the api_root %s of this taxii server" % - (collection_id, api_root_name)) + self.collection = collection def get(self, stix_id, _composite_filters=None): """ @@ -244,54 +166,3 @@ class TAXIICollectionSource(DataSource): taxii_field = "match[" + filter_["field"] + ']' params[taxii_field] = filter_["value"] return params - - # utility functions for the current attached collection and api root - def get_api_root_info(self): - """ - """ - return self.api_root.get_information() - - def get_api_root_collections(self): - """ - """ - return self.api_root.get_collections() - - def get_collection_manifest(self): - """ - """ - return self.collection.get_collection_manifest() - - -def get_server_api_roots(taxii_client): - """ - """ - api_root_info = [] - taxii_client.populate_available_information() - - for api_root in taxii_client.api_roots: - api_root_info.append(api_root.information()) - - return api_root_info - - -def get_server_collections(taxii_client): - """ - """ - server_collections = [] - - taxii_client.populate_available_information() - - for api_root in taxii_client.api_roots: - server_collections.extend(api_root.get_collections()) - - return server_collections - - -def get_api_root_collections(taxii_client, api_root_name): - """ - """ - taxii_client.populate_available_information() - - for api_root in taxii_client.api_roots: - if api_root == api_root_name: - return api_root.get_collections() diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 24adcf9..0ee317f 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,25 +1,37 @@ +import pytest +from taxii2_client import Collection + from stix2.sources import taxii - -def test_ds_taxii(): - ds = taxii.TAXIICollectionSource() - assert ds.name == 'TAXII' +COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' -def test_ds_taxii_name(): - ds = taxii.TAXIICollectionSource(name='My Data Source Name') +class MockTAXIIClient(object): + """Mock for taxii2_client.TAXIIClient""" + + def get(self): + return {} + + def post(self): + return {} + + +@pytest.fixture +def collection(): + return Collection(COLLECTION_URL, MockTAXIIClient()) + + +def test_ds_taxii(collection): + ds = taxii.TAXIICollectionSource(collection) + assert ds.name == 'TAXIICollectionSource' + + +def test_ds_taxii_name(collection): + ds = taxii.TAXIICollectionSource(collection, name='My Data Source Name') assert ds.name == "My Data Source Name" -def test_ds_params(): - url = "http://taxii_url.com:5000" - creds = {"username": "Wade", "password": "Wilson"} - ds = taxii.TAXIICollectionSource(api_root=url, auth=creds) - assert ds.taxii_info['api_root']['url'] == url - assert ds.taxii_info['auth'] == creds - - -def test_parse_taxii_filters(): +def test_parse_taxii_filters(collection): query = [ { "field": "added_after", @@ -55,14 +67,14 @@ def test_parse_taxii_filters(): "match[version]": "first" } - ds = taxii.TAXIICollectionSource() + ds = taxii.TAXIICollectionSource(collection) taxii_filters = ds._parse_taxii_filters(query) assert taxii_filters == expected_params -def test_add_get_remove_filter(): +def test_add_get_remove_filter(collection): class dummy(object): x = 4 @@ -114,7 +126,7 @@ def test_add_get_remove_filter(): "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" ] - ds = taxii.TAXIICollectionSource() + ds = taxii.TAXIICollectionSource(collection) # add ids, statuses = ds.add_filter(filters) @@ -156,7 +168,7 @@ def test_add_get_remove_filter(): assert id_ in ids[:3] -def test_apply_common_filters(): +def test_apply_common_filters(collection): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -210,7 +222,7 @@ def test_apply_common_filters(): } ] - ds = taxii.TAXIICollectionSource() + ds = taxii.TAXIICollectionSource(collection) resp = ds.apply_common_filters(stix_objs, [filters[0]]) ids = [r['id'] for r in resp] @@ -224,7 +236,7 @@ def test_apply_common_filters(): assert resp[0]['id'] == stix_objs[0]['id'] -def test_deduplicate(): +def test_deduplicate(collection): stix_objs = [ { "created": "2017-01-27T13:49:53.935Z", @@ -288,7 +300,7 @@ def test_deduplicate(): } ] - ds = taxii.TAXIICollectionSource() + ds = taxii.TAXIICollectionSource(collection) unique = ds.deduplicate(stix_objs) # Only 3 objects are unique From b8c96e37a20ef5c483b6f86ff983fbeb60aa22bb Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 20 Jul 2017 20:28:21 +0000 Subject: [PATCH 18/82] Update to get tests passing --- stix2/sources/__init__.py | 8 ++++---- stix2/test/test_data_sources.py | 27 +++++++++++---------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 38c8e41..57c2498 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -292,7 +292,7 @@ class DataSource(object): "status": "added as a common filter", "filter": filter_, "data_source_name": self.name, - "data_source_id": self.id_ + "data_source_id": self.id, }) else: self.filter_allowed[id_] = False @@ -301,7 +301,7 @@ class DataSource(object): "filter": filter_, "errors": copy.deepcopy(errors), "data_source_name": self.name, - "data_source_id": self.id_ + "data_source_id": self.id, }) del errors[:] @@ -661,7 +661,7 @@ class CompositeDataSource(object): "status": "added as a common filter", "filter": filter_, "data_source_name": self.name, - "data_source_id": self.id_ + "data_source_id": self.id }) else: self.filter_allowed[id_] = False @@ -670,7 +670,7 @@ class CompositeDataSource(object): "filter": filter_, "errors": errors, "data_source_name": self.name, - "data_source_id": self.id_ + "data_source_id": self.id }) del errors[:] diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0ee317f..872499f 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,7 @@ import pytest from taxii2_client import Collection -from stix2.sources import taxii +from stix2.sources import DataSource, taxii COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -31,7 +31,7 @@ def test_ds_taxii_name(collection): assert ds.name == "My Data Source Name" -def test_parse_taxii_filters(collection): +def test_parse_taxii_filters(): query = [ { "field": "added_after", @@ -74,12 +74,7 @@ def test_parse_taxii_filters(collection): assert taxii_filters == expected_params -def test_add_get_remove_filter(collection): - - class dummy(object): - x = 4 - - obj_1 = dummy() +def test_add_get_remove_filter(): # First 3 filters are valid, remaining fields are erroneous in some way filters = [ @@ -103,7 +98,7 @@ def test_add_get_remove_filter(collection): "value": "filter missing \'op\' field" }, { - "field": "granular_markings", + "field": "description", "op": "=", "value": "not supported field - just place holder" }, @@ -115,7 +110,7 @@ def test_add_get_remove_filter(collection): { "field": "created", "op": "=", - "value": obj_1 + "value": set(), } ] @@ -126,7 +121,7 @@ def test_add_get_remove_filter(collection): "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" ] - ds = taxii.TAXIICollectionSource(collection) + ds = DataSource() # add ids, statuses = ds.add_filter(filters) @@ -168,7 +163,7 @@ def test_add_get_remove_filter(collection): assert id_ in ids[:3] -def test_apply_common_filters(collection): +def test_apply_common_filters(): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -218,11 +213,11 @@ def test_apply_common_filters(collection): { "field": "labels", "op": "in", - "value": "trojan" + "value": "remote-access-trojan" } ] - ds = taxii.TAXIICollectionSource(collection) + ds = DataSource() resp = ds.apply_common_filters(stix_objs, [filters[0]]) ids = [r['id'] for r in resp] @@ -236,7 +231,7 @@ def test_apply_common_filters(collection): assert resp[0]['id'] == stix_objs[0]['id'] -def test_deduplicate(collection): +def test_deduplicate(): stix_objs = [ { "created": "2017-01-27T13:49:53.935Z", @@ -300,7 +295,7 @@ def test_deduplicate(collection): } ] - ds = taxii.TAXIICollectionSource(collection) + ds = DataSource() unique = ds.deduplicate(stix_objs) # Only 3 objects are unique From a4ead4f6e74c138ef83c83a1f9300a4235ff569f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 9 Aug 2017 13:31:07 -0400 Subject: [PATCH 19/82] Formatting changes, skip add/remove filter test, change deduplicate() approach. --- stix2/sources/__init__.py | 34 +++++++++++---------------------- stix2/sources/filesystem.py | 5 ++--- stix2/sources/memory.py | 10 +++++----- stix2/test/test_data_sources.py | 1 + 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 57c2498..5a61339 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -23,10 +23,6 @@ from six import iteritems def make_id(): return str(uuid.uuid4()) - -# STIX 2.0 fields used to denote object version -STIX_VERSION_FIELDS = ['id', 'modified'] - # Currently, only STIX 2.0 common SDO fields (that are not compex objects) # are supported for filtering on STIX_COMMON_FIELDS = [ @@ -705,8 +701,6 @@ class CompositeDataSource(object): def filters(self): """return filters attached to Composite Data Source - Args: - Returns: (list): the list of filters currently attached to the Data Source @@ -727,18 +721,12 @@ class CompositeDataSource(object): (list): unique set of the passed list of STIX objects """ - unique = [] - dont_have = False - for i in stix_obj_list: - dont_have = False - for j in unique: - for field in STIX_VERSION_FIELDS: - if not i[field] == j[field]: - dont_have = True - break - if dont_have: - unique.append(i) - return unique + unique_objs = {} + + for obj in stix_obj_list: + unique_objs[(obj["id"], obj["modified"])] = obj + + return list(unique_objs.values()) class STIXCommonPropertyFilters(): @@ -775,7 +763,7 @@ class STIXCommonPropertyFilters(): return -1 @classmethod - def _boolean(filter_, stix_obj_field): + def _boolean(cls, filter_, stix_obj_field): if filter_["op"] == "=": return stix_obj_field == filter_["value"] elif filter_["op"] == "!=": @@ -802,7 +790,7 @@ class STIXCommonPropertyFilters(): @classmethod def external_references(cls, filter_, stix_obj): - ''' + """ stix object's can have a list of external references external-reference properties: @@ -811,7 +799,7 @@ class STIXCommonPropertyFilters(): external_reference.url (string) external_reference.hashes (hash, but for filtering purposes , a string) external_reference.external_id (string) - ''' + """ for er in stix_obj["external_references"]: # grab er property name from filter field filter_field = filter_["field"].split(".")[1] @@ -822,13 +810,13 @@ class STIXCommonPropertyFilters(): @classmethod def granular_markings(cls, filter_, stix_obj): - ''' + """ stix object's can have a list of granular marking references granular-marking properties: granular-marking.marking_ref (id) granular-marking.selectors (string) - ''' + """ for gm in stix_obj["granular_markings"]: # grab gm property name from filter field filter_field = filter_["field"].split(".")[1] diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 77d5229..9d634c6 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -12,13 +12,12 @@ TODO: Test everything import json import os -from sources import DataSink, DataSource, DataStore, make_id +from stix2.sources import DataSink, DataSource, DataStore, make_id from stix2 import Bundle class FileSystemStore(DataStore): """ - """ def __init__(self, stix_dir="stix_data", name="FileSystemStore"): self.name = name @@ -54,7 +53,7 @@ class FileSystemSink(DataSink): stix_objs = [] for stix_obj in stix_objs: path = os.path.join(self.stix_dir, stix_obj["type"], stix_obj["id"]) - json.dump(Bundle([stix_obj]), open(path, 'w+', indent=4)) + json.dump(Bundle([stix_obj]), open(path, 'w+'), indent=4) class FileSystemSource(DataSource): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 82bcf2a..dbbf479 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -58,7 +58,7 @@ class MemoryStore(DataStore): if r.is_valid: self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) print(r) self.source = MemorySource(stix_data=self.data, _store=True) @@ -112,7 +112,7 @@ class MemorySink(DataSink): if r.is_valid: self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) print(r) else: raise ValueError("stix_data must be in bundle format or raw list") @@ -136,7 +136,7 @@ class MemorySink(DataSink): if r.is_valid: self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) print(r) else: raise ValueError("stix_data must be in bundle format or raw list") @@ -185,7 +185,7 @@ class MemorySource(DataSource): if r.is_valid: self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX object %s is not valid under STIX 2 validator.") % stix_obj["id"] + print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) print(r) else: raise ValueError("stix_data must be in bundle format or raw list") @@ -269,5 +269,5 @@ class MemorySource(DataSource): for stix_obj in stix_data["objects"]: self.data[stix_obj["id"]] = stix_obj else: - print("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator") % file_path + print("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator" % file_path) print(r) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 872499f..b733a19 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -74,6 +74,7 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params +@pytest.skip def test_add_get_remove_filter(): # First 3 filters are valid, remaining fields are erroneous in some way From 87f7503c0a3fd66a0ab83dba5ee1b2751e604cf9 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Wed, 9 Aug 2017 18:49:06 +0000 Subject: [PATCH 20/82] Convert filters to be NamedTuples --- stix2/sources/__init__.py | 246 +++++--------------------------- stix2/test/test_data_sources.py | 109 +++++--------- 2 files changed, 67 insertions(+), 288 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 5a61339..a813a13 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -14,15 +14,28 @@ NOTE: add_filter(), remove_filter(), deduplicate() - if these functions remain make those functions an interface to inherit? """ +import collections import copy import uuid from six import iteritems +class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): + __slots__ = () + + def __new__(cls, field, op, value): + # If value is a list, convert it to a tuple so it is hashable. + if isinstance(value, list): + value = tuple(value) + self = super(Filter, cls).__new__(cls, field, op, value) + return self + + def make_id(): return str(uuid.uuid4()) + # Currently, only STIX 2.0 common SDO fields (that are not compex objects) # are supported for filtering on STIX_COMMON_FIELDS = [ @@ -44,10 +57,6 @@ STIX_COMMON_FIELDS = [ "granular_markings" ] - -# Required fields in filter(dict) -FILTER_FIELDS = ['field', 'op', 'value'] - # Supported filter operations FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] @@ -162,8 +171,7 @@ class DataSource(object): def __init__(self, name="DataSource"): self.name = name self.id = make_id() - self.filters = {} - self.filter_allowed = {} + self.filters = set() def get(self, stix_id, _composite_filters=None): """ @@ -230,114 +238,32 @@ class DataSource(object): raise NotImplementedError() - def add_filter(self, filters): - """add/attach a filter to the Data Source instance + def add_filters(self, filters): + """Add multiple filters to the DataSource. Args: - filters (list): list of filters (dict) to add to the Data Source - - Returns: - status (list): list of status/error messages - + filter (list): list of filters (dict) to add to the Data Source. """ + for filter in filters: + self.add_filter(filter) - status = [] - errors = [] - ids = [] - allowed = True + def add_filter(self, filter): + """Add a filter.""" + # check filter field is a supported STIX 2.0 common field + if filter.field not in STIX_COMMON_FIELDS: + raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - for filter_ in filters: - # check required filter components ("field", "op", "value") exist - for field in FILTER_FIELDS: - if field not in filter_.keys(): - allowed = False - errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") - break + # check filter operator is supported + if filter.op not in FILTER_OPS: + raise ValueError("Filter operation(from 'op' field) not supported") - if allowed: - # no need for further checks if filter is missing parameters + # check filter value type is supported + if type(filter.value) not in FILTER_VALUE_TYPES: + raise ValueError("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - # check filter field is a supported STIX 2.0 common field - if filter_['field'] not in STIX_COMMON_FIELDS: - allowed = False - errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") + self.filters.add(filter) - # check filter operator is supported - if filter_['op'] not in FILTER_OPS: - allowed = False - errors.append("Filter operation(from 'op' field) not supported") - - # check filter value type is supported - if type(filter_['value']) not in FILTER_VALUE_TYPES: - allowed = False - errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - - # Filter is added regardless of whether it fits requirements - # to be a common filter. This is done because some filters - # may be added and used by third party Data Sources, where the - # filtering may be conducted within those plugins, just not here - - id_ = make_id() - filter_['id'] = id_ - self.filters[id_] = filter_ - ids.append(id_) - - if allowed: - self.filter_allowed[id_] = True - status.append({ - "status": "added as a common filter", - "filter": filter_, - "data_source_name": self.name, - "data_source_id": self.id, - }) - else: - self.filter_allowed[id_] = False - status.append({ - "status": "added but is not a common filter", - "filter": filter_, - "errors": copy.deepcopy(errors), - "data_source_name": self.name, - "data_source_id": self.id, - }) - del errors[:] - - allowed = True - - return ids, status - - def remove_filter(self, filter_ids): - """remove/detach a filter from the Data Source instance - - Args: - filter_ids (list): list of filter ids to dettach/remove - from Data Source - - Returns: - - - """ - for filter_id in filter_ids: - try: - if filter_id in self.filters: - del self.filters[filter_id] - del self.filter_allowed[filter_id] - except KeyError: - # filter 'id' not found list of filters attached to Data Source - pass - - return - - def get_filters(self): - """return copy of all filters currently attached to Data Source - - TODO: make this a property? - - Returns: - (list): a copy of all the filters(dict) which are attached - to Data Source - - """ - return copy.deepcopy(list(self.filters.values())) + # TODO: Do we need a remove_filter function? def apply_common_filters(self, stix_objs, query): """evaluates filters against a set of STIX 2.0 objects @@ -599,114 +525,6 @@ class CompositeDataSource(object): """ return copy.deepcopy(self.data_sources.values()) - def add_filter(self, filters): - """add/attach a filter to the Composite Data Source instance - - Args: - filters (list): list of filters (dict) to add to the Data Source - - Returns: - status (list): list of status/error messages - - """ - - status = [] - errors = [] - ids = [] - allowed = True - - for filter_ in filters: - # check required filter components ("field", "op", "value") exist - for field in FILTER_FIELDS: - if field not in filter_.keys(): - allowed = False - errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.") - break - - if allowed: - # no need for further checks if filter is missing parameters - - # check filter field is a supported STIX 2.0 common field - if filter_['field'] not in STIX_COMMON_FIELDS: - allowed = False - errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") - - # check filter operator is supported - if filter_['op'] not in FILTER_OPS: - allowed = False - errors.append("Filter operation(from 'op' field) not supported") - - # check filter value type is supported - if type(filter_['value']) not in FILTER_VALUE_TYPES: - allowed = False - errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - - # Filter is added regardless of whether it fits requirements - # to be a common filter. This is done because some filters - # may be added and used by third party Data Sources, where the - # filtering may be conducted within those plugins, just not here - - id_ = make_id() - filter_['id'] = id_ - self.filters['id_'] = filter_ - ids.append(id_) - - if allowed: - self.filter_allowed[id_] = True - status.append({ - "status": "added as a common filter", - "filter": filter_, - "data_source_name": self.name, - "data_source_id": self.id - }) - else: - self.filter_allowed[id_] = False - status.append({ - "status": "added but is not a common filter", - "filter": filter_, - "errors": errors, - "data_source_name": self.name, - "data_source_id": self.id - }) - del errors[:] - - allowed = True - - return ids, status - - def remove_filter(self, filter_ids): - """Remove/detach a filter from the Data Source instance - - Args: - filter_ids (list): list of filter id's (which are strings) - dettach from the Composite Data Source - - Returns: - - """ - - for filter_id in filter_ids: - try: - if filter_id in self.filters: - del self.filters[filter_id] - del self.filter_allowed[filter_id] - except KeyError: - # filter id not found in list of filters - # attached to the Composite Data Source - pass - - return - - @property - def filters(self): - """return filters attached to Composite Data Source - - Returns: - (list): the list of filters currently attached to the Data Source - - """ - return copy.deepcopy(list(self.filters.values())) - def deduplicate(self, stix_obj_list): """deduplicate a list fo STIX objects to a unique set diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index b733a19..54fde11 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,7 @@ import pytest from taxii2_client import Collection -from stix2.sources import DataSource, taxii +from stix2.sources import DataSource, Filter, taxii COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -74,94 +74,55 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -@pytest.skip def test_add_get_remove_filter(): # First 3 filters are valid, remaining fields are erroneous in some way - filters = [ - { - "field": "type", - "op": '=', - "value": "malware" - }, - { - "field": "id", - "op": "!=", - "value": "stix object id" - }, - { - "field": "labels", - "op": "in", - "value": ["heartbleed", "malicious-activity"] - }, - { - "field": "revoked", - "value": "filter missing \'op\' field" - }, - { - "field": "description", - "op": "=", - "value": "not supported field - just place holder" - }, - { - "field": "modified", - "op": "*", - "value": "not supported operator - just place holder" - }, - { - "field": "created", - "op": "=", - "value": set(), - } + valid_filters = [ + Filter('type', '=', 'malware'), + Filter('id', '!=', 'stix object id'), + Filter('labels', 'in', ["heartbleed", "malicious-activity"]), ] - - expected_errors = [ - "Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.", - "Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported", - "Filter operation(from 'op' field) not supported", - "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" + invalid_filters = [ + Filter('description', '=', 'not supported field - just place holder'), + Filter('modified', '*', 'not supported operator - just place holder'), + Filter('created', '=', object()), ] ds = DataSource() - # add - ids, statuses = ds.add_filter(filters) - # 7 filters should have been successfully added - assert len(ids) == 7 + assert len(ds.filters) == 0 - # all filters added to data source - for idx, status in enumerate(statuses): - assert status['filter'] == filters[idx] + ds.add_filter(valid_filters[0]) + assert len(ds.filters) == 1 - # proper status warnings were triggered - assert statuses[3]['errors'][0] == expected_errors[0] - assert statuses[4]['errors'][0] == expected_errors[1] - assert statuses[5]['errors'][0] == expected_errors[2] - assert statuses[6]['errors'][0] == expected_errors[3] + # Addin the same filter again will have no effect since `filters` uses a set + ds.add_filter(valid_filters[0]) + assert len(ds.filters) == 1 - # get - ds_filters = ds.get_filters() + ds.add_filter(valid_filters[1]) + assert len(ds.filters) == 2 + ds.add_filter(valid_filters[2]) + assert len(ds.filters) == 3 - # TODO: what are we trying to test here? - for idx, flt in enumerate(filters): - assert flt['value'] == ds_filters[idx]['value'] + # TODO: make better error messages + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[0]) + assert str(excinfo.value) == "Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported" + + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[1]) + assert str(excinfo.value) == "Filter operation(from 'op' field) not supported" + + with pytest.raises(ValueError) as excinfo: + ds.add_filter(invalid_filters[2]) + assert str(excinfo.value) == "Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary" + + assert set(valid_filters) == ds.filters # remove - ds.remove_filter([ids[3]]) - ds.remove_filter([ids[4]]) - ds.remove_filter([ids[5]]) - ds.remove_filter([ids[6]]) + ds.filters.remove(valid_filters[0]) - rem_filters = ds.get_filters() - - assert len(rem_filters) == 3 - - # check remaining filters - rem_ids = [f['id'] for f in rem_filters] - - # check remaining - for id_ in rem_ids: - assert id_ in ids[:3] + assert len(ds.filters) == 2 def test_apply_common_filters(): From 961dfdc984a2052c8feb75b2348d5e492eecca18 Mon Sep 17 00:00:00 2001 From: Greg Back Date: Wed, 9 Aug 2017 19:25:06 +0000 Subject: [PATCH 21/82] Convert rest of code to use namedtuple Filters --- stix2/sources/__init__.py | 54 ++++++++++++++++----------------- stix2/sources/filesystem.py | 22 +++++++------- stix2/sources/taxii.py | 10 +++--- stix2/test/test_data_sources.py | 48 +++++------------------------ 4 files changed, 51 insertions(+), 83 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index a813a13..b2b4109 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -289,17 +289,17 @@ class DataSource(object): # skip filter as filter was identified (when added) as # not a common filter - if 'id' in filter_ and self.filter_allowed[filter_['id']] is False: + if filter_.field not in STIX_COMMON_FIELDS: continue # check filter "field" is in STIX object - if cant be applied # due to STIX object, STIX object is discarded (i.e. did not # make it through the filter) - if filter_['field'] not in stix_obj.keys(): + if filter_.field not in stix_obj.keys(): clean = False break try: - match = getattr(STIXCommonPropertyFilters, filter_['field'])(filter_, stix_obj) + match = getattr(STIXCommonPropertyFilters, filter_.field)(filter_, stix_obj) if not match: clean = False break @@ -553,39 +553,39 @@ class STIXCommonPropertyFilters(): @classmethod def _all(cls, filter_, stix_obj_field): """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_["op"] == '=': - return stix_obj_field == filter_["value"] - elif filter_["op"] == "!=": - return stix_obj_field != filter_["value"] - elif filter_["op"] == "in": - return stix_obj_field in filter_["value"] - elif filter_["op"] == ">": - return stix_obj_field > filter_["value"] - elif filter_["op"] == "<": - return stix_obj_field < filter_["value"] - elif filter_["op"] == ">=": - return stix_obj_field >= filter_["value"] - elif filter_["op"] == "<=": - return stix_obj_field <= filter_["value"] + if filter_.op == '=': + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value + elif filter_.op == "in": + return stix_obj_field in filter_.value + elif filter_.op == ">": + return stix_obj_field > filter_.value + elif filter_.op == "<": + return stix_obj_field < filter_.value + elif filter_.op == ">=": + return stix_obj_field >= filter_.value + elif filter_.op == "<=": + return stix_obj_field <= filter_.value else: return -1 @classmethod def _id(cls, filter_, stix_obj_id): """base filter types""" - if filter_["op"] == "=": - return stix_obj_id == filter_["value"] - elif filter_["op"] == "!=": - return stix_obj_id != filter_["value"] + if filter_.op == "=": + return stix_obj_id == filter_.value + elif filter_.op == "!=": + return stix_obj_id != filter_.value else: return -1 @classmethod def _boolean(cls, filter_, stix_obj_field): - if filter_["op"] == "=": - return stix_obj_field == filter_["value"] - elif filter_["op"] == "!=": - return stix_obj_field != filter_["value"] + if filter_.op == "=": + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value else: return -1 @@ -620,7 +620,7 @@ class STIXCommonPropertyFilters(): """ for er in stix_obj["external_references"]: # grab er property name from filter field - filter_field = filter_["field"].split(".")[1] + filter_field = filter_.field.split(".")[1] r = cls._string(filter_, er[filter_field]) if r: return r @@ -637,7 +637,7 @@ class STIXCommonPropertyFilters(): """ for gm in stix_obj["granular_markings"]: # grab gm property name from filter field - filter_field = filter_["field"].split(".")[1] + filter_field = filter_.field.split(".")[1] if filter_field == "marking_ref": return cls._id(filter_, gm[filter_field]) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 9d634c6..e08c0be 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -12,8 +12,8 @@ TODO: Test everything import json import os -from stix2.sources import DataSink, DataSource, DataStore, make_id from stix2 import Bundle +from stix2.sources import DataSink, DataSource, DataStore, make_id class FileSystemStore(DataStore): @@ -136,13 +136,13 @@ class FileSystemSource(DataSource): # the corresponding subdirectories as well include_paths = [] declude_paths = [] - if "type" in [filter_["field"] for filter_ in file_filters]: + if "type" in [filter_.field for filter_ in file_filters]: for filter_ in file_filters: - if filter_["field"] == "type": - if filter_["op"] == '=': - include_paths.append(os.path.join(self.stix_dir, filter_["value"])) - elif filter_["op"] == "!=": - declude_paths.append(os.path.join(self.stix_dir, filter_["value"])) + if filter_.field == "type": + if filter_.op == '=': + include_paths.append(os.path.join(self.stix_dir, filter_.value)) + elif filter_.op == "!=": + declude_paths.append(os.path.join(self.stix_dir, filter_.value)) else: # have to walk entire STIX directory include_paths.append(self.stix_dir) @@ -165,10 +165,10 @@ class FileSystemSource(DataSource): # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory - if "id" in [filter_["field"] for filter_ in file_filters]: + if "id" in [filter_.field for filter_ in file_filters]: for filter_ in file_filters: - if filter_["field"] == "id" and filter_["field"] == '=': - id_ = filter_["value"] + if filter_.field == "id" and filter_.field == '=': + id_ = filter_.value else: id_ = None @@ -196,6 +196,6 @@ class FileSystemSource(DataSource): """ file_filters = [] for filter_ in query: - if filter_["field"] == "id" or filter_["field"] == "type": + if filter_.field == "id" or filter_.field == "type": file_filters.append(filter_) return file_filters diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 581a3d5..1e3fb30 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -159,10 +159,10 @@ class TAXIICollectionSource(DataSource): params = {} for filter_ in query: - if filter_["field"] in TAXII_FILTERS: - if filter_["field"] == "added_after": - params[filter_["field"]] = filter_["value"] + if filter_.field in TAXII_FILTERS: + if filter_.field == "added_after": + params[filter_.field] = filter_.value else: - taxii_field = "match[" + filter_["field"] + ']' - params[taxii_field] = filter_["value"] + taxii_field = "match[" + filter_.field + ']' + params[taxii_field] = filter_.value return params diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 54fde11..bbb7f31 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -33,31 +33,11 @@ def test_ds_taxii_name(collection): def test_parse_taxii_filters(): query = [ - { - "field": "added_after", - "op": "=", - "value": "2016-02-01T00:00:01.000Z" - }, - { - "field": "id", - "op": "=", - "value": "taxii stix object ID" - }, - { - "field": "type", - "op": "=", - "value": "taxii stix object ID" - }, - { - "field": "version", - "op": "=", - "value": "first" - }, - { - "field": "created_by_ref", - "op": "=", - "value": "Bane" - } + Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), + Filter("id", "=", "taxii stix object ID"), + Filter("type", "=", "taxii stix object ID"), + Filter("version", "=", "first"), + Filter("created_by_ref", "=", "Bane"), ] expected_params = { @@ -162,21 +142,9 @@ def test_apply_common_filters(): ] filters = [ - { - "field": "type", - "op": "!=", - "value": "relationship" - }, - { - "field": "id", - "op": "=", - "value": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" - }, - { - "field": "labels", - "op": "in", - "value": "remote-access-trojan" - } + Filter("type", "!=", "relationship"), + Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), + Filter("labels", "in", "remote-access-trojan"), ] ds = DataSource() From 86fd3778f50cee3530db55ba62b0b7e20a24ba16 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 08:10:20 -0400 Subject: [PATCH 22/82] Formatting changes, replace deduplicate() code in DataSource, missing super() calls to initialize objects. --- stix2/sources/__init__.py | 211 ++++++++++++++++---------------- stix2/sources/filesystem.py | 20 +-- stix2/sources/memory.py | 57 +++++---- stix2/sources/taxii.py | 28 ++--- stix2/test/test_data_sources.py | 2 +- 5 files changed, 162 insertions(+), 156 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 5a61339..e49b469 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -9,9 +9,11 @@ Classes: TODO:Test everything -NOTE: add_filter(), remove_filter(), deduplicate() - if these functions remain - the exact same for DataSource, DataSink, CompositeDataSource etc... -> just - make those functions an interface to inherit? +Notes: + add_filter(), remove_filter(), deduplicate() - if these functions remain + the exact same for DataSource, DataSink, CompositeDataSource etc... -> just + make those functions an interface to inherit? + """ import copy @@ -23,7 +25,8 @@ from six import iteritems def make_id(): return str(uuid.uuid4()) -# Currently, only STIX 2.0 common SDO fields (that are not compex objects) + +# Currently, only STIX 2.0 common SDO fields (that are not complex objects) # are supported for filtering on STIX_COMMON_FIELDS = [ "created", @@ -59,6 +62,7 @@ class DataStore(object): """ An implementer will create a concrete subclass from this abstract class for the specific data store. + """ def __init__(self, name="DataStore"): self.name = name @@ -69,29 +73,25 @@ class DataStore(object): def get(self, stix_id): """ Implement: - -translate API get() call to the appropriate DataSource call + Translate API get() call to the appropriate DataSource call Args: - stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (list): list of filters passed along from - the Composite Data Filter. - Returns: stix_obj (dictionary): the STIX object to be returned + """ return self.source.get(stix_id=stix_id) def all_versions(self, stix_id): """ Implement: - -translate all_versions() call to the appropriate DataSource call + Translate all_versions() call to the appropriate DataSource call Args: - stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". @@ -102,21 +102,19 @@ class DataStore(object): Returns: stix_objs (list): a list of STIX objects (where each object is a STIX object) + """ return self.source.all_versions(stix_id=stix_id) def query(self, query): """ Fill: - -implement the specific data source API calls, processing, + Implement the specific data source API calls, processing, functionality required for retrieving query from the data source Args: query (list): a list of filters (which collectively are the query) - to conduct search on - - _composite_filters (list): a list of filters passed from the - Composite Data Source + to conduct search on. Returns: stix_objs (list): a list of STIX objects (where each object is a @@ -136,8 +134,13 @@ class DataStore(object): class DataSink(object): """ - An implementer will create a concrete subclass from this - abstract class for the specific data sink. + Abstract class for defining a data sink. Intended for subclassing into + different sink components. + + Attributes: + id (str): A unique UUIDv4 to identify this DataSink. + name (str): The descriptive name that identifies this DataSink. + """ def __init__(self, name="DataSink"): @@ -147,16 +150,25 @@ class DataSink(object): def add(self, stix_objs): """ Fill: - -implement the specific data sink API calls, processing, + Implement the specific data sink API calls, processing, functionality required for adding data to the sink + """ raise NotImplementedError() class DataSource(object): """ - An implementer will create a concrete subclass from - this abstract class for the specific data source. + Abstract class for defining a data source. Intended for subclassing into + different source components. + + Attributes: + id (str): A unique UUIDv4 to identify this DataSource. + name (str): The descriptive name that identifies this DataSource. + filters (dict): A collection of filters present in this DataSource. + filter_allowed (dict): A collection of the allowed filters in this + DataSource. + """ def __init__(self, name="DataSource"): @@ -168,11 +180,10 @@ class DataSource(object): def get(self, stix_id, _composite_filters=None): """ Fill: - -implement the specific data source API calls, processing, + Implement the specific data source API calls, processing, functionality required for retrieving data from the data source Args: - stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". @@ -196,7 +207,7 @@ class DataSource(object): functionality required for retrieving data from the data source Args: - id (str): The id of the STIX 2.0 object to retrieve. Should + stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". @@ -206,8 +217,8 @@ class DataSource(object): Returns: stix_objs (list): a list of STIX objects (where each object is a STIX object) - """ + """ raise NotImplementedError() def query(self, query, _composite_filters=None): @@ -225,13 +236,11 @@ class DataSource(object): Returns: - """ - raise NotImplementedError() def add_filter(self, filters): - """add/attach a filter to the Data Source instance + """Add/attach a filter to the Data Source instance Args: filters (list): list of filters (dict) to add to the Data Source @@ -240,14 +249,13 @@ class DataSource(object): status (list): list of status/error messages """ - status = [] errors = [] ids = [] allowed = True for filter_ in filters: - # check required filter components ("field", "op", "value") exist + # check required filter components ('field', 'op', 'value') exist for field in FILTER_FIELDS: if field not in filter_.keys(): allowed = False @@ -306,14 +314,11 @@ class DataSource(object): return ids, status def remove_filter(self, filter_ids): - """remove/detach a filter from the Data Source instance + """Remove/detach a filter from the Data Source instance Args: - filter_ids (list): list of filter ids to dettach/remove - from Data Source - - Returns: - + filter_ids (list): list of filter ids to detach/remove + from Data Source. """ for filter_id in filter_ids: @@ -328,7 +333,7 @@ class DataSource(object): return def get_filters(self): - """return copy of all filters currently attached to Data Source + """Return copy of all filters currently attached to Data Source TODO: make this a property? @@ -340,7 +345,7 @@ class DataSource(object): return copy.deepcopy(list(self.filters.values())) def apply_common_filters(self, stix_objs, query): - """evaluates filters against a set of STIX 2.0 objects + """Evaluates filters against a set of STIX 2.0 objects Supports only STIX 2.0 common property fields @@ -350,10 +355,9 @@ class DataSource(object): Returns: (list): list of STIX objects that successfully evaluate against - the query + the query. """ - filtered_stix_objs = [] # evaluate objects against filter @@ -390,9 +394,9 @@ class DataSource(object): return filtered_stix_objs def deduplicate(self, stix_obj_list): - """deduplicate a list of STIX objects into a unique set + """Deduplicate a list of STIX objects to a unique set - reduces a set of STIX objects to unique set by looking + Reduces a set of STIX objects to unique set by looking at 'id' and 'modified' fields - as a unique object version is determined by the combination of those fields @@ -400,30 +404,34 @@ class DataSource(object): stix_obj_list (list): list of STIX objects (dicts) Returns: - (list): a unique set of the passed STIX object list - + A list with a unique set of the passed list of STIX objects. """ - unique = [] - have = False - for i in stix_obj_list: - for j in unique: - if i['id'] == j['id'] and i['modified'] == j['modified']: - have = True - break - if not have: - unique.append(i) - have = False - return unique + unique_objs = {} + + for obj in stix_obj_list: + unique_objs[(obj['id'], obj['modified'])] = obj + + return list(unique_objs.values()) class CompositeDataSource(object): """Composite Data Source Acts as a controller for all the defined/configured STIX Data Sources - e.g. a user can defined n Data Sources - creating Data Source (objects) + e.g. a user can define n Data Sources - creating Data Source (objects) for each. There is only one instance of this for any python STIX 2.0 - application + application. + + Attributes: + id (str): A UUIDv4 to identify this CompositeDataSource. + name (str): The name that identifies this CompositeDataSource. + data_sources (dict): A dictionary of DataSource objects; to be + controlled and used by the Data Source Controller object. + filters (dict): A collection of filters present in this + CompositeDataSource. + filter_allowed (dict): A collection of the allowed filters in this + CompositeDataSource. """ def __init__(self, name="CompositeDataSource"): @@ -431,11 +439,9 @@ class CompositeDataSource(object): Creates a new STIX Data Source. Args: - 'data_sources' (dict): a dict of DataSource objects; to be - controlled and used by the Data Source Controller object + name (str): A string containing the name to attach in the + CompositeDataSource instance. - filters : - name : """ self.id = make_id() self.name = name @@ -446,23 +452,23 @@ class CompositeDataSource(object): def get(self, stix_id): """Retrieve STIX object by 'id' - federated retrieve method-iterates through all STIX data sources + Federated retrieve method-iterates through all STIX data sources defined in the "data_sources" parameter. Each data source has a specific API retrieve-like function and associated parameters. This function does a federated retrieval and consolidation of the data returned from all the STIX data sources. - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + Notes: + A composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle. Args: - id (str): the id of the STIX object to retrieve + stix_id (str): the id of the STIX object to retrieve. Returns: - stix_obj (dict): the STIX object to be returned + stix_obj (dict): the STIX object to be returned. """ - all_data = [] # for every configured Data Source, call its retrieve handler @@ -485,14 +491,16 @@ class CompositeDataSource(object): Federated all_versions retrieve method - iterates through all STIX data sources defined in "data_sources" - note: a composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + Notes: + A composite data source will pass its attached filters to + each configured data source, pushing filtering to them to handle Args: - id_ (str): id of the STIX objects to retrieve + stix_id (str): id of the STIX objects to retrieve Returns: all_data (list): list of STIX objects that have the specified id + """ all_data = [] @@ -509,10 +517,10 @@ class CompositeDataSource(object): return all_data def query(self, query=None): - """composite data source query + """Composite data source query - Federate the query to all Data Sources attached - to the Composite Data Source + Federate the query to all Data Sources attached to the + Composite Data Source. Args: query (list): list of filters to search on @@ -540,16 +548,13 @@ class CompositeDataSource(object): return all_data def add_data_source(self, data_sources): - """add/attach Data Source to the Composite Data Source instance + """Add/attach Data Source to the Composite Data Source instance Args: data_sources (list): a list of Data Source objects to attach to the Composite Data Source - Returns: - """ - for ds in data_sources: if issubclass(ds, DataSource): if self.data_sources[ds['id']] in self.data_sources.keys(): @@ -568,7 +573,7 @@ class CompositeDataSource(object): return def remove_data_source(self, data_source_ids): - """remove/detach Data Source from the Composite Data Source instance + """Remove/detach Data Source from the Composite Data Source instance Args: data_source_ids (list): a list of Data Source @@ -590,17 +595,13 @@ class CompositeDataSource(object): @property def data_sources(self): - """return all attached Data Sources - - Args: - - Returns: + """Return all attached Data Sources """ return copy.deepcopy(self.data_sources.values()) def add_filter(self, filters): - """add/attach a filter to the Composite Data Source instance + """Add/attach a filter to the Composite Data Source instance Args: filters (list): list of filters (dict) to add to the Data Source @@ -609,7 +610,6 @@ class CompositeDataSource(object): status (list): list of status/error messages """ - status = [] errors = [] ids = [] @@ -679,12 +679,9 @@ class CompositeDataSource(object): Args: filter_ids (list): list of filter id's (which are strings) - dettach from the Composite Data Source - - Returns: + detach from the Composite Data Source. """ - for filter_id in filter_ids: try: if filter_id in self.filters: @@ -699,7 +696,7 @@ class CompositeDataSource(object): @property def filters(self): - """return filters attached to Composite Data Source + """Return filters attached to Composite Data Source Returns: (list): the list of filters currently attached to the Data Source @@ -708,7 +705,7 @@ class CompositeDataSource(object): return copy.deepcopy(list(self.filters.values())) def deduplicate(self, stix_obj_list): - """deduplicate a list fo STIX objects to a unique set + """Deduplicate a list of STIX objects to a unique set Reduces a set of STIX objects to unique set by looking at 'id' and 'modified' fields - as a unique object version @@ -718,9 +715,9 @@ class CompositeDataSource(object): stix_obj_list (list): list of STIX objects (dicts) Returns: - (list): unique set of the passed list of STIX objects - """ + A list with a unique set of the passed list of STIX objects. + """ unique_objs = {} for obj in stix_obj_list: @@ -729,13 +726,13 @@ class CompositeDataSource(object): return list(unique_objs.values()) -class STIXCommonPropertyFilters(): +class STIXCommonPropertyFilters(object): """ """ @classmethod def _all(cls, filter_, stix_obj_field): """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_["op"] == '=': + if filter_["op"] == "=": return stix_obj_field == filter_["value"] elif filter_["op"] == "!=": return stix_obj_field != filter_["value"] @@ -791,14 +788,15 @@ class STIXCommonPropertyFilters(): @classmethod def external_references(cls, filter_, stix_obj): """ - stix object's can have a list of external references + STIX object's can have a list of external references + + external_references properties: + external_references.source_name (string) + external_references.description (string) + external_references.url (string) + external_references.hashes (hash, but for filtering purposes, a string) + external_references.external_id (string) - external-reference properties: - external_reference.source_name (string) - external_reference.description (string) - external_reference.url (string) - external_reference.hashes (hash, but for filtering purposes , a string) - external_reference.external_id (string) """ for er in stix_obj["external_references"]: # grab er property name from filter field @@ -811,11 +809,12 @@ class STIXCommonPropertyFilters(): @classmethod def granular_markings(cls, filter_, stix_obj): """ - stix object's can have a list of granular marking references + STIX object's can have a list of granular marking references + + granular_markings properties: + granular_markings.marking_ref (id) + granular_markings.selectors (string) - granular-marking properties: - granular-marking.marking_ref (id) - granular-marking.selectors (string) """ for gm in stix_obj["granular_markings"]: # grab gm property name from filter field diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 9d634c6..39f7c52 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -12,16 +12,15 @@ TODO: Test everything import json import os -from stix2.sources import DataSink, DataSource, DataStore, make_id from stix2 import Bundle +from stix2.sources import DataSink, DataSource, DataStore class FileSystemStore(DataStore): """ """ def __init__(self, stix_dir="stix_data", name="FileSystemStore"): - self.name = name - self.id = make_id() + super(FileSystemStore, self).__init__(name=name) self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) @@ -94,8 +93,11 @@ class FileSystemSource(DataSource): def all_versions(self, stix_id, _composite_filters=None): """ - NOTE: since FileSystem sources/sinks dont handle mutliple verions of a STIX object, - this operation is futile. Pass call to get(). (Appoved by G.B.) + Notes: + Since FileSystem sources/sinks don't handle multiple versions + of a STIX object, this operation is futile. Pass call to get(). + (Approved by G.B.) + """ # query = [ @@ -139,7 +141,7 @@ class FileSystemSource(DataSource): if "type" in [filter_["field"] for filter_ in file_filters]: for filter_ in file_filters: if filter_["field"] == "type": - if filter_["op"] == '=': + if filter_["op"] == "=": include_paths.append(os.path.join(self.stix_dir, filter_["value"])) elif filter_["op"] == "!=": declude_paths.append(os.path.join(self.stix_dir, filter_["value"])) @@ -167,8 +169,11 @@ class FileSystemSource(DataSource): # may forgo the loading of STIX content into memory if "id" in [filter_["field"] for filter_ in file_filters]: for filter_ in file_filters: - if filter_["field"] == "id" and filter_["field"] == '=': + if filter_["field"] == "id" and filter_["op"] == "=": id_ = filter_["value"] + break + else: + id_ = None else: id_ = None @@ -188,7 +193,6 @@ class FileSystemSource(DataSource): all_data.extend(self.apply_common_filters([stix_obj], query)) all_data = self.deduplicate(all_data) - return all_data def _parse_file_filters(self, query): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index dbbf479..24f3c1f 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -11,9 +11,10 @@ TODO: Test everything. TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) can save a lot of time for successive queries -NOTE: Not worrying about STIX versioning. The in memory STIX data at anytime - will only hold one version of a STIX object. As such, when save() is called, - the single versions of all the STIX objects are what is written to file. +Notes: + Not worrying about STIX versioning. The in memory STIX data at anytime + will only hold one version of a STIX object. As such, when save() is called, + the single versions of all the STIX objects are what is written to file. """ @@ -21,7 +22,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources import DataSink, DataSource, DataStore from stix2validator import validate_string @@ -30,12 +31,13 @@ class MemoryStore(DataStore): """ def __init__(self, stix_data=None, name="MemoryStore"): """ - Note: It doesnt make sense to create a MemoryStore by passing - in existing MemorySource and MemorySink because there could - be data concurrency issues. Just as easy to create new MemoryStore. + Notes: + It doesn't make sense to create a MemoryStore by passing + in existing MemorySource and MemorySink because there could + be data concurrency issues. Just as easy to create new MemoryStore. + """ - self.name = name - self.id = make_id() + super(MemoryStore, self).__init__(name=name) self.data = {} if stix_data: @@ -46,7 +48,6 @@ class MemoryStore(DataStore): # make dictionary of the objects for easy lookup if r.is_valid: for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj else: print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") @@ -73,16 +74,16 @@ class MemoryStore(DataStore): class MemorySink(DataSink): """ - """ def __init__(self, stix_data=None, name="MemorySink", _store=False): """ Args: - - data (dictionary OR list): valid STIX 2.0 content in bundle or a list + stix_data (dictionary OR list): valid STIX 2.0 content in + bundle or a list. name (string): optional name tag of the data source - _store (bool): if the MemorySink is a part of a DataStore, in which case - "stix_data" is a direct reference to shared memory with DataSource + _store (bool): if the MemorySink is a part of a DataStore, + in which case "stix_data" is a direct reference to + shared memory with DataSource. """ super(MemorySink, self).__init__(name=name) @@ -152,11 +153,12 @@ class MemorySource(DataSource): def __init__(self, stix_data=None, name="MemorySource", _store=False): """ Args: - - data (dictionary OR list): valid STIX 2.0 content in bundle or list - name (string): optional name tag of the data source - _store (bool): if the MemorySource is a part of a DataStore, in which case - "stix_data" is a direct reference to shared memory with DataSink + stix_data (dictionary OR list): valid STIX 2.0 content in + bundle or list. + name (string): optional name tag of the data source. + _store (bool): if the MemorySource is a part of a DataStore, + in which case "stix_data" is a direct reference to shared + memory with DataSink. """ super(MemorySource, self).__init__(name=name) @@ -167,7 +169,7 @@ class MemorySource(DataSource): self.data = {} if stix_data: if type(stix_data) == dict: - # stix objects are in a bundle + # STIX objects are in a bundle # verify STIX json data r = validate_string(json.dumps(stix_data)) # make dictionary of the objects for easy lookup @@ -179,7 +181,7 @@ class MemorySource(DataSource): print(r) self.data = {} elif type(stix_data) == list: - # stix objects are in a list + # STIX objects are in a list for stix_obj in stix_data: r = validate_string(json.dumps(stix_obj)) if r.is_valid: @@ -219,8 +221,11 @@ class MemorySource(DataSource): def all_versions(self, stix_id, _composite_filters=None): """ - NOTE: since Memory sources/sinks dont handle mutliple verions of a STIX object, - this operation is futile. Translate call to get(). (Appoved by G.B.) + Notes: + Since Memory sources/sinks don't handle multiple versions of a + STIX object, this operation is futile. Translate call to get(). + (Approved by G.B.) + """ # query = [ @@ -237,9 +242,7 @@ class MemorySource(DataSource): def query(self, query=None, _composite_filters=None): """ - """ - if query is None: query = [] @@ -250,7 +253,7 @@ class MemorySource(DataSource): query.extend(_composite_filters) # deduplicate data before filtering -> Deduplication is not required as Memory only ever holds one version of an object - # all_data = self.depuplicate(all_data) + # all_data = self.deduplicate(all_data) # apply STIX common property filters all_data = self.apply_common_filters(self.data.values(), query) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 581a3d5..2a567c7 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -11,7 +11,6 @@ TODO: Test everything """ import json -import uuid from stix2.sources import DataSink, DataSource, DataStore, make_id @@ -27,8 +26,8 @@ class TAXIICollectionStore(DataStore): Args: collection (taxii2.Collection): Collection instance - """ + """ self.name = name self.id = make_id() self.source = TAXIICollectionSource(collection) @@ -38,7 +37,6 @@ class TAXIICollectionStore(DataStore): class TAXIICollectionSink(DataSink): """ """ - def __init__(self, collection, name="TAXIICollectionSink"): super(TAXIICollectionSink, self).__init__(name=name) @@ -51,7 +49,7 @@ class TAXIICollectionSink(DataSink): @staticmethod def create_bundle(objects): - return dict(id="bundle--" + str(uuid.uuid4()), + return dict(id="bundle--%s" % make_id(), objects=objects, spec_version="2.0", type="bundle") @@ -137,15 +135,17 @@ class TAXIICollectionSource(DataSource): return all_data def _parse_taxii_filters(self, query): - """Parse out TAXII filters that the TAXII server can filter on + """Parse out TAXII filters that the TAXII server can filter on. - For instance - "?match[type]=indicator,sighting" should be in a query dict as follows - { - "field": "type" - "op": "=", - "value": "indicator,sighting" - } + Notes: + For instance - "?match[type]=indicator,sighting" should be in a + query dict as follows: + + { + "field": "type" + "op": "=", + "value": "indicator,sighting" + } Args: query (list): list of filters to extract which ones are TAXII @@ -154,8 +154,8 @@ class TAXIICollectionSource(DataSource): Returns: params (dict): dict of the TAXII filters but in format required for 'requests.get()'. - """ + """ params = {} for filter_ in query: @@ -163,6 +163,6 @@ class TAXIICollectionSource(DataSource): if filter_["field"] == "added_after": params[filter_["field"]] = filter_["value"] else: - taxii_field = "match[" + filter_["field"] + ']' + taxii_field = "match[%s]" % filter_["field"] params[taxii_field] = filter_["value"] return params diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index b733a19..f318e38 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -74,7 +74,7 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -@pytest.skip +@pytest.mark.skip(reason="test_add_get_remove_filter() - Determine what are we testing.") def test_add_get_remove_filter(): # First 3 filters are valid, remaining fields are erroneous in some way From e52575e01a45538899a97be3758225a5a1b575d8 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 09:52:29 -0400 Subject: [PATCH 23/82] Add missing call to super() in TAXIICollectionStore. --- stix2/sources/taxii.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 2a567c7..2b25e0d 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -28,8 +28,7 @@ class TAXIICollectionStore(DataStore): collection (taxii2.Collection): Collection instance """ - self.name = name - self.id = make_id() + super(TAXIICollectionStore, self).__init__(name=name) self.source = TAXIICollectionSource(collection) self.sink = TAXIICollectionSink(collection) @@ -39,7 +38,6 @@ class TAXIICollectionSink(DataSink): """ def __init__(self, collection, name="TAXIICollectionSink"): super(TAXIICollectionSink, self).__init__(name=name) - self.collection = collection def add(self, stix_obj): @@ -60,7 +58,6 @@ class TAXIICollectionSource(DataSource): """ def __init__(self, collection, name="TAXIICollectionSource"): super(TAXIICollectionSource, self).__init__(name=name) - self.collection = collection def get(self, stix_id, _composite_filters=None): From 56d8ca49392a98602a5321e3766708512c98da12 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 15:04:58 -0400 Subject: [PATCH 24/82] Missing instantiation of STIXdatetime in parser. Added __repr__() to STIXdatetime class. --- stix2/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stix2/utils.py b/stix2/utils.py index 12b889c..602ff7a 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -24,6 +24,9 @@ class STIXdatetime(dt.datetime): self.precision = precision return self + def __repr__(self): + return "'%s'" % format_datetime(self) + def get_timestamp(): return STIXdatetime.now(tz=pytz.UTC) @@ -77,7 +80,7 @@ def parse_into_datetime(value, precision=None): # Ensure correct precision if not precision: - return ts + return STIXdatetime(ts, precision=precision) ms = ts.microsecond if precision == 'second': ts = ts.replace(microsecond=0) From 4763695ad5569c6d4b69dff5c99c35dfa2d4f97d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 15:07:25 -0400 Subject: [PATCH 25/82] Add simplejson dependency. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7fe46e4..92b237b 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ install_requires = [ 'six', 'python-dateutil', 'requests', + 'simplejson', ] setup( From 5172f86a7b37884463c61a45b5255fd8dd1a42bb Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 15:10:44 -0400 Subject: [PATCH 26/82] Change _STIXBase to output properties in spec order. closes #39 --- stix2/base.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 7de193b..62bb8c8 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -3,7 +3,7 @@ import collections import copy import datetime as dt -import json +import simplejson as json from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, @@ -36,6 +36,9 @@ def get_required_properties(properties): class _STIXBase(collections.Mapping): """Base class for STIX object types""" + def _object_properties(self): + return list(self._properties.keys()) + def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: if hasattr(prop, 'default'): @@ -141,12 +144,14 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - # TODO: put keys in specific order. Probably need custom JSON encoder. - return json.dumps(self, indent=4, sort_keys=True, cls=STIXJSONEncoder, - separators=(",", ": ")) # Don't include spaces after commas. + properties = self._object_properties() + # separators kwarg -> don't include spaces after commas. + return json.dumps(self, indent=4, cls=STIXJSONEncoder, + item_sort_key=lambda x: properties.index(x[0]), + separators=(",", ": ")) def __repr__(self): - props = [(k, self[k]) for k in sorted(self._properties) if self.get(k)] + props = [(k, self[k]) for k in self._object_properties() if self.get(k)] return "{0}({1})".format(self.__class__.__name__, ", ".join(["{0!s}={1!r}".format(k, v) for k, v in props])) From 228f488f5bde687eb439ac26dd52cb257922c19b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 15:11:54 -0400 Subject: [PATCH 27/82] Update commons.py to new OrderedDict format, just in case. --- stix2/common.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/stix2/common.py b/stix2/common.py index 7c6e747..638dba5 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,18 +1,22 @@ """STIX 2 Common Data Types and Properties""" +from collections import OrderedDict + from .other import ExternalReference, GranularMarking from .properties import (BooleanProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty) from .utils import NOW -COMMON_PROPERTIES = { +COMMON_PROPERTIES = OrderedDict() + +COMMON_PROPERTIES.update([ # 'type' and 'id' should be defined on each individual type - 'created': TimestampProperty(default=lambda: NOW, precision='millisecond'), - 'modified': TimestampProperty(default=lambda: NOW, precision='millisecond'), - 'external_references': ListProperty(ExternalReference), - 'revoked': BooleanProperty(), - 'labels': ListProperty(StringProperty), - 'created_by_ref': ReferenceProperty(type="identity"), - 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), - 'granular_markings': ListProperty(GranularMarking), -} + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), +]) From 8086447fce08da174ee29d694512b13f506dad7c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 11 Aug 2017 15:12:45 -0400 Subject: [PATCH 28/82] Change SDOs to OrderedDict approach. Removed COMMON_PROPERTIES. --- stix2/sdo.py | 376 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 242 insertions(+), 134 deletions(-) diff --git a/stix2/sdo.py b/stix2/sdo.py index 8115b9d..eb8a977 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -1,201 +1,293 @@ """STIX 2.0 Domain Objects""" +from collections import OrderedDict + import stix2 from .base import _STIXBase -from .common import COMMON_PROPERTIES from .observables import ObservableProperty -from .other import KillChainPhase -from .properties import (IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) +from .other import ExternalReference, GranularMarking, KillChainPhase +from .properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW class AttackPattern(_STIXBase): _type = 'attack-pattern' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Campaign(_STIXBase): _type = 'campaign' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'first_seen': TimestampProperty(), - 'last_seen': TimestampProperty(), - 'objective': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('objective', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class CourseOfAction(_STIXBase): _type = 'course-of-action' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Identity(_STIXBase): _type = 'identity' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'identity_class': StringProperty(required=True), - 'sectors': ListProperty(StringProperty), - 'contact_information': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('identity_class', StringProperty(required=True)), + ('sectors', ListProperty(StringProperty)), + ('contact_information', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Indicator(_STIXBase): _type = 'indicator' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(), - 'description': StringProperty(), - 'pattern': StringProperty(required=True), - 'valid_from': TimestampProperty(default=lambda: NOW), - 'valid_until': TimestampProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('labels', ListProperty(StringProperty, required=True)), + ('name', StringProperty()), + ('description', StringProperty()), + ('pattern', StringProperty(required=True)), + ('valid_from', TimestampProperty(default=lambda: NOW)), + ('valid_until', TimestampProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class IntrusionSet(_STIXBase): _type = 'intrusion-set' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'first_seen': TimestampProperty(), - 'last_seen ': TimestampProperty(), - 'goals': ListProperty(StringProperty), - 'resource_level': StringProperty(), - 'primary_motivation': StringProperty(), - 'secondary_motivations': ListProperty(StringProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen ', TimestampProperty()), + ('goals', ListProperty(StringProperty)), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Malware(_STIXBase): _type = 'malware' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class ObservedData(_STIXBase): _type = 'observed-data' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'first_observed': TimestampProperty(required=True), - 'last_observed': TimestampProperty(required=True), - 'number_observed': IntegerProperty(required=True), - 'objects': ObservableProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_observed', TimestampProperty(required=True)), + ('last_observed', TimestampProperty(required=True)), + ('number_observed', IntegerProperty(required=True)), + ('objects', ObservableProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Report(_STIXBase): _type = 'report' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'published': TimestampProperty(), - 'object_refs': ListProperty(ReferenceProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('published', TimestampProperty()), + ('object_refs', ListProperty(ReferenceProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class ThreatActor(_STIXBase): _type = 'threat-actor' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'aliases': ListProperty(StringProperty), - 'roles': ListProperty(StringProperty), - 'goals': ListProperty(StringProperty), - 'sophistication': StringProperty(), - 'resource_level': StringProperty(), - 'primary_motivation': StringProperty(), - 'secondary_motivations': ListProperty(StringProperty), - 'personal_motivations': ListProperty(StringProperty), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('roles', ListProperty(StringProperty)), + ('goals', ListProperty(StringProperty)), + ('sophistication', StringProperty()), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('personal_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Tool(_STIXBase): _type = 'tool' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'labels': ListProperty(StringProperty, required=True), - 'name': StringProperty(required=True), - 'description': StringProperty(), - 'kill_chain_phases': ListProperty(KillChainPhase), - 'tool_version': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('tool_version', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) class Vulnerability(_STIXBase): _type = 'vulnerability' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'name': StringProperty(required=True), - 'description': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) -def CustomObject(type='x-custom-type', properties={}): +def CustomObject(type='x-custom-type', properties=None): """Custom STIX Object type decorator Example 1: @@ -226,13 +318,29 @@ def CustomObject(type='x-custom-type', properties={}): class _Custom(cls, _STIXBase): _type = type - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ]) + + if properties is None: + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + _properties.update(properties) + # This is to follow the general properties structure. + _properties.update([ + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) cls.__init__(self, **kwargs) From ee8013d782a456a26f07578631b77552dd1d1fe6 Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 11 Aug 2017 16:18:20 -0400 Subject: [PATCH 29/82] Parse bundles correctly This required refactoring parts of the library. Code in __init__.py merged into bundle.py, which was renamed core.py. Code in other.py was merged into common.py. Fixes #40. --- stix2/__init__.py | 56 ++--------------- stix2/bundle.py | 25 -------- stix2/common.py | 129 ++++++++++++++++++++++++++++++++++++-- stix2/core.py | 99 +++++++++++++++++++++++++++++ stix2/other.py | 128 ------------------------------------- stix2/properties.py | 3 + stix2/sdo.py | 3 +- stix2/test/test_bundle.py | 12 ++++ 8 files changed, 244 insertions(+), 211 deletions(-) delete mode 100644 stix2/bundle.py create mode 100644 stix2/core.py delete mode 100644 stix2/other.py diff --git a/stix2/__init__.py b/stix2/__init__.py index 98697a9..b9b6764 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,7 +3,10 @@ # flake8: noqa from . import exceptions -from .bundle import Bundle +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) +from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomObservable, Directory, @@ -18,9 +21,6 @@ from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, WindowsRegistryValueType, WindowsServiceExt, X509Certificate, X509V3ExtenstionsType, parse_observable) -from .other import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) from .patterns import (AndBooleanExpression, AndObservationExpression, BasicObjectPathComponent, EqualityComparisonExpression, FloatConstant, FollowedByObservationExpression, @@ -44,51 +44,3 @@ from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, from .sro import Relationship, Sighting from .utils import get_dict from .version import __version__ - -OBJ_MAP = { - 'attack-pattern': AttackPattern, - 'campaign': Campaign, - 'course-of-action': CourseOfAction, - 'identity': Identity, - 'indicator': Indicator, - 'intrusion-set': IntrusionSet, - 'malware': Malware, - 'marking-definition': MarkingDefinition, - 'observed-data': ObservedData, - 'report': Report, - 'relationship': Relationship, - 'threat-actor': ThreatActor, - 'tool': Tool, - 'sighting': Sighting, - 'vulnerability': Vulnerability, -} - - -def parse(data, allow_custom=False): - """Deserialize a string or file-like object into a STIX object. - - Args: - data: The STIX 2 string to be parsed. - allow_custom (bool): Whether to allow custom properties or not. Default: False. - - Returns: - An instantiated Python STIX object. - """ - - obj = get_dict(data) - - if 'type' not in obj: - raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(obj)) - - try: - obj_class = OBJ_MAP[obj['type']] - except KeyError: - raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj['type']) - return obj_class(allow_custom=allow_custom, **obj) - - -def _register_type(new_type): - """Register a custom STIX Object type. - """ - - OBJ_MAP[new_type._type] = new_type diff --git a/stix2/bundle.py b/stix2/bundle.py deleted file mode 100644 index b598ceb..0000000 --- a/stix2/bundle.py +++ /dev/null @@ -1,25 +0,0 @@ -"""STIX 2 Bundle object""" - -from .base import _STIXBase -from .properties import IDProperty, Property, TypeProperty - - -class Bundle(_STIXBase): - - _type = 'bundle' - _properties = { - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'spec_version': Property(fixed="2.0"), - 'objects': Property(), - } - - def __init__(self, *args, **kwargs): - # Add any positional arguments to the 'objects' kwarg. - if args: - if isinstance(args[0], list): - kwargs['objects'] = args[0] + list(args[1:]) + kwargs.get('objects', []) - else: - kwargs['objects'] = list(args) + kwargs.get('objects', []) - - super(Bundle, self).__init__(**kwargs) diff --git a/stix2/common.py b/stix2/common.py index 7c6e747..57f99d0 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,10 +1,131 @@ """STIX 2 Common Data Types and Properties""" -from .other import ExternalReference, GranularMarking -from .properties import (BooleanProperty, ListProperty, ReferenceProperty, - StringProperty, TimestampProperty) -from .utils import NOW +from .base import _STIXBase +from .properties import (BooleanProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) +from .utils import NOW, get_dict + +class ExternalReference(_STIXBase): + _properties = { + 'source_name': StringProperty(required=True), + 'description': StringProperty(), + 'url': StringProperty(), + 'external_id': StringProperty(), + } + + def _check_object_constraints(self): + super(ExternalReference, self)._check_object_constraints() + self._check_at_least_one_property(["description", "external_id", "url"]) + + +class KillChainPhase(_STIXBase): + _properties = { + 'kill_chain_name': StringProperty(required=True), + 'phase_name': StringProperty(required=True), + } + + +class GranularMarking(_STIXBase): + _properties = { + 'marking_ref': ReferenceProperty(required=True, type="marking-definition"), + 'selectors': ListProperty(SelectorProperty, required=True), + } + + +class TLPMarking(_STIXBase): + # TODO: don't allow the creation of any other TLPMarkings than the ones below + _properties = { + 'tlp': Property(required=True) + } + + +class StatementMarking(_STIXBase): + _properties = { + 'statement': StringProperty(required=True) + } + + def __init__(self, statement=None, **kwargs): + # Allow statement as positional args. + if statement and not kwargs.get('statement'): + kwargs['statement'] = statement + + super(StatementMarking, self).__init__(**kwargs) + + +class MarkingProperty(Property): + """Represent the marking objects in the `definition` property of + marking-definition objects. + """ + + def clean(self, value): + if type(value) in [TLPMarking, StatementMarking]: + return value + else: + raise ValueError("must be a Statement or TLP Marking.") + + +class MarkingDefinition(_STIXBase): + _type = 'marking-definition' + _properties = { + 'created': TimestampProperty(default=lambda: NOW), + 'external_references': ListProperty(ExternalReference), + 'created_by_ref': ReferenceProperty(type="identity"), + 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), + 'granular_markings': ListProperty(GranularMarking), + 'type': TypeProperty(_type), + 'id': IDProperty(_type), + 'definition_type': StringProperty(required=True), + 'definition': MarkingProperty(required=True), + } + marking_map = { + 'tlp': TLPMarking, + 'statement': StatementMarking, + } + + def __init__(self, **kwargs): + if set(('definition_type', 'definition')).issubset(kwargs.keys()): + # Create correct marking type object + try: + marking_type = self.marking_map[kwargs['definition_type']] + except KeyError: + raise ValueError("definition_type must be a valid marking type") + + if not isinstance(kwargs['definition'], marking_type): + defn = get_dict(kwargs['definition']) + kwargs['definition'] = marking_type(**defn) + + super(MarkingDefinition, self).__init__(**kwargs) + + +TLP_WHITE = MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="white") +) + +TLP_GREEN = MarkingDefinition( + id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="green") +) + +TLP_AMBER = MarkingDefinition( + id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="amber") +) + +TLP_RED = MarkingDefinition( + id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="red") +) COMMON_PROPERTIES = { # 'type' and 'id' should be defined on each individual type 'created': TimestampProperty(default=lambda: NOW, precision='millisecond'), diff --git a/stix2/core.py b/stix2/core.py new file mode 100644 index 0000000..81dd492 --- /dev/null +++ b/stix2/core.py @@ -0,0 +1,99 @@ +"""STIX 2.0 Objects that are neither SDOs nor SROs""" + + +from . import exceptions +from .base import _STIXBase +from .common import MarkingDefinition +from .properties import IDProperty, ListProperty, Property, TypeProperty +from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, + IntrusionSet, Malware, ObservedData, Report, ThreatActor, + Tool, Vulnerability) +from .sro import Relationship, Sighting +from .utils import get_dict + + +class STIXObjectProperty(Property): + + def clean(self, value): + try: + dictified = get_dict(value) + except ValueError: + raise ValueError("This property may only contain a dictionary or object") + if dictified == {}: + raise ValueError("This property may only contain a non-empty dictionary or object") + if 'type' in dictified and dictified['type'] == 'bundle': + raise ValueError('This property may not contain a Bundle object') + + parsed_obj = parse(dictified) + return parsed_obj + + +class Bundle(_STIXBase): + + _type = 'bundle' + _properties = { + 'type': TypeProperty(_type), + 'id': IDProperty(_type), + 'spec_version': Property(fixed="2.0"), + 'objects': ListProperty(STIXObjectProperty), + } + + def __init__(self, *args, **kwargs): + # Add any positional arguments to the 'objects' kwarg. + if args: + if isinstance(args[0], list): + kwargs['objects'] = args[0] + list(args[1:]) + kwargs.get('objects', []) + else: + kwargs['objects'] = list(args) + kwargs.get('objects', []) + + super(Bundle, self).__init__(**kwargs) + + +OBJ_MAP = { + 'attack-pattern': AttackPattern, + 'bundle': Bundle, + 'campaign': Campaign, + 'course-of-action': CourseOfAction, + 'identity': Identity, + 'indicator': Indicator, + 'intrusion-set': IntrusionSet, + 'malware': Malware, + 'marking-definition': MarkingDefinition, + 'observed-data': ObservedData, + 'report': Report, + 'relationship': Relationship, + 'threat-actor': ThreatActor, + 'tool': Tool, + 'sighting': Sighting, + 'vulnerability': Vulnerability, +} + + +def parse(data, allow_custom=False): + """Deserialize a string or file-like object into a STIX object. + + Args: + data: The STIX 2 string to be parsed. + allow_custom (bool): Whether to allow custom properties or not. Default: False. + + Returns: + An instantiated Python STIX object. + """ + + obj = get_dict(data) + + if 'type' not in obj: + raise exceptions.ParseError("Can't parse object with no 'type' property: %s" % str(obj)) + + try: + obj_class = OBJ_MAP[obj['type']] + except KeyError: + raise exceptions.ParseError("Can't parse unknown object type '%s'! For custom types, use the CustomObject decorator." % obj['type']) + return obj_class(allow_custom=allow_custom, **obj) + + +def _register_type(new_type): + """Register a custom STIX Object type. + """ + + OBJ_MAP[new_type._type] = new_type diff --git a/stix2/other.py b/stix2/other.py deleted file mode 100644 index cd75745..0000000 --- a/stix2/other.py +++ /dev/null @@ -1,128 +0,0 @@ -"""STIX 2.0 Objects that are neither SDOs nor SROs""" - -from .base import _STIXBase -from .properties import (IDProperty, ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) -from .utils import NOW, get_dict - - -class ExternalReference(_STIXBase): - _properties = { - 'source_name': StringProperty(required=True), - 'description': StringProperty(), - 'url': StringProperty(), - 'external_id': StringProperty(), - } - - def _check_object_constraints(self): - super(ExternalReference, self)._check_object_constraints() - self._check_at_least_one_property(["description", "external_id", "url"]) - - -class KillChainPhase(_STIXBase): - _properties = { - 'kill_chain_name': StringProperty(required=True), - 'phase_name': StringProperty(required=True), - } - - -class GranularMarking(_STIXBase): - _properties = { - 'marking_ref': ReferenceProperty(required=True, type="marking-definition"), - 'selectors': ListProperty(SelectorProperty, required=True), - } - - -class TLPMarking(_STIXBase): - # TODO: don't allow the creation of any other TLPMarkings than the ones below - _properties = { - 'tlp': Property(required=True) - } - - -class StatementMarking(_STIXBase): - _properties = { - 'statement': StringProperty(required=True) - } - - def __init__(self, statement=None, **kwargs): - # Allow statement as positional args. - if statement and not kwargs.get('statement'): - kwargs['statement'] = statement - - super(StatementMarking, self).__init__(**kwargs) - - -class MarkingProperty(Property): - """Represent the marking objects in the `definition` property of - marking-definition objects. - """ - - def clean(self, value): - if type(value) in [TLPMarking, StatementMarking]: - return value - else: - raise ValueError("must be a Statement or TLP Marking.") - - -class MarkingDefinition(_STIXBase): - _type = 'marking-definition' - _properties = { - 'created': TimestampProperty(default=lambda: NOW), - 'external_references': ListProperty(ExternalReference), - 'created_by_ref': ReferenceProperty(type="identity"), - 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), - 'granular_markings': ListProperty(GranularMarking), - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'definition_type': StringProperty(required=True), - 'definition': MarkingProperty(required=True), - } - marking_map = { - 'tlp': TLPMarking, - 'statement': StatementMarking, - } - - def __init__(self, **kwargs): - if set(('definition_type', 'definition')).issubset(kwargs.keys()): - # Create correct marking type object - try: - marking_type = self.marking_map[kwargs['definition_type']] - except KeyError: - raise ValueError("definition_type must be a valid marking type") - - if not isinstance(kwargs['definition'], marking_type): - defn = get_dict(kwargs['definition']) - kwargs['definition'] = marking_type(**defn) - - super(MarkingDefinition, self).__init__(**kwargs) - - -TLP_WHITE = MarkingDefinition( - id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="white") -) - -TLP_GREEN = MarkingDefinition( - id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="green") -) - -TLP_AMBER = MarkingDefinition( - id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="amber") -) - -TLP_RED = MarkingDefinition( - id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="red") -) diff --git a/stix2/properties.py b/stix2/properties.py index db06763..f63ec8b 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -118,6 +118,9 @@ class ListProperty(Property): if type(self.contained) is EmbeddedObjectProperty: obj_type = self.contained.type + elif type(self.contained).__name__ is 'STIXObjectProperty': + # ^ this way of checking doesn't require a circular import + obj_type = type(valid) else: obj_type = self.contained diff --git a/stix2/sdo.py b/stix2/sdo.py index 8115b9d..43c8328 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -3,9 +3,8 @@ import stix2 from .base import _STIXBase -from .common import COMMON_PROPERTIES +from .common import COMMON_PROPERTIES, KillChainPhase from .observables import ObservableProperty -from .other import KillChainPhase from .properties import (IDProperty, IntegerProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty) diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 54d7080..0733637 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -116,3 +116,15 @@ def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationsh bundle = stix2.Bundle([indicator], malware, objects=[relationship]) assert str(bundle) == EXPECTED_BUNDLE + + +def test_parse_bundle(): + bundle = stix2.parse(EXPECTED_BUNDLE) + + assert bundle.type == "bundle" + assert bundle.id.startswith("bundle--") + assert bundle.spec_version == "2.0" + assert type(bundle.objects[0]) is stix2.Indicator + assert bundle.objects[0].type == 'indicator' + assert bundle.objects[1].type == 'malware' + assert bundle.objects[2].type == 'relationship' From dd2a1db5c15139507f7318922cd4aa38452a9cff Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 08:27:49 -0400 Subject: [PATCH 30/82] Change SROs to OrderedDict approach. Removed COMMON_PROPERTIES. --- stix2/sro.py | 75 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/stix2/sro.py b/stix2/sro.py index c13fff3..49e607a 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -1,31 +1,39 @@ """STIX 2.0 Relationship Objects.""" +from collections import OrderedDict + from .base import _STIXBase -from .common import COMMON_PROPERTIES -from .properties import (IDProperty, IntegerProperty, ListProperty, +from .other import ExternalReference, GranularMarking +from .properties import (BooleanProperty, IDProperty, IntegerProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty) +from .utils import NOW class Relationship(_STIXBase): _type = 'relationship' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - 'relationship_type': StringProperty(required=True), - 'description': StringProperty(), - 'source_ref': ReferenceProperty(required=True), - 'target_ref': ReferenceProperty(required=True), - }) + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('relationship_type', StringProperty(required=True)), + ('description', StringProperty()), + ('source_ref', ReferenceProperty(required=True)), + ('target_ref', ReferenceProperty(required=True)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) # Explicitly define the first three kwargs to make readable Relationship declarations. - def __init__(self, source_ref=None, relationship_type=None, target_ref=None, - **kwargs): - # TODO: - # - description - + def __init__(self, source_ref=None, relationship_type=None, + target_ref=None, **kwargs): # Allow (source_ref, relationship_type, target_ref) as positional args. if source_ref and not kwargs.get('source_ref'): kwargs['source_ref'] = source_ref @@ -39,24 +47,29 @@ class Relationship(_STIXBase): class Sighting(_STIXBase): _type = 'sighting' - _properties = COMMON_PROPERTIES.copy() - _properties.update({ - 'id': IDProperty(_type), - 'type': TypeProperty(_type), - 'first_seen': TimestampProperty(), - 'last_seen': TimestampProperty(), - 'count': IntegerProperty(), - 'sighting_of_ref': ReferenceProperty(required=True), - 'observed_data_refs': ListProperty(ReferenceProperty(type="observed-data")), - 'where_sighted_refs': ListProperty(ReferenceProperty(type="identity")), - 'summary': StringProperty(), - }) + _properties = OrderedDict() + _properties.update([ + ('id', IDProperty(_type)), + ('type', TypeProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('count', IntegerProperty()), + ('sighting_of_ref', ReferenceProperty(required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))), + ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))), + ('summary', BooleanProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) # Explicitly define the first kwargs to make readable Sighting declarations. def __init__(self, sighting_of_ref=None, **kwargs): - # TODO: - # - description - # Allow sighting_of_ref as a positional arg. if sighting_of_ref and not kwargs.get('sighting_of_ref'): kwargs['sighting_of_ref'] = sighting_of_ref From ccfcffb2f5256835fe63b7a5c6fa04d53e854a13 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 08:31:08 -0400 Subject: [PATCH 31/82] Minor change. --- stix2/sro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/sro.py b/stix2/sro.py index 49e607a..cf26c32 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -49,8 +49,8 @@ class Sighting(_STIXBase): _type = 'sighting' _properties = OrderedDict() _properties.update([ - ('id', IDProperty(_type)), ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), ('created_by_ref', ReferenceProperty(type="identity")), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), From add14c2490abd5cc8d29db26d8f6109263e48cdd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 09:06:20 -0400 Subject: [PATCH 32/82] Update CustomObject docstring, re-organize object properties. --- stix2/sdo.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/stix2/sdo.py b/stix2/sdo.py index eb8a977..75218b7 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -292,22 +292,22 @@ def CustomObject(type='x-custom-type', properties=None): Example 1: - @CustomObject('x-type-name', { - 'property1': StringProperty(required=True), - 'property2': IntegerProperty(), - }) + @CustomObject('x-type-name', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) class MyNewObjectType(): pass Supply an __init__() function to add any special validations to the custom - type. Don't call super().__init() though - doing so will cause an error. + type. Don't call super().__init__() though - doing so will cause an error. Example 2: - @CustomObject('x-type-name', { - 'property1': StringProperty(required=True), - 'property2': IntegerProperty(), - }) + @CustomObject('x-type-name', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) class MyNewObjectType(): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: @@ -327,10 +327,13 @@ def CustomObject(type='x-custom-type', properties=None): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ]) - if properties is None: + if not properties: raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - _properties.update(properties) + normal_properties = [x for x in properties if not x[0].startswith("x_")] + custom_properties = [x for x in properties if x[0].startswith("x_")] + + _properties.update(normal_properties) # This is to follow the general properties structure. _properties.update([ @@ -341,6 +344,9 @@ def CustomObject(type='x-custom-type', properties=None): ('granular_markings', ListProperty(GranularMarking)), ]) + # Put all custom properties at the bottom, sorted alphabetically. + _properties.update(sorted(custom_properties, key=lambda x: x[0])) + def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) cls.__init__(self, **kwargs) From 1329e2e76fbd144594243c12655b58e424d6edcd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 09:24:41 -0400 Subject: [PATCH 33/82] Apply OrderedDict changes to Bundle. --- stix2/bundle.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/stix2/bundle.py b/stix2/bundle.py index b598ceb..29d4d8b 100644 --- a/stix2/bundle.py +++ b/stix2/bundle.py @@ -1,5 +1,7 @@ """STIX 2 Bundle object""" +from collections import OrderedDict + from .base import _STIXBase from .properties import IDProperty, Property, TypeProperty @@ -7,12 +9,13 @@ from .properties import IDProperty, Property, TypeProperty class Bundle(_STIXBase): _type = 'bundle' - _properties = { - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'spec_version': Property(fixed="2.0"), - 'objects': Property(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('spec_version', Property(fixed="2.0")), + ('objects', Property()), + ]) def __init__(self, *args, **kwargs): # Add any positional arguments to the 'objects' kwarg. From e2c9ecccaf522d3e6d8aab4d7b9155c57b8f27f2 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 10:29:17 -0400 Subject: [PATCH 34/82] Apply OrderedDict changes to Observables. --- stix2/observables.py | 752 +++++++++++++++++++++++-------------------- 1 file changed, 402 insertions(+), 350 deletions(-) diff --git a/stix2/observables.py b/stix2/observables.py index 366e007..fd66406 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -5,6 +5,8 @@ embedded in Email Message objects, inherit from _STIXBase instead of Observable and do not have a '_type' attribute. """ +from collections import OrderedDict + from .base import _Extension, _Observable, _STIXBase from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, ParseError) @@ -75,13 +77,14 @@ class ExtensionsProperty(DictionaryProperty): class Artifact(_Observable): _type = 'artifact' - _properties = { - 'type': TypeProperty(_type), - 'mime_type': StringProperty(), - 'payload_bin': BinaryProperty(), - 'url': StringProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('mime_type', StringProperty()), + ('payload_bin', BinaryProperty()), + ('url', StringProperty()), + ('hashes', HashesProperty()), + ]) def _check_object_constraints(self): super(Artifact, self)._check_object_constraints() @@ -91,54 +94,59 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): _type = 'autonomous-system' - _properties = { - 'type': TypeProperty(_type), - 'number': IntegerProperty(), - 'name': StringProperty(), - 'rir': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('number', IntegerProperty()), + ('name', StringProperty()), + ('rir', StringProperty()), + ]) class Directory(_Observable): _type = 'directory' - _properties = { - 'type': TypeProperty(_type), - 'path': StringProperty(required=True), - 'path_enc': StringProperty(), + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('path', StringProperty(required=True)), + ('path_enc', StringProperty()), # these are not the created/modified timestamps of the object itself - 'created': TimestampProperty(), - 'modified': TimestampProperty(), - 'accessed': TimestampProperty(), - 'contains_refs': ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory'])), - } + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), + ]) class DomainName(_Observable): _type = 'domain-name' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'])), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), + ]) class EmailAddress(_Observable): _type = 'email-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'display_name': StringProperty(), - 'belongs_to_ref': ObjectReferenceProperty(valid_types='user-account'), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('display_name', StringProperty()), + ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), + ]) class EmailMIMEComponent(_STIXBase): - _properties = { - 'body': StringProperty(), - 'body_raw_ref': ObjectReferenceProperty(valid_types=['artifact', 'file']), - 'content_type': StringProperty(), - 'content_disposition': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('body', StringProperty()), + ('body_raw_ref', ObjectReferenceProperty(valid_types=['artifact', 'file'])), + ('content_type', StringProperty()), + ('content_disposition', StringProperty()), + ]) def _check_object_constraints(self): super(EmailMIMEComponent, self)._check_object_constraints() @@ -147,23 +155,24 @@ class EmailMIMEComponent(_STIXBase): class EmailMessage(_Observable): _type = 'email-message' - _properties = { - 'type': TypeProperty(_type), - 'is_multipart': BooleanProperty(required=True), - 'date': TimestampProperty(), - 'content_type': StringProperty(), - 'from_ref': ObjectReferenceProperty(valid_types='email-addr'), - 'sender_ref': ObjectReferenceProperty(valid_types='email-addr'), - 'to_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'cc_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'bcc_refs': ListProperty(ObjectReferenceProperty(valid_types='email-addr')), - 'subject': StringProperty(), - 'received_lines': ListProperty(StringProperty), - 'additional_header_fields': DictionaryProperty(), - 'body': StringProperty(), - 'body_multipart': ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent)), - 'raw_email_ref': ObjectReferenceProperty(valid_types='artifact'), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('is_multipart', BooleanProperty(required=True)), + ('date', TimestampProperty()), + ('content_type', StringProperty()), + ('from_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('sender_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('to_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('cc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('subject', StringProperty()), + ('received_lines', ListProperty(StringProperty)), + ('additional_header_fields', DictionaryProperty()), + ('body', StringProperty()), + ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), + ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) def _check_object_constraints(self): super(EmailMessage, self)._check_object_constraints() @@ -174,82 +183,88 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): - _properties = { - 'contains_refs': ListProperty(ObjectReferenceProperty(valid_types='file'), required=True), - 'version': StringProperty(), - 'comment': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), + ('version', StringProperty()), + ('comment', StringProperty()), + ]) class AlternateDataStream(_STIXBase): - _properties = { - 'name': StringProperty(required=True), - 'hashes': HashesProperty(), - 'size': IntegerProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('name', StringProperty(required=True)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ]) class NTFSExt(_Extension): - _properties = { - 'sid': StringProperty(), - 'alternate_data_streams': ListProperty(EmbeddedObjectProperty(type=AlternateDataStream)), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('sid', StringProperty()), + ('alternate_data_streams', ListProperty(EmbeddedObjectProperty(type=AlternateDataStream))), + ]) class PDFExt(_Extension): - _properties = { - 'version': StringProperty(), - 'is_optimized': BooleanProperty(), - 'document_info_dict': DictionaryProperty(), - 'pdfid0': StringProperty(), - 'pdfid1': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('version', StringProperty()), + ('is_optimized', BooleanProperty()), + ('document_info_dict', DictionaryProperty()), + ('pdfid0', StringProperty()), + ('pdfid1', StringProperty()), + ]) class RasterImageExt(_Extension): - _properties = { - 'image_height': IntegerProperty(), - 'image_weight': IntegerProperty(), - 'bits_per_pixel': IntegerProperty(), - 'image_compression_algorithm': StringProperty(), - 'exif_tags': DictionaryProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('image_height', IntegerProperty()), + ('image_weight', IntegerProperty()), + ('bits_per_pixel', IntegerProperty()), + ('image_compression_algorithm', StringProperty()), + ('exif_tags', DictionaryProperty()), + ]) class WindowsPEOptionalHeaderType(_STIXBase): - _properties = { - 'magic_hex': HexProperty(), - 'major_linker_version': IntegerProperty(), - 'minor_linker_version': IntegerProperty(), - 'size_of_code': IntegerProperty(), - 'size_of_initialized_data': IntegerProperty(), - 'size_of_uninitialized_data': IntegerProperty(), - 'address_of_entry_point': IntegerProperty(), - 'base_of_code': IntegerProperty(), - 'base_of_data': IntegerProperty(), - 'image_base': IntegerProperty(), - 'section_alignment': IntegerProperty(), - 'file_alignment': IntegerProperty(), - 'major_os_version': IntegerProperty(), - 'minor_os_version': IntegerProperty(), - 'major_image_version': IntegerProperty(), - 'minor_image_version': IntegerProperty(), - 'major_subsystem_version': IntegerProperty(), - 'minor_subsystem_version': IntegerProperty(), - 'win32_version_value_hex': HexProperty(), - 'size_of_image': IntegerProperty(), - 'size_of_headers': IntegerProperty(), - 'checksum_hex': HexProperty(), - 'subsystem_hex': HexProperty(), - 'dll_characteristics_hex': HexProperty(), - 'size_of_stack_reserve': IntegerProperty(), - 'size_of_stack_commit': IntegerProperty(), - 'size_of_heap_reserve': IntegerProperty(), - 'size_of_heap_commit': IntegerProperty(), - 'loader_flags_hex': HexProperty(), - 'number_of_rva_and_sizes': IntegerProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('magic_hex', HexProperty()), + ('major_linker_version', IntegerProperty()), + ('minor_linker_version', IntegerProperty()), + ('size_of_code', IntegerProperty()), + ('size_of_initialized_data', IntegerProperty()), + ('size_of_uninitialized_data', IntegerProperty()), + ('address_of_entry_point', IntegerProperty()), + ('base_of_code', IntegerProperty()), + ('base_of_data', IntegerProperty()), + ('image_base', IntegerProperty()), + ('section_alignment', IntegerProperty()), + ('file_alignment', IntegerProperty()), + ('major_os_version', IntegerProperty()), + ('minor_os_version', IntegerProperty()), + ('major_image_version', IntegerProperty()), + ('minor_image_version', IntegerProperty()), + ('major_subsystem_version', IntegerProperty()), + ('minor_subsystem_version', IntegerProperty()), + ('win32_version_value_hex', HexProperty()), + ('size_of_image', IntegerProperty()), + ('size_of_headers', IntegerProperty()), + ('checksum_hex', HexProperty()), + ('subsystem_hex', HexProperty()), + ('dll_characteristics_hex', HexProperty()), + ('size_of_stack_reserve', IntegerProperty()), + ('size_of_stack_commit', IntegerProperty()), + ('size_of_heap_reserve', IntegerProperty()), + ('size_of_heap_commit', IntegerProperty()), + ('loader_flags_hex', HexProperty()), + ('number_of_rva_and_sizes', IntegerProperty()), + ('hashes', HashesProperty()), + ]) def _check_object_constraints(self): super(WindowsPEOptionalHeaderType, self)._check_object_constraints() @@ -257,53 +272,56 @@ class WindowsPEOptionalHeaderType(_STIXBase): class WindowsPESection(_STIXBase): - _properties = { - 'name': StringProperty(required=True), - 'size': IntegerProperty(), - 'entropy': FloatProperty(), - 'hashes': HashesProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('name', StringProperty(required=True)), + ('size', IntegerProperty()), + ('entropy', FloatProperty()), + ('hashes', HashesProperty()), + ]) class WindowsPEBinaryExt(_Extension): - _properties = { - 'pe_type': StringProperty(required=True), # open_vocab - 'imphash': StringProperty(), - 'machine_hex': HexProperty(), - 'number_of_sections': IntegerProperty(), - 'time_date_stamp': TimestampProperty(precision='second'), - 'pointer_to_symbol_table_hex': HexProperty(), - 'number_of_symbols': IntegerProperty(), - 'size_of_optional_header': IntegerProperty(), - 'characteristics_hex': HexProperty(), - 'file_header_hashes': HashesProperty(), - 'optional_header': EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType), - 'sections': ListProperty(EmbeddedObjectProperty(type=WindowsPESection)), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('pe_type', StringProperty(required=True)), # open_vocab + ('imphash', StringProperty()), + ('machine_hex', HexProperty()), + ('number_of_sections', IntegerProperty()), + ('time_date_stamp', TimestampProperty(precision='second')), + ('pointer_to_symbol_table_hex', HexProperty()), + ('number_of_symbols', IntegerProperty()), + ('size_of_optional_header', IntegerProperty()), + ('characteristics_hex', HexProperty()), + ('file_header_hashes', HashesProperty()), + ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), + ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), + ]) class File(_Observable): _type = 'file' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'hashes': HashesProperty(), - 'size': IntegerProperty(), - 'name': StringProperty(), - 'name_enc': StringProperty(), - 'magic_number_hex': HexProperty(), - 'mime_type': StringProperty(), + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ('name', StringProperty()), + ('name_enc', StringProperty()), + ('magic_number_hex', HexProperty()), + ('mime_type', StringProperty()), # these are not the created/modified timestamps of the object itself - 'created': TimestampProperty(), - 'modified': TimestampProperty(), - 'accessed': TimestampProperty(), - 'parent_directory_ref': ObjectReferenceProperty(valid_types='directory'), - 'is_encrypted': BooleanProperty(), - 'encryption_algorithm': StringProperty(), - 'decryption_key': StringProperty(), - 'contains_refs': ListProperty(ObjectReferenceProperty), - 'content_ref': ObjectReferenceProperty(valid_types='artifact'), - } + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), + ('is_encrypted', BooleanProperty()), + ('encryption_algorithm', StringProperty()), + ('decryption_key', StringProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty)), + ('content_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) def _check_object_constraints(self): super(File, self)._check_object_constraints() @@ -313,61 +331,68 @@ class File(_Observable): class IPv4Address(_Observable): _type = 'ipv4-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), - 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ]) class IPv6Address(_Observable): _type = 'ipv6-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), - 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ]) class MACAddress(_Observable): _type = 'mac-addr' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ]) class Mutex(_Observable): _type = 'mutex' - _properties = { - 'type': TypeProperty(_type), - 'name': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty()), + ]) class HTTPRequestExt(_Extension): - _properties = { - 'request_method': StringProperty(required=True), - 'request_value': StringProperty(required=True), - 'request_version': StringProperty(), - 'request_header': DictionaryProperty(), - 'message_body_length': IntegerProperty(), - 'message_body_data_ref': ObjectReferenceProperty(valid_types='artifact'), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('request_method', StringProperty(required=True)), + ('request_value', StringProperty(required=True)), + ('request_version', StringProperty()), + ('request_header', DictionaryProperty()), + ('message_body_length', IntegerProperty()), + ('message_body_data_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) class ICMPExt(_Extension): - _properties = { - 'icmp_type_hex': HexProperty(required=True), - 'icmp_code_hex': HexProperty(required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('icmp_type_hex', HexProperty(required=True)), + ('icmp_code_hex', HexProperty(required=True)), + ]) class SocketExt(_Extension): - _properties = { - 'address_family': EnumProperty([ + _properties = OrderedDict() + _properties = _properties.update([ + ('address_family', EnumProperty([ "AF_UNSPEC", "AF_INET", "AF_IPX", @@ -376,58 +401,60 @@ class SocketExt(_Extension): "AF_INET6", "AF_IRDA", "AF_BTH", - ], required=True), - 'is_blocking': BooleanProperty(), - 'is_listening': BooleanProperty(), - 'protocol_family': EnumProperty([ + ], required=True)), + ('is_blocking', BooleanProperty()), + ('is_listening', BooleanProperty()), + ('protocol_family', EnumProperty([ "PF_INET", "PF_IPX", "PF_APPLETALK", "PF_INET6", "PF_AX25", "PF_NETROM" - ]), - 'options': DictionaryProperty(), - 'socket_type': EnumProperty([ + ])), + ('options', DictionaryProperty()), + ('socket_type', EnumProperty([ "SOCK_STREAM", "SOCK_DGRAM", "SOCK_RAW", "SOCK_RDM", "SOCK_SEQPACKET", - ]), - } + ])), + ]) class TCPExt(_Extension): - _properties = { - 'src_flags_hex': HexProperty(), - 'dst_flags_hex': HexProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('src_flags_hex', HexProperty()), + ('dst_flags_hex', HexProperty()), + ]) class NetworkTraffic(_Observable): _type = 'network-traffic' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'start': TimestampProperty(), - 'end': TimestampProperty(), - 'is_active': BooleanProperty(), - 'src_ref': ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name']), - 'dst_ref': ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name']), - 'src_port': IntegerProperty(), - 'dst_port': IntegerProperty(), - 'protocols': ListProperty(StringProperty, required=True), - 'src_byte_count': IntegerProperty(), - 'dst_byte_count': IntegerProperty(), - 'src_packets': IntegerProperty(), - 'dst_packets': IntegerProperty(), - 'ipfix': DictionaryProperty(), - 'src_payload_ref': ObjectReferenceProperty(valid_types='artifact'), - 'dst_payload_ref': ObjectReferenceProperty(valid_types='artifact'), - 'encapsulates_refs': ListProperty(ObjectReferenceProperty(valid_types='network-traffic')), - 'encapsulates_by_ref': ObjectReferenceProperty(valid_types='network-traffic'), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('start', TimestampProperty()), + ('end', TimestampProperty()), + ('is_active', BooleanProperty()), + ('src_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('dst_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('src_port', IntegerProperty()), + ('dst_port', IntegerProperty()), + ('protocols', ListProperty(StringProperty, required=True)), + ('src_byte_count', IntegerProperty()), + ('dst_byte_count', IntegerProperty()), + ('src_packets', IntegerProperty()), + ('dst_packets', IntegerProperty()), + ('ipfix', DictionaryProperty()), + ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), + ]) def _check_object_constraints(self): super(NetworkTraffic, self)._check_object_constraints() @@ -435,37 +462,39 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): - _properties = { - 'aslr_enabled': BooleanProperty(), - 'dep_enabled': BooleanProperty(), - 'priority': StringProperty(), - 'owner_sid': StringProperty(), - 'window_title': StringProperty(), - 'startup_info': DictionaryProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('aslr_enabled', BooleanProperty()), + ('dep_enabled', BooleanProperty()), + ('priority', StringProperty()), + ('owner_sid', StringProperty()), + ('window_title', StringProperty()), + ('startup_info', DictionaryProperty()), + ]) class WindowsServiceExt(_Extension): - _properties = { - 'service_name': StringProperty(required=True), - 'descriptions': ListProperty(StringProperty), - 'display_name': StringProperty(), - 'group_name': StringProperty(), - 'start_type': EnumProperty([ + _properties = OrderedDict() + _properties = _properties.update([ + ('service_name', StringProperty(required=True)), + ('descriptions', ListProperty(StringProperty)), + ('display_name', StringProperty()), + ('group_name', StringProperty()), + ('start_type', EnumProperty([ "SERVICE_AUTO_START", "SERVICE_BOOT_START", "SERVICE_DEMAND_START", "SERVICE_DISABLED", "SERVICE_SYSTEM_ALERT", - ]), - 'service_dll_refs': ListProperty(ObjectReferenceProperty(valid_types='file')), - 'service_type': EnumProperty([ + ])), + ('service_dll_refs', ListProperty(ObjectReferenceProperty(valid_types='file'))), + ('service_type', EnumProperty([ "SERVICE_KERNEL_DRIVER", "SERVICE_FILE_SYSTEM_DRIVER", "SERVICE_WIN32_OWN_PROCESS", "SERVICE_WIN32_SHARE_PROCESS", - ]), - 'service_status': EnumProperty([ + ])), + ('service_status', EnumProperty([ "SERVICE_CONTINUE_PENDING", "SERVICE_PAUSE_PENDING", "SERVICE_PAUSED", @@ -473,30 +502,31 @@ class WindowsServiceExt(_Extension): "SERVICE_START_PENDING", "SERVICE_STOP_PENDING", "SERVICE_STOPPED", - ]), - } + ])), + ]) class Process(_Observable): _type = 'process' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'is_hidden': BooleanProperty(), - 'pid': IntegerProperty(), - 'name': StringProperty(), + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('is_hidden', BooleanProperty()), + ('pid', IntegerProperty()), + ('name', StringProperty()), # this is not the created timestamps of the object itself - 'created': TimestampProperty(), - 'cwd': StringProperty(), - 'arguments': ListProperty(StringProperty), - 'command_line': StringProperty(), - 'environment_variables': DictionaryProperty(), - 'opened_connection_refs': ListProperty(ObjectReferenceProperty(valid_types='network-traffic')), - 'creator_user_ref': ObjectReferenceProperty(valid_types='user-account'), - 'binary_ref': ObjectReferenceProperty(valid_types='file'), - 'parent_ref': ObjectReferenceProperty(valid_types='process'), - 'child_refs': ListProperty(ObjectReferenceProperty('process')), - } + ('created', TimestampProperty()), + ('cwd', StringProperty()), + ('arguments', ListProperty(StringProperty)), + ('command_line', StringProperty()), + ('environment_variables', DictionaryProperty()), + ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('binary_ref', ObjectReferenceProperty(valid_types='file')), + ('parent_ref', ObjectReferenceProperty(valid_types='process')), + ('child_refs', ListProperty(ObjectReferenceProperty('process'))), + ]) def _check_object_constraints(self): # no need to check windows-service-ext, since it has a required property @@ -515,60 +545,65 @@ class Process(_Observable): class Software(_Observable): _type = 'software' - _properties = { - 'type': TypeProperty(_type), - 'name': StringProperty(required=True), - 'cpe': StringProperty(), - 'languages': ListProperty(StringProperty), - 'vendor': StringProperty(), - 'version': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty(required=True)), + ('cpe', StringProperty()), + ('languages', ListProperty(StringProperty)), + ('vendor', StringProperty()), + ('version', StringProperty()), + ]) class URL(_Observable): _type = 'url' - _properties = { - 'type': TypeProperty(_type), - 'value': StringProperty(required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ]) class UNIXAccountExt(_Extension): - _properties = { - 'gid': IntegerProperty(), - 'groups': ListProperty(StringProperty), - 'home_dir': StringProperty(), - 'shell': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('gid', IntegerProperty()), + ('groups', ListProperty(StringProperty)), + ('home_dir', StringProperty()), + ('shell', StringProperty()), + ]) class UserAccount(_Observable): _type = 'user-account' - _properties = { - 'type': TypeProperty(_type), - 'extensions': ExtensionsProperty(enclosing_type=_type), - 'user_id': StringProperty(required=True), - 'account_login': StringProperty(), - 'account_type': StringProperty(), # open vocab - 'display_name': StringProperty(), - 'is_service_account': BooleanProperty(), - 'is_privileged': BooleanProperty(), - 'can_escalate_privs': BooleanProperty(), - 'is_disabled': BooleanProperty(), - 'account_created': TimestampProperty(), - 'account_expires': TimestampProperty(), - 'password_last_changed': TimestampProperty(), - 'account_first_login': TimestampProperty(), - 'account_last_login': TimestampProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ('user_id', StringProperty(required=True)), + ('account_login', StringProperty()), + ('account_type', StringProperty()), # open vocab + ('display_name', StringProperty()), + ('is_service_account', BooleanProperty()), + ('is_privileged', BooleanProperty()), + ('can_escalate_privs', BooleanProperty()), + ('is_disabled', BooleanProperty()), + ('account_created', TimestampProperty()), + ('account_expires', TimestampProperty()), + ('password_last_changed', TimestampProperty()), + ('account_first_login', TimestampProperty()), + ('account_last_login', TimestampProperty()), + ]) class WindowsRegistryValueType(_STIXBase): _type = 'windows-registry-value-type' - _properties = { - 'name': StringProperty(required=True), - 'data': StringProperty(), - 'data_type': EnumProperty([ + _properties = OrderedDict() + _properties = _properties.update([ + ('name', StringProperty(required=True)), + ('data', StringProperty()), + ('data_type', EnumProperty([ 'REG_NONE', 'REG_SZ', 'REG_EXPAND_SZ', @@ -582,21 +617,22 @@ class WindowsRegistryValueType(_STIXBase): 'REG_RESOURCE_REQUIREMENTS_LIST', 'REG_QWORD', 'REG_INVALID_TYPE', - ]), - } + ])), + ]) class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' - _properties = { - 'type': TypeProperty(_type), - 'key': StringProperty(required=True), - 'values': ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType)), + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('key', StringProperty(required=True)), + ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), # this is not the modified timestamps of the object itself - 'modified': TimestampProperty(), - 'creator_user_ref': ObjectReferenceProperty(valid_types='user-account'), - 'number_of_subkeys': IntegerProperty(), - } + ('modified', TimestampProperty()), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('number_of_subkeys', IntegerProperty()), + ]) @property def values(self): @@ -606,44 +642,46 @@ class WindowsRegistryKey(_Observable): class X509V3ExtenstionsType(_STIXBase): _type = 'x509-v3-extensions-type' - _properties = { - 'basic_constraints': StringProperty(), - 'name_constraints': StringProperty(), - 'policy_constraints': StringProperty(), - 'key_usage': StringProperty(), - 'extended_key_usage': StringProperty(), - 'subject_key_identifier': StringProperty(), - 'authority_key_identifier': StringProperty(), - 'subject_alternative_name': StringProperty(), - 'issuer_alternative_name': StringProperty(), - 'subject_directory_attributes': StringProperty(), - 'crl_distribution_points': StringProperty(), - 'inhibit_any_policy': StringProperty(), - 'private_key_usage_period_not_before': TimestampProperty(), - 'private_key_usage_period_not_after': TimestampProperty(), - 'certificate_policies': StringProperty(), - 'policy_mappings': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('basic_constraints', StringProperty()), + ('name_constraints', StringProperty()), + ('policy_constraints', StringProperty()), + ('key_usage', StringProperty()), + ('extended_key_usage', StringProperty()), + ('subject_key_identifier', StringProperty()), + ('authority_key_identifier', StringProperty()), + ('subject_alternative_name', StringProperty()), + ('issuer_alternative_name', StringProperty()), + ('subject_directory_attributes', StringProperty()), + ('crl_distribution_points', StringProperty()), + ('inhibit_any_policy', StringProperty()), + ('private_key_usage_period_not_before', TimestampProperty()), + ('private_key_usage_period_not_after', TimestampProperty()), + ('certificate_policies', StringProperty()), + ('policy_mappings', StringProperty()), + ]) class X509Certificate(_Observable): _type = 'x509-certificate' - _properties = { - 'type': TypeProperty(_type), - 'is_self_signed': BooleanProperty(), - 'hashes': HashesProperty(), - 'version': StringProperty(), - 'serial_number': StringProperty(), - 'signature_algorithm': StringProperty(), - 'issuer': StringProperty(), - 'validity_not_before': TimestampProperty(), - 'validity_not_after': TimestampProperty(), - 'subject': StringProperty(), - 'subject_public_key_algorithm': StringProperty(), - 'subject_public_key_modulus': StringProperty(), - 'subject_public_key_exponent': IntegerProperty(), - 'x509_v3_extensions': EmbeddedObjectProperty(type=X509V3ExtenstionsType), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ('is_self_signed', BooleanProperty()), + ('hashes', HashesProperty()), + ('version', StringProperty()), + ('serial_number', StringProperty()), + ('signature_algorithm', StringProperty()), + ('issuer', StringProperty()), + ('validity_not_before', TimestampProperty()), + ('validity_not_after', TimestampProperty()), + ('subject', StringProperty()), + ('subject_public_key_algorithm', StringProperty()), + ('subject_public_key_modulus', StringProperty()), + ('subject_public_key_exponent', IntegerProperty()), + ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), + ]) OBJ_MAP_OBSERVABLE = { @@ -700,7 +738,7 @@ EXT_MAP = { } -def parse_observable(data, _valid_refs=[], allow_custom=False): +def parse_observable(data, _valid_refs, allow_custom=False): """Deserialize a string or file-like object into a STIX Cyber Observable object. Args: @@ -739,17 +777,31 @@ def _register_observable(new_observable): OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable -def CustomObservable(type='x-custom-observable', properties={}): +def CustomObservable(type='x-custom-observable', properties=None): """Custom STIX Cyber Observable type decorator + + Example 1: + + @CustomObservable('x-custom-observable', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) + class MyNewObservableType(): + pass """ def custom_builder(cls): class _Custom(cls, _Observable): _type = type - _properties = { - 'type': TypeProperty(_type), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('type', TypeProperty(_type)), + ]) + + if not properties: + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + _properties.update(properties) def __init__(self, **kwargs): From 81c9eab730a44f889dd35d16b07b51df174c51d1 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 10:34:53 -0400 Subject: [PATCH 35/82] Apply OrderedDict changes to other.py --- stix2/other.py | 70 ++++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/stix2/other.py b/stix2/other.py index cd75745..2dfb804 100644 --- a/stix2/other.py +++ b/stix2/other.py @@ -1,5 +1,7 @@ """STIX 2.0 Objects that are neither SDOs nor SROs""" +from collections import OrderedDict + from .base import _STIXBase from .properties import (IDProperty, ListProperty, Property, ReferenceProperty, SelectorProperty, StringProperty, TimestampProperty, @@ -8,12 +10,13 @@ from .utils import NOW, get_dict class ExternalReference(_STIXBase): - _properties = { - 'source_name': StringProperty(required=True), - 'description': StringProperty(), - 'url': StringProperty(), - 'external_id': StringProperty(), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('source_name', StringProperty(required=True)), + ('description', StringProperty()), + ('url', StringProperty()), + ('external_id', StringProperty()), + ]) def _check_object_constraints(self): super(ExternalReference, self)._check_object_constraints() @@ -21,30 +24,34 @@ class ExternalReference(_STIXBase): class KillChainPhase(_STIXBase): - _properties = { - 'kill_chain_name': StringProperty(required=True), - 'phase_name': StringProperty(required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('kill_chain_name', StringProperty(required=True)), + ('phase_name', StringProperty(required=True)), + ]) class GranularMarking(_STIXBase): - _properties = { - 'marking_ref': ReferenceProperty(required=True, type="marking-definition"), - 'selectors': ListProperty(SelectorProperty, required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('marking_ref', ReferenceProperty(required=True, type="marking-definition")), + ('selectors', ListProperty(SelectorProperty, required=True)), + ]) class TLPMarking(_STIXBase): # TODO: don't allow the creation of any other TLPMarkings than the ones below - _properties = { - 'tlp': Property(required=True) - } + _properties = OrderedDict() + _properties = _properties.update([ + ('tlp', Property(required=True)) + ]) class StatementMarking(_STIXBase): - _properties = { - 'statement': StringProperty(required=True) - } + _properties = OrderedDict() + _properties = _properties.update([ + ('statement', StringProperty(required=True)) + ]) def __init__(self, statement=None, **kwargs): # Allow statement as positional args. @@ -68,17 +75,18 @@ class MarkingProperty(Property): class MarkingDefinition(_STIXBase): _type = 'marking-definition' - _properties = { - 'created': TimestampProperty(default=lambda: NOW), - 'external_references': ListProperty(ExternalReference), - 'created_by_ref': ReferenceProperty(type="identity"), - 'object_marking_refs': ListProperty(ReferenceProperty(type="marking-definition")), - 'granular_markings': ListProperty(GranularMarking), - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'definition_type': StringProperty(required=True), - 'definition': MarkingProperty(required=True), - } + _properties = OrderedDict() + _properties = _properties.update([ + ('created', TimestampProperty(default=lambda: NOW)), + ('external_references', ListProperty(ExternalReference)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('definition_type', StringProperty(required=True)), + ('definition', MarkingProperty(required=True)), + ]) marking_map = { 'tlp': TLPMarking, 'statement': StatementMarking, From 68afd6b38e8f13a36c8977149de4b962b56b1f39 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 10:36:47 -0400 Subject: [PATCH 36/82] Minor changes. --- stix2/other.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/stix2/other.py b/stix2/other.py index 2dfb804..83d42cf 100644 --- a/stix2/other.py +++ b/stix2/other.py @@ -108,29 +108,29 @@ class MarkingDefinition(_STIXBase): TLP_WHITE = MarkingDefinition( - id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="white") + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="white") ) TLP_GREEN = MarkingDefinition( - id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="green") + id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="green") ) TLP_AMBER = MarkingDefinition( - id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="amber") + id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="amber") ) TLP_RED = MarkingDefinition( - id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", - created="2017-01-20T00:00:00.000Z", - definition_type="tlp", - definition=TLPMarking(tlp="red") + id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="red") ) From 26297f97302dddb056c296359b91c409a44c6e35 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 11:52:34 -0400 Subject: [PATCH 37/82] Fix call to update(), add a register_marking decorator. Add type to Markings. --- stix2/bundle.py | 2 +- stix2/observables.py | 74 ++++++++++++++++++++++---------------------- stix2/other.py | 46 +++++++++++++++------------ 3 files changed, 65 insertions(+), 57 deletions(-) diff --git a/stix2/bundle.py b/stix2/bundle.py index 29d4d8b..f3d5b2a 100644 --- a/stix2/bundle.py +++ b/stix2/bundle.py @@ -10,7 +10,7 @@ class Bundle(_STIXBase): _type = 'bundle' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('id', IDProperty(_type)), ('spec_version', Property(fixed="2.0")), diff --git a/stix2/observables.py b/stix2/observables.py index fd66406..5e1bbf8 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -78,7 +78,7 @@ class ExtensionsProperty(DictionaryProperty): class Artifact(_Observable): _type = 'artifact' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('mime_type', StringProperty()), ('payload_bin', BinaryProperty()), @@ -95,7 +95,7 @@ class Artifact(_Observable): class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('number', IntegerProperty()), ('name', StringProperty()), @@ -106,7 +106,7 @@ class AutonomousSystem(_Observable): class Directory(_Observable): _type = 'directory' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('path', StringProperty(required=True)), ('path_enc', StringProperty()), @@ -121,7 +121,7 @@ class Directory(_Observable): class DomainName(_Observable): _type = 'domain-name' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), @@ -131,7 +131,7 @@ class DomainName(_Observable): class EmailAddress(_Observable): _type = 'email-addr' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ('display_name', StringProperty()), @@ -141,7 +141,7 @@ class EmailAddress(_Observable): class EmailMIMEComponent(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('body', StringProperty()), ('body_raw_ref', ObjectReferenceProperty(valid_types=['artifact', 'file'])), ('content_type', StringProperty()), @@ -156,7 +156,7 @@ class EmailMIMEComponent(_STIXBase): class EmailMessage(_Observable): _type = 'email-message' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('is_multipart', BooleanProperty(required=True)), ('date', TimestampProperty()), @@ -184,7 +184,7 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), ('version', StringProperty()), ('comment', StringProperty()), @@ -193,7 +193,7 @@ class ArchiveExt(_Extension): class AlternateDataStream(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('name', StringProperty(required=True)), ('hashes', HashesProperty()), ('size', IntegerProperty()), @@ -202,7 +202,7 @@ class AlternateDataStream(_STIXBase): class NTFSExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('sid', StringProperty()), ('alternate_data_streams', ListProperty(EmbeddedObjectProperty(type=AlternateDataStream))), ]) @@ -210,7 +210,7 @@ class NTFSExt(_Extension): class PDFExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('version', StringProperty()), ('is_optimized', BooleanProperty()), ('document_info_dict', DictionaryProperty()), @@ -221,7 +221,7 @@ class PDFExt(_Extension): class RasterImageExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('image_height', IntegerProperty()), ('image_weight', IntegerProperty()), ('bits_per_pixel', IntegerProperty()), @@ -232,7 +232,7 @@ class RasterImageExt(_Extension): class WindowsPEOptionalHeaderType(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('magic_hex', HexProperty()), ('major_linker_version', IntegerProperty()), ('minor_linker_version', IntegerProperty()), @@ -273,7 +273,7 @@ class WindowsPEOptionalHeaderType(_STIXBase): class WindowsPESection(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('name', StringProperty(required=True)), ('size', IntegerProperty()), ('entropy', FloatProperty()), @@ -283,7 +283,7 @@ class WindowsPESection(_STIXBase): class WindowsPEBinaryExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('pe_type', StringProperty(required=True)), # open_vocab ('imphash', StringProperty()), ('machine_hex', HexProperty()), @@ -302,7 +302,7 @@ class WindowsPEBinaryExt(_Extension): class File(_Observable): _type = 'file' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('extensions', ExtensionsProperty(enclosing_type=_type)), ('hashes', HashesProperty()), @@ -332,7 +332,7 @@ class File(_Observable): class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), @@ -343,7 +343,7 @@ class IPv4Address(_Observable): class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), @@ -354,7 +354,7 @@ class IPv6Address(_Observable): class MACAddress(_Observable): _type = 'mac-addr' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ]) @@ -363,7 +363,7 @@ class MACAddress(_Observable): class Mutex(_Observable): _type = 'mutex' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('name', StringProperty()), ]) @@ -371,7 +371,7 @@ class Mutex(_Observable): class HTTPRequestExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('request_method', StringProperty(required=True)), ('request_value', StringProperty(required=True)), ('request_version', StringProperty()), @@ -383,7 +383,7 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('icmp_type_hex', HexProperty(required=True)), ('icmp_code_hex', HexProperty(required=True)), ]) @@ -391,7 +391,7 @@ class ICMPExt(_Extension): class SocketExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('address_family', EnumProperty([ "AF_UNSPEC", "AF_INET", @@ -425,7 +425,7 @@ class SocketExt(_Extension): class TCPExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('src_flags_hex', HexProperty()), ('dst_flags_hex', HexProperty()), ]) @@ -434,7 +434,7 @@ class TCPExt(_Extension): class NetworkTraffic(_Observable): _type = 'network-traffic' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('extensions', ExtensionsProperty(enclosing_type=_type)), ('start', TimestampProperty()), @@ -463,7 +463,7 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('aslr_enabled', BooleanProperty()), ('dep_enabled', BooleanProperty()), ('priority', StringProperty()), @@ -475,7 +475,7 @@ class WindowsProcessExt(_Extension): class WindowsServiceExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('service_name', StringProperty(required=True)), ('descriptions', ListProperty(StringProperty)), ('display_name', StringProperty()), @@ -509,7 +509,7 @@ class WindowsServiceExt(_Extension): class Process(_Observable): _type = 'process' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('extensions', ExtensionsProperty(enclosing_type=_type)), ('is_hidden', BooleanProperty()), @@ -546,7 +546,7 @@ class Process(_Observable): class Software(_Observable): _type = 'software' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('name', StringProperty(required=True)), ('cpe', StringProperty()), @@ -559,7 +559,7 @@ class Software(_Observable): class URL(_Observable): _type = 'url' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('value', StringProperty(required=True)), ]) @@ -567,7 +567,7 @@ class URL(_Observable): class UNIXAccountExt(_Extension): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('gid', IntegerProperty()), ('groups', ListProperty(StringProperty)), ('home_dir', StringProperty()), @@ -578,7 +578,7 @@ class UNIXAccountExt(_Extension): class UserAccount(_Observable): _type = 'user-account' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('extensions', ExtensionsProperty(enclosing_type=_type)), ('user_id', StringProperty(required=True)), @@ -600,7 +600,7 @@ class UserAccount(_Observable): class WindowsRegistryValueType(_STIXBase): _type = 'windows-registry-value-type' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('name', StringProperty(required=True)), ('data', StringProperty()), ('data_type', EnumProperty([ @@ -624,7 +624,7 @@ class WindowsRegistryValueType(_STIXBase): class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('key', StringProperty(required=True)), ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), @@ -643,7 +643,7 @@ class WindowsRegistryKey(_Observable): class X509V3ExtenstionsType(_STIXBase): _type = 'x509-v3-extensions-type' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('basic_constraints', StringProperty()), ('name_constraints', StringProperty()), ('policy_constraints', StringProperty()), @@ -666,7 +666,7 @@ class X509V3ExtenstionsType(_STIXBase): class X509Certificate(_Observable): _type = 'x509-certificate' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ('is_self_signed', BooleanProperty()), ('hashes', HashesProperty()), @@ -795,7 +795,7 @@ def CustomObservable(type='x-custom-observable', properties=None): class _Custom(cls, _Observable): _type = type _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('type', TypeProperty(_type)), ]) diff --git a/stix2/other.py b/stix2/other.py index 83d42cf..b2492c4 100644 --- a/stix2/other.py +++ b/stix2/other.py @@ -11,7 +11,7 @@ from .utils import NOW, get_dict class ExternalReference(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), @@ -25,7 +25,7 @@ class ExternalReference(_STIXBase): class KillChainPhase(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('kill_chain_name', StringProperty(required=True)), ('phase_name', StringProperty(required=True)), ]) @@ -33,23 +33,24 @@ class KillChainPhase(_STIXBase): class GranularMarking(_STIXBase): _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('marking_ref', ReferenceProperty(required=True, type="marking-definition")), ('selectors', ListProperty(SelectorProperty, required=True)), ]) class TLPMarking(_STIXBase): - # TODO: don't allow the creation of any other TLPMarkings than the ones below + _type = 'tlp' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('tlp', Property(required=True)) ]) class StatementMarking(_STIXBase): + _type = 'statement' _properties = OrderedDict() - _properties = _properties.update([ + _properties.update([ ('statement', StringProperty(required=True)) ]) @@ -67,36 +68,32 @@ class MarkingProperty(Property): """ def clean(self, value): - if type(value) in [TLPMarking, StatementMarking]: + if type(value) in OBJ_MAP_MARKING.values(): return value else: - raise ValueError("must be a Statement or TLP Marking.") + raise ValueError("must be a Statement, TLP Marking or a registered marking.") class MarkingDefinition(_STIXBase): _type = 'marking-definition' _properties = OrderedDict() - _properties = _properties.update([ - ('created', TimestampProperty(default=lambda: NOW)), - ('external_references', ListProperty(ExternalReference)), - ('created_by_ref', ReferenceProperty(type="identity")), - ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), - ('granular_markings', ListProperty(GranularMarking)), + _properties.update([ ('type', TypeProperty(_type)), ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), ('definition_type', StringProperty(required=True)), ('definition', MarkingProperty(required=True)), ]) - marking_map = { - 'tlp': TLPMarking, - 'statement': StatementMarking, - } def __init__(self, **kwargs): if set(('definition_type', 'definition')).issubset(kwargs.keys()): # Create correct marking type object try: - marking_type = self.marking_map[kwargs['definition_type']] + marking_type = OBJ_MAP_MARKING[kwargs['definition_type']] except KeyError: raise ValueError("definition_type must be a valid marking type") @@ -107,6 +104,17 @@ class MarkingDefinition(_STIXBase): super(MarkingDefinition, self).__init__(**kwargs) +def register_marking(new_marking): + """Register a custom STIX Marking Definition type. + """ + OBJ_MAP_MARKING[new_marking._type] = new_marking + + +OBJ_MAP_MARKING = { + 'tlp': TLPMarking, + 'statement': StatementMarking, +} + TLP_WHITE = MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", From 00462eb683cf519eb5fe90b2bfe7093b5cd9fd1e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 14:37:49 -0400 Subject: [PATCH 38/82] Add hashes property to ExternalReference, more documentation for parse_observable --- stix2/observables.py | 10 ++++++---- stix2/other.py | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/stix2/observables.py b/stix2/observables.py index 5e1bbf8..87f95e2 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -734,17 +734,19 @@ EXT_MAP = { 'network-traffic': EXT_MAP_NETWORK_TRAFFIC, 'process': EXT_MAP_PROCESS, 'user-account': EXT_MAP_USER_ACCOUNT, - } def parse_observable(data, _valid_refs, allow_custom=False): - """Deserialize a string or file-like object into a STIX Cyber Observable object. + """Deserialize a string or file-like object into a STIX Cyber Observable + object. Args: data: The STIX 2 string to be parsed. - _valid_refs: A list of object references valid for the scope of the object being parsed. - allow_custom: Whether to allow custom properties or not. Default: False. + _valid_refs: A list of object references valid for the scope of the + object being parsed. Use empty list if no valid refs are present. + allow_custom: Whether to allow custom properties or not. + Default: False. Returns: An instantiated Python STIX Cyber Observable object. diff --git a/stix2/other.py b/stix2/other.py index b2492c4..9659dca 100644 --- a/stix2/other.py +++ b/stix2/other.py @@ -3,9 +3,9 @@ from collections import OrderedDict from .base import _STIXBase -from .properties import (IDProperty, ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) +from .properties import (IDProperty, HashesProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW, get_dict @@ -15,6 +15,7 @@ class ExternalReference(_STIXBase): ('source_name', StringProperty(required=True)), ('description', StringProperty()), ('url', StringProperty()), + ('hashes', HashesProperty()), ('external_id', StringProperty()), ]) From 569ca34d78e74154cf1faacd0d328d28e6738328 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 14 Aug 2017 15:02:26 -0400 Subject: [PATCH 39/82] Remove test skip for data_sources. --- stix2/test/test_data_sources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 7f4ced5..bbb7f31 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -54,7 +54,6 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -@pytest.mark.skip(reason="test_add_get_remove_filter() - Determine what are we testing.") def test_add_get_remove_filter(): # First 3 filters are valid, remaining fields are erroneous in some way From e3f82effc7235f71c3fb28fd6ce101198d36f9ec Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 08:24:43 -0400 Subject: [PATCH 40/82] Apply OrderedDict changes to bundle, fix import in sdo.py --- stix2/core.py | 14 ++++++++------ stix2/sro.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index 81dd492..be2a53d 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -1,5 +1,6 @@ """STIX 2.0 Objects that are neither SDOs nor SROs""" +from collections import OrderedDict from . import exceptions from .base import _STIXBase @@ -31,12 +32,13 @@ class STIXObjectProperty(Property): class Bundle(_STIXBase): _type = 'bundle' - _properties = { - 'type': TypeProperty(_type), - 'id': IDProperty(_type), - 'spec_version': Property(fixed="2.0"), - 'objects': ListProperty(STIXObjectProperty), - } + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('spec_version', Property(fixed="2.0")), + ('objects', ListProperty(STIXObjectProperty)), + ]) def __init__(self, *args, **kwargs): # Add any positional arguments to the 'objects' kwarg. diff --git a/stix2/sro.py b/stix2/sro.py index cf26c32..23f0dd0 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -3,7 +3,7 @@ from collections import OrderedDict from .base import _STIXBase -from .other import ExternalReference, GranularMarking +from .common import ExternalReference, GranularMarking from .properties import (BooleanProperty, IDProperty, IntegerProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty) From 1b48ad9778c4aeb24e48cd7ca09c37e261dfb34f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 13:40:47 -0400 Subject: [PATCH 41/82] Changes to object serialization. --- stix2/base.py | 10 ++++++++-- stix2/utils.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 62bb8c8..a302eb5 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -3,6 +3,7 @@ import collections import copy import datetime as dt + import simplejson as json from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, @@ -11,7 +12,8 @@ from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, MissingPropertiesError, MutuallyExclusivePropertiesError, RevokeError, UnmodifiablePropertyError) -from .utils import NOW, format_datetime, get_timestamp, parse_into_datetime +from .utils import (NOW, find_property_index, format_datetime, get_timestamp, + parse_into_datetime) __all__ = ['STIXJSONEncoder', '_STIXBase'] @@ -145,9 +147,13 @@ class _STIXBase(collections.Mapping): def __str__(self): properties = self._object_properties() + + def sort_by(element): + return find_property_index(self, properties, element) + # separators kwarg -> don't include spaces after commas. return json.dumps(self, indent=4, cls=STIXJSONEncoder, - item_sort_key=lambda x: properties.index(x[0]), + item_sort_key=sort_by, separators=(",", ": ")) def __repr__(self): diff --git a/stix2/utils.py b/stix2/utils.py index 602ff7a..dc25799 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,5 +1,7 @@ """Utility functions and classes for the stix2 library.""" +from stix2 import base + import datetime as dt import json @@ -115,3 +117,37 @@ def get_dict(data): return dict(data) except (ValueError, TypeError): raise ValueError("Cannot convert '%s' to dictionary." % str(data)) + + +def find_property_index(obj, properties, tuple_to_find): + """Recursively find the property in the object model, return the index + according to the _properties OrderedDict. If its a list look for + individual objects. + """ + try: + if tuple_to_find[1] in obj._inner.values(): + return properties.index(tuple_to_find[0]) + raise ValueError + except ValueError: + for pv in obj._inner.values(): + if isinstance(pv, list): + for item in pv: + if isinstance(item, base._STIXBase): + val = find_property_index(item, + item._object_properties(), + tuple_to_find) + if val is not None: + return val + elif isinstance(pv, dict): + if pv.get(tuple_to_find[0]) is not None: + try: + return int(tuple_to_find[0]) + except ValueError: + return len(tuple_to_find[0]) + for item in pv.values(): + if isinstance(item, base._STIXBase): + val = find_property_index(item, + item._object_properties(), + tuple_to_find) + if val is not None: + return val From 4ffc8edeebdecead6f5a2006cebbe80cac9182da Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 13:41:51 -0400 Subject: [PATCH 42/82] Update all tests. Re-organize EXPECTED values, update some regex expressions. --- stix2/test/test_attack_pattern.py | 14 ++++---- stix2/test/test_bundle.py | 30 ++++++++--------- stix2/test/test_campaign.py | 8 ++--- stix2/test/test_course_of_action.py | 8 ++--- stix2/test/test_custom.py | 23 +++++++------- stix2/test/test_external_reference.py | 32 +++++++++++-------- stix2/test/test_identity.py | 6 ++-- stix2/test/test_indicator.py | 18 +++++------ stix2/test/test_intrusion_set.py | 16 +++++----- stix2/test/test_malware.py | 10 +++--- stix2/test/test_markings.py | 38 +++++++++++----------- stix2/test/test_observed_data.py | 46 +++++++++++++-------------- stix2/test/test_relationship.py | 6 ++-- stix2/test/test_report.py | 16 +++++----- stix2/test/test_sighting.py | 4 +-- stix2/test/test_threat_actor.py | 14 ++++---- stix2/test/test_tool.py | 12 +++---- stix2/test/test_vulnerability.py | 16 +++++----- 18 files changed, 162 insertions(+), 155 deletions(-) diff --git a/stix2/test/test_attack_pattern.py b/stix2/test/test_attack_pattern.py index 5bd5af2..07d0898 100644 --- a/stix2/test/test_attack_pattern.py +++ b/stix2/test/test_attack_pattern.py @@ -9,18 +9,18 @@ from .constants import ATTACK_PATTERN_ID EXPECTED = """{ + "type": "attack-pattern", + "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "name": "Spear Phishing", "description": "...", "external_references": [ { - "external_id": "CAPEC-163", - "source_name": "capec" + "source_name": "capec", + "external_id": "CAPEC-163" } - ], - "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", - "modified": "2016-05-12T08:17:27.000Z", - "name": "Spear Phishing", - "type": "attack-pattern" + ] }""" diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 0733637..9b57712 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -4,41 +4,41 @@ import stix2 EXPECTED_BUNDLE = """{ + "type": "bundle", "id": "bundle--00000000-0000-0000-0000-000000000004", + "spec_version": "2.0", "objects": [ { - "created": "2017-01-01T12:34:56.000Z", + "type": "indicator", "id": "indicator--00000000-0000-0000-0000-000000000001", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", "labels": [ "malicious-activity" ], - "modified": "2017-01-01T12:34:56.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", - "type": "indicator", "valid_from": "2017-01-01T12:34:56Z" }, { - "created": "2017-01-01T12:34:56.000Z", + "type": "malware", "id": "malware--00000000-0000-0000-0000-000000000002", - "labels": [ - "ransomware" - ], + "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "name": "Cryptolocker", - "type": "malware" + "labels": [ + "ransomware" + ] }, { - "created": "2017-01-01T12:34:56.000Z", + "type": "relationship", "id": "relationship--00000000-0000-0000-0000-000000000003", + "created": "2017-01-01T12:34:56.000Z", "modified": "2017-01-01T12:34:56.000Z", "relationship_type": "indicates", "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "type": "relationship" + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" } - ], - "spec_version": "2.0", - "type": "bundle" + ] }""" @@ -84,7 +84,7 @@ def test_bundle_with_wrong_spec_version(): def test_create_bundle(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) - + print(bundle) assert str(bundle) == EXPECTED_BUNDLE diff --git a/stix2/test/test_campaign.py b/stix2/test/test_campaign.py index 30b9444..202534d 100644 --- a/stix2/test/test_campaign.py +++ b/stix2/test/test_campaign.py @@ -9,13 +9,13 @@ from .constants import CAMPAIGN_ID EXPECTED = """{ - "created": "2016-04-06T20:03:00.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "type": "campaign", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", "name": "Green Group Attacks Against Finance", - "type": "campaign" + "description": "Campaign by Green Group against a series of targets in the financial services sector." }""" diff --git a/stix2/test/test_course_of_action.py b/stix2/test/test_course_of_action.py index e7a1b22..3dc379d 100644 --- a/stix2/test/test_course_of_action.py +++ b/stix2/test/test_course_of_action.py @@ -9,13 +9,13 @@ from .constants import COURSE_OF_ACTION_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ...", + "type": "course-of-action", "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "Add TCP port 80 Filter Rule to the existing Block UDP 1434 Filter", - "type": "course-of-action" + "description": "This is how to add a filter rule to block inbound access to TCP port 80 to the existing UDP 1434 filter ..." }""" diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 60e982c..4ce99c6 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -74,11 +74,11 @@ def test_parse_identity_custom_property(data): assert identity.foo == "bar" -@stix2.sdo.CustomObject('x-new-type', { - 'property1': stix2.properties.StringProperty(required=True), - 'property2': stix2.properties.IntegerProperty(), -}) -class NewType(): +@stix2.sdo.CustomObject('x-new-type', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), +]) +class NewType(object): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") @@ -106,11 +106,12 @@ def test_parse_custom_object_type(): assert nt.property1 == 'something' -@stix2.observables.CustomObservable('x-new-observable', { - 'property1': stix2.properties.StringProperty(required=True), - 'property2': stix2.properties.IntegerProperty(), -}) -class NewObservable(): +@stix2.observables.CustomObservable('x-new-observable', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ('x_property3', stix2.properties.BooleanProperty()), +]) +class NewObservable(object): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") @@ -133,7 +134,7 @@ def test_parse_custom_observable_object(): "property1": "something" }""" - nt = stix2.parse_observable(nt_string) + nt = stix2.parse_observable(nt_string, []) assert nt.property1 == 'something' diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py index 10a6e53..c4c3755 100644 --- a/stix2/test/test_external_reference.py +++ b/stix2/test/test_external_reference.py @@ -8,9 +8,12 @@ import stix2 VERIS = """{ - "external_id": "0001AA7F-C601-424A-B2B8-BE6C9F5164E7", "source_name": "veris", - "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json" + "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", + "hashes": { + "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b" + }, + "external_id": "0001AA7F-C601-424A-B2B8-BE6C9F5164E7" }""" @@ -18,6 +21,9 @@ def test_external_reference_veris(): ref = stix2.ExternalReference( source_name="veris", external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7", + hashes={ + "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b" + }, url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json", ) @@ -25,8 +31,8 @@ def test_external_reference_veris(): CAPEC = """{ - "external_id": "CAPEC-550", - "source_name": "capec" + "source_name": "capec", + "external_id": "CAPEC-550" }""" @@ -37,13 +43,13 @@ def test_external_reference_capec(): ) assert str(ref) == CAPEC - assert re.match("ExternalReference\(external_id=u?'CAPEC-550', source_name=u?'capec'\)", repr(ref)) + assert re.match("ExternalReference\(source_name=u?'capec', external_id=u?'CAPEC-550'\)", repr(ref)) CAPEC_URL = """{ - "external_id": "CAPEC-550", "source_name": "capec", - "url": "http://capec.mitre.org/data/definitions/550.html" + "url": "http://capec.mitre.org/data/definitions/550.html", + "external_id": "CAPEC-550" }""" @@ -58,8 +64,8 @@ def test_external_reference_capec_url(): THREAT_REPORT = """{ - "description": "Threat report", "source_name": "ACME Threat Intel", + "description": "Threat report", "url": "http://www.example.com/threat-report.pdf" }""" @@ -75,9 +81,9 @@ def test_external_reference_threat_report(): BUGZILLA = """{ - "external_id": "1370", "source_name": "ACME Bugzilla", - "url": "https://www.example.com/bugs/1370" + "url": "https://www.example.com/bugs/1370", + "external_id": "1370" }""" @@ -92,8 +98,8 @@ def test_external_reference_bugzilla(): OFFLINE = """{ - "description": "Threat report", - "source_name": "ACME Threat Intel" + "source_name": "ACME Threat Intel", + "description": "Threat report" }""" @@ -104,7 +110,7 @@ def test_external_reference_offline(): ) assert str(ref) == OFFLINE - assert re.match("ExternalReference\(description=u?'Threat report', source_name=u?'ACME Threat Intel'\)", repr(ref)) + assert re.match("ExternalReference\(source_name=u?'ACME Threat Intel', description=u?'Threat report'\)", repr(ref)) # Yikes! This works assert eval("stix2." + repr(ref)) == ref diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py index ed51958..a9415fe 100644 --- a/stix2/test/test_identity.py +++ b/stix2/test/test_identity.py @@ -9,12 +9,12 @@ from .constants import IDENTITY_ID EXPECTED = """{ - "created": "2015-12-21T19:59:11.000Z", + "type": "identity", "id": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c", - "identity_class": "individual", + "created": "2015-12-21T19:59:11.000Z", "modified": "2015-12-21T19:59:11.000Z", "name": "John Smith", - "type": "identity" + "identity_class": "individual" }""" diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index 5daa0f6..5a7ae5a 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -10,25 +10,25 @@ from .constants import FAKE_TIME, INDICATOR_ID, INDICATOR_KWARGS EXPECTED_INDICATOR = """{ - "created": "2017-01-01T00:00:01.000Z", + "type": "indicator", "id": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "created": "2017-01-01T00:00:01.000Z", + "modified": "2017-01-01T00:00:01.000Z", "labels": [ "malicious-activity" ], - "modified": "2017-01-01T00:00:01.000Z", "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", - "type": "indicator", "valid_from": "1970-01-01T00:00:01Z" }""" EXPECTED_INDICATOR_REPR = "Indicator(" + " ".join(""" - created=STIXdatetime(2017, 1, 1, 0, 0, 1, tzinfo=), - id='indicator--01234567-89ab-cdef-0123-456789abcdef', - labels=['malicious-activity'], - modified=STIXdatetime(2017, 1, 1, 0, 0, 1, tzinfo=), - pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", type='indicator', - valid_from=datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=) + id='indicator--01234567-89ab-cdef-0123-456789abcdef', + created='2017-01-01T00:00:01.000Z', + modified='2017-01-01T00:00:01.000Z', + labels=['malicious-activity'], + pattern="[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + valid_from='1970-01-01T00:00:01Z' """.split()) + ")" diff --git a/stix2/test/test_intrusion_set.py b/stix2/test/test_intrusion_set.py index a6eee7f..481b3cb 100644 --- a/stix2/test/test_intrusion_set.py +++ b/stix2/test/test_intrusion_set.py @@ -9,21 +9,21 @@ from .constants import INTRUSION_SET_ID EXPECTED = """{ + "type": "intrusion-set", + "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", + "modified": "2016-04-06T20:03:48.000Z", + "name": "Bobcat Breakin", + "description": "Incidents usually feature a shared TTP of a bobcat being released...", "aliases": [ "Zookeeper" ], - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Incidents usually feature a shared TTP of a bobcat being released...", "goals": [ "acquisition-theft", "harassment", "damage" - ], - "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29", - "modified": "2016-04-06T20:03:48.000Z", - "name": "Bobcat Breakin", - "type": "intrusion-set" + ] }""" diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py index ff0b394..7f665ea 100644 --- a/stix2/test/test_malware.py +++ b/stix2/test/test_malware.py @@ -10,14 +10,14 @@ from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS EXPECTED_MALWARE = """{ - "created": "2016-05-12T08:17:27.000Z", + "type": "malware", "id": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "labels": [ - "ransomware" - ], + "created": "2016-05-12T08:17:27.000Z", "modified": "2016-05-12T08:17:27.000Z", "name": "Cryptolocker", - "type": "malware" + "labels": [ + "ransomware" + ] }""" diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 70d67dd..f36b032 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -10,36 +10,36 @@ from .constants import MARKING_DEFINITION_ID EXPECTED_TLP_MARKING_DEFINITION = """{ + "type": "marking-definition", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "created": "2017-01-20T00:00:00Z", + "definition_type": "tlp", "definition": { "tlp": "white" - }, - "definition_type": "tlp", - "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - "type": "marking-definition" + } }""" EXPECTED_STATEMENT_MARKING_DEFINITION = """{ + "type": "marking-definition", + "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "created": "2017-01-20T00:00:00Z", + "definition_type": "statement", "definition": { "statement": "Copyright 2016, Example Corp" - }, - "definition_type": "statement", - "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", - "type": "marking-definition" + } }""" EXPECTED_CAMPAIGN_WITH_OBJECT_MARKING = """{ - "created": "2016-04-06T20:03:00.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "type": "campaign", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", "modified": "2016-04-06T20:03:00.000Z", "name": "Green Group Attacks Against Finance", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", "object_marking_refs": [ "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" - ], - "type": "campaign" + ] }""" EXPECTED_GRANULAR_MARKING = """{ @@ -53,8 +53,12 @@ EXPECTED_GRANULAR_MARKING = """{ }""" EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ - "created": "2016-04-06T20:03:00.000Z", + "type": "campaign", + "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "name": "Green Group Attacks Against Finance", "description": "Campaign by Green Group against a series of targets in the financial services sector.", "granular_markings": [ { @@ -63,11 +67,7 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ "description" ] } - ], - "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "modified": "2016-04-06T20:03:00.000Z", - "name": "Green Group Attacks Against Finance", - "type": "campaign" + ] }""" diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index d5641e7..45610f9 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -10,20 +10,20 @@ from .constants import OBSERVED_DATA_ID EXPECTED = """{ - "created": "2016-04-06T19:58:16.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "first_observed": "2015-12-21T19:00:00Z", + "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - "last_observed": "2015-12-21T19:00:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "objects": { "0": { - "name": "foo.exe", - "type": "file" + "type": "file", + "name": "foo.exe" } - }, - "type": "observed-data" + } }""" @@ -48,27 +48,27 @@ def test_observed_data_example(): EXPECTED_WITH_REF = """{ - "created": "2016-04-06T19:58:16.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "first_observed": "2015-12-21T19:00:00Z", + "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - "last_observed": "2015-12-21T19:00:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "objects": { "0": { - "name": "foo.exe", - "type": "file" + "type": "file", + "name": "foo.exe" }, "1": { + "type": "directory", + "path": "/usr/home", "contains_refs": [ "0" - ], - "path": "/usr/home", - "type": "directory" + ] } - }, - "type": "observed-data" + } }""" @@ -173,7 +173,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +194,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -215,7 +215,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +358,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('"objects".+\},', re.DOTALL).sub('"objects": { %s },' % data, EXPECTED) + odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py index 362348b..6d65544 100644 --- a/stix2/test/test_relationship.py +++ b/stix2/test/test_relationship.py @@ -10,13 +10,13 @@ from .constants import (FAKE_TIME, INDICATOR_ID, MALWARE_ID, RELATIONSHIP_ID, EXPECTED_RELATIONSHIP = """{ - "created": "2016-04-06T20:06:37.000Z", + "type": "relationship", "id": "relationship--00000000-1111-2222-3333-444444444444", + "created": "2016-04-06T20:06:37.000Z", "modified": "2016-04-06T20:06:37.000Z", "relationship_type": "indicates", "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210", - "type": "relationship" + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" }""" diff --git a/stix2/test/test_report.py b/stix2/test/test_report.py index 4a1f905..a5775e3 100644 --- a/stix2/test/test_report.py +++ b/stix2/test/test_report.py @@ -9,22 +9,22 @@ from .constants import INDICATOR_KWARGS, REPORT_ID EXPECTED = """{ - "created": "2015-12-21T19:59:11.000Z", - "created_by_ref": "identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", - "description": "A simple report with an indicator and campaign", + "type": "report", "id": "report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3", - "labels": [ - "campaign" - ], + "created_by_ref": "identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283", + "created": "2015-12-21T19:59:11.000Z", "modified": "2015-12-21T19:59:11.000Z", "name": "The Black Vine Cyberespionage Group", + "description": "A simple report with an indicator and campaign", + "published": "2016-01-20T17:00:00Z", "object_refs": [ "indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2", "campaign--83422c77-904c-4dc1-aff5-5c38f3a2c55c", "relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a" ], - "published": "2016-01-20T17:00:00Z", - "type": "report" + "labels": [ + "campaign" + ] }""" diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py index 2036457..af91413 100644 --- a/stix2/test/test_sighting.py +++ b/stix2/test/test_sighting.py @@ -9,11 +9,11 @@ from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS EXPECTED_SIGHTING = """{ - "created": "2016-04-06T20:06:37.000Z", + "type": "sighting", "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb", + "created": "2016-04-06T20:06:37.000Z", "modified": "2016-04-06T20:06:37.000Z", "sighting_of_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", - "type": "sighting", "where_sighted_refs": [ "identity--8cc7afd6-5455-4d2b-a736-e614ee631d99" ] diff --git a/stix2/test/test_threat_actor.py b/stix2/test/test_threat_actor.py index 1bab744..c095c3b 100644 --- a/stix2/test/test_threat_actor.py +++ b/stix2/test/test_threat_actor.py @@ -9,16 +9,16 @@ from .constants import THREAT_ACTOR_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "description": "The Evil Org threat actor group", + "type": "threat-actor", "id": "threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "labels": [ - "crime-syndicate" - ], + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "Evil Org", - "type": "threat-actor" + "description": "The Evil Org threat actor group", + "labels": [ + "crime-syndicate" + ] }""" diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py index 04da7b3..be52f6d 100644 --- a/stix2/test/test_tool.py +++ b/stix2/test/test_tool.py @@ -9,15 +9,15 @@ from .constants import TOOL_ID EXPECTED = """{ - "created": "2016-04-06T20:03:48.000Z", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "type": "tool", "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "labels": [ - "remote-access" - ], + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "name": "VNC", - "type": "tool" + "labels": [ + "remote-access" + ] }""" diff --git a/stix2/test/test_vulnerability.py b/stix2/test/test_vulnerability.py index 27ab85f..a6426d8 100644 --- a/stix2/test/test_vulnerability.py +++ b/stix2/test/test_vulnerability.py @@ -9,17 +9,17 @@ from .constants import VULNERABILITY_ID EXPECTED = """{ - "created": "2016-05-12T08:17:27.000Z", - "external_references": [ - { - "external_id": "CVE-2016-1234", - "source_name": "cve" - } - ], + "type": "vulnerability", "id": "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", + "created": "2016-05-12T08:17:27.000Z", "modified": "2016-05-12T08:17:27.000Z", "name": "CVE-2016-1234", - "type": "vulnerability" + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2016-1234" + } + ] }""" From 15b5e107d57e09ed0b81f01c6626ee863b345525 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 13:54:16 -0400 Subject: [PATCH 43/82] Fix imports, sort requirements. --- setup.py | 6 +++--- stix2/sources/__init__.py | 1 - stix2/sro.py | 6 +++--- stix2/utils.py | 8 ++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index a034bda..3a681fa 100644 --- a/setup.py +++ b/setup.py @@ -47,10 +47,10 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'pytz', - 'six', 'python-dateutil', + 'pytz', 'requests', - 'simplejson' + 'simplejson', + 'six' ], ) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index c4a0f0f..6b8513e 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -529,7 +529,6 @@ class CompositeDataSource(object): """ return copy.deepcopy(self.data_sources.values()) - def deduplicate(self, stix_obj_list): """Deduplicate a list of STIX objects to a unique set diff --git a/stix2/sro.py b/stix2/sro.py index 23f0dd0..af483bc 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -4,9 +4,9 @@ from collections import OrderedDict from .base import _STIXBase from .common import ExternalReference, GranularMarking -from .properties import (BooleanProperty, IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) +from .properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW diff --git a/stix2/utils.py b/stix2/utils.py index dc25799..aa0caa6 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,13 +1,13 @@ """Utility functions and classes for the stix2 library.""" -from stix2 import base - import datetime as dt import json from dateutil import parser import pytz +from .base import _STIXBase + # Sentinel value for properties that should be set to the current time. # We can't use the standard 'default' approach, since if there are multiple # timestamps in a single object, the timestamps will vary by a few microseconds. @@ -132,7 +132,7 @@ def find_property_index(obj, properties, tuple_to_find): for pv in obj._inner.values(): if isinstance(pv, list): for item in pv: - if isinstance(item, base._STIXBase): + if isinstance(item, _STIXBase): val = find_property_index(item, item._object_properties(), tuple_to_find) @@ -145,7 +145,7 @@ def find_property_index(obj, properties, tuple_to_find): except ValueError: return len(tuple_to_find[0]) for item in pv.values(): - if isinstance(item, base._STIXBase): + if isinstance(item, _STIXBase): val = find_property_index(item, item._object_properties(), tuple_to_find) From a18804a19537f354b80a9de96470ebaa9f565577 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 14:12:21 -0400 Subject: [PATCH 44/82] Add ordereddict dependency for 2.6 support. Import change in utils.py --- setup.py | 1 + stix2/common.py | 5 ++++- stix2/core.py | 5 ++++- stix2/observables.py | 5 ++++- stix2/sdo.py | 5 ++++- stix2/sro.py | 5 ++++- stix2/utils.py | 3 +-- 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 3a681fa..7b544ca 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ + 'ordereddict' 'python-dateutil', 'pytz', 'requests', diff --git a/stix2/common.py b/stix2/common.py index 555ca98..c126d76 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,6 +1,9 @@ """STIX 2 Common Data Types and Properties""" -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from .base import _STIXBase from .properties import (BooleanProperty, HashesProperty, IDProperty, diff --git a/stix2/core.py b/stix2/core.py index be2a53d..0d0d1d2 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -1,6 +1,9 @@ """STIX 2.0 Objects that are neither SDOs nor SROs""" -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from . import exceptions from .base import _STIXBase diff --git a/stix2/observables.py b/stix2/observables.py index 87f95e2..4a1319c 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -5,7 +5,10 @@ embedded in Email Message objects, inherit from _STIXBase instead of Observable and do not have a '_type' attribute. """ -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from .base import _Extension, _Observable, _STIXBase from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, diff --git a/stix2/sdo.py b/stix2/sdo.py index 4904a53..b2b0e7e 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -1,6 +1,9 @@ """STIX 2.0 Domain Objects""" -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict import stix2 diff --git a/stix2/sro.py b/stix2/sro.py index af483bc..05a29ed 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -1,6 +1,9 @@ """STIX 2.0 Relationship Objects.""" -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from .base import _STIXBase from .common import ExternalReference, GranularMarking diff --git a/stix2/utils.py b/stix2/utils.py index aa0caa6..9e0f33a 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -6,8 +6,6 @@ import json from dateutil import parser import pytz -from .base import _STIXBase - # Sentinel value for properties that should be set to the current time. # We can't use the standard 'default' approach, since if there are multiple # timestamps in a single object, the timestamps will vary by a few microseconds. @@ -124,6 +122,7 @@ def find_property_index(obj, properties, tuple_to_find): according to the _properties OrderedDict. If its a list look for individual objects. """ + from .base import _STIXBase try: if tuple_to_find[1] in obj._inner.values(): return properties.index(tuple_to_find[0]) From f67d34990c7105d4db83d791e7b851ae730ac556 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 15 Aug 2017 14:16:49 -0400 Subject: [PATCH 45/82] Style error. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7b544ca..c10da46 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'ordereddict' + 'ordereddict', 'python-dateutil', 'pytz', 'requests', From 681be1a5d9b036407c3bedfbe27882016ca8cef9 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 16 Aug 2017 09:58:33 -0400 Subject: [PATCH 46/82] Make CompositeDataStore subclass DataStore. Remove deduplicate() from CompositeDataSource. --- stix2/sources/__init__.py | 58 ++++++++++++--------------------------- stix2/test/test_bundle.py | 2 +- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 6b8513e..d8676ca 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -344,7 +344,7 @@ class DataSource(object): return list(unique_objs.values()) -class CompositeDataSource(object): +class CompositeDataSource(DataSource): """Composite Data Source Acts as a controller for all the defined/configured STIX Data Sources @@ -353,14 +353,9 @@ class CompositeDataSource(object): application. Attributes: - id (str): A UUIDv4 to identify this CompositeDataSource. name (str): The name that identifies this CompositeDataSource. data_sources (dict): A dictionary of DataSource objects; to be controlled and used by the Data Source Controller object. - filters (dict): A collection of filters present in this - CompositeDataSource. - filter_allowed (dict): A collection of the allowed filters in this - CompositeDataSource. """ def __init__(self, name="CompositeDataSource"): @@ -372,13 +367,10 @@ class CompositeDataSource(object): CompositeDataSource instance. """ - self.id = make_id() - self.name = name + super(CompositeDataSource, self).__init__(name=name) self.data_sources = {} - self.filters = {} - self.filter_allowed = {} - def get(self, stix_id): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object by 'id' Federated retrieve method-iterates through all STIX data sources @@ -394,6 +386,9 @@ class CompositeDataSource(object): Args: stix_id (str): the id of the STIX object to retrieve. + _composite_filters (list): a list of filters passed from the + Composite Data Source + Returns: stix_obj (dict): the STIX object to be returned. @@ -402,7 +397,7 @@ class CompositeDataSource(object): # for every configured Data Source, call its retrieve handler for ds_id, ds in iteritems(self.data_sources): - data = ds.get(stix_id=stix_id, _composite_filters=self.filters.values()) + data = ds.get(stix_id=stix_id, _composite_filters=list(self.filters)) all_data.extend(data) # remove duplicate versions @@ -414,7 +409,7 @@ class CompositeDataSource(object): return stix_obj - def all_versions(self, stix_id): + def all_versions(self, stix_id, _composite_filters=None): """Retrieve STIX objects by 'id' Federated all_versions retrieve method - iterates through all STIX data @@ -427,6 +422,9 @@ class CompositeDataSource(object): Args: stix_id (str): id of the STIX objects to retrieve + _composite_filters (list): a list of filters passed from the + Composite Data Source + Returns: all_data (list): list of STIX objects that have the specified id @@ -435,7 +433,7 @@ class CompositeDataSource(object): # retrieve STIX objects from all configured data sources for ds_id, ds in iteritems(self.data_sources): - data = ds.all_versions(stix_id=stix_id, _composite_filters=self.filters.values()) + data = ds.all_versions(stix_id=stix_id, _composite_filters=list(self.filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -445,14 +443,17 @@ class CompositeDataSource(object): return all_data - def query(self, query=None): + def query(self, query=None, _composite_filters=None): """Composite data source query Federate the query to all Data Sources attached to the Composite Data Source. Args: - query (list): list of filters to search on + query (list): list of filters to search on. + + _composite_filters (list): a list of filters passed from the + Composite Data Source Returns: all_data (list): list of STIX objects to be returned @@ -466,7 +467,7 @@ class CompositeDataSource(object): # federate query to all attached data sources, # pass composite filters to id for ds_id, ds in iteritems(self.data_sources): - data = ds.query(query=query, _composite_filters=self.filters.values()) + data = ds.query(query=query, _composite_filters=list(self.filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 @@ -508,8 +509,6 @@ class CompositeDataSource(object): data_source_ids (list): a list of Data Source id's(which are strings) - Returns: - """ for id_ in data_source_ids: @@ -529,27 +528,6 @@ class CompositeDataSource(object): """ return copy.deepcopy(self.data_sources.values()) - def deduplicate(self, stix_obj_list): - """Deduplicate a list of STIX objects to a unique set - - Reduces a set of STIX objects to unique set by looking - at 'id' and 'modified' fields - as a unique object version - is determined by the combination of those fields - - Args: - stix_obj_list (list): list of STIX objects (dicts) - - Returns: - A list with a unique set of the passed list of STIX objects. - - """ - unique_objs = {} - - for obj in stix_obj_list: - unique_objs[(obj["id"], obj["modified"])] = obj - - return list(unique_objs.values()) - class STIXCommonPropertyFilters(object): """ diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 9b57712..b5c1da5 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -84,7 +84,7 @@ def test_bundle_with_wrong_spec_version(): def test_create_bundle(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) - print(bundle) + assert str(bundle) == EXPECTED_BUNDLE From cd1851c56b7a974107e2c7852ec3bb810d1f6f4a Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 18 Aug 2017 08:59:27 -0400 Subject: [PATCH 47/82] Fix pattern in code example in README See also #43. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b2a8c9c..03c7cc3 100644 --- a/README.rst +++ b/README.rst @@ -39,8 +39,8 @@ constructor: from stix2 import Indicator indicator = Indicator(name="File hash for malware variant", - labels=['malicious-activity'], - pattern='file:hashes.md5 = "d41d8cd98f00b204e9800998ecf8427e"') + labels=["malicious-activity"], + pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") Certain required attributes of all objects will be set automatically if not provided as keyword arguments: From c64b7de761ecd9d246352fe8aa24db6fac510222 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 18 Aug 2017 12:05:12 -0400 Subject: [PATCH 48/82] Change query dict style for Filter namedtuple. --- stix2/sources/filesystem.py | 8 ++------ stix2/sources/memory.py | 8 ++------ stix2/sources/taxii.py | 14 +++----------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 0613ac0..e16ee0c 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -13,7 +13,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): @@ -78,11 +78,7 @@ class FileSystemSource(DataSource): """ """ query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } + Filter("id", "=", stix_id) ] all_data = self.query(query=query, _composite_filters=_composite_filters) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 24f3c1f..2f45d68 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -22,7 +22,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter from stix2validator import validate_string @@ -205,11 +205,7 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } + Filter("id", "=", stix_id) ] all_data = self.query(query=query, _composite_filters=_composite_filters) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 47ad8ed..4edeeed 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -12,7 +12,7 @@ TODO: Test everything import json -from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources import DataSink, DataSource, DataStore, Filter, make_id TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -89,16 +89,8 @@ class TAXIICollectionSource(DataSource): """ # make query in TAXII query format since 'id' is TAXII field query = [ - { - "field": "match[id]", - "op": "=", - "value": stix_id - }, - { - "field": "match[version]", - "op": "=", - "value": "all" - } + Filter("match[id]", "=", stix_id), + Filter("match[version]", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) From c0d02fbfcd61668615327bd250e5822446fb5006 Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 18 Aug 2017 12:36:34 -0400 Subject: [PATCH 49/82] [WIP] Touch up marking code --- stix2/exceptions.py | 17 +++++++++++++++-- stix2/markings/__init__.py | 4 +--- stix2/markings/granular_markings.py | 6 +++--- stix2/test/test_granular_markings.py | 13 +++++-------- stix2/test/test_object_markings.py | 4 ++-- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 708233a..864dec8 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -172,14 +172,27 @@ class InvalidSelectorError(STIXError, ValueError): return msg.format(self.key, self.__class__.__name__) -class DuplicateMarkingError(STIXError, ValueError): +class InvalidMarkingError(STIXError, ValueError): """Marking violation. The marking reference must be a valid identifier.""" + def __init__(self, cls, key): + super(InvalidMarkingError, self).__init__() + self.cls = cls + self.key = key + + def __str__(self): + msg = "Marking '{0}' in '{1}' is not a valid marking reference." + return msg.format(self.key, self.__class__.__name__) + + +class DuplicateMarkingError(STIXError, ValueError): + """Marking violation. The marking reference is a duplicate.""" + def __init__(self, cls, key): super(DuplicateMarkingError, self).__init__() self.cls = cls self.key = key def __str__(self): - msg = "Marking '{0}' in '{1}' is not a valid marking reference." + msg = "Marking '{0}' in '{1}' is a duplicate marking reference." return msg.format(self.key, self.__class__.__name__) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 14d21bb..f543f34 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -5,9 +5,7 @@ These high level functions will operate on both object level markings and granular markings unless otherwise noted in each of the functions. """ -from stix2.markings import utils -from stix2.markings import granular_markings -from stix2.markings import object_markings +from stix2.markings import granular_markings, object_markings def get_markings(obj, selectors, inherited=False, descendants=False): diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 6d69f16..903aae9 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -8,7 +8,7 @@ def get_markings(obj, selectors, inherited=False, descendants=False): Args: obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in + selectors: string or list of selector strings relative to the TLO in which the field(s) appear(s). inherited: If True, include markings inherited relative to the field(s). descendants: If True, include granular markings applied to any children @@ -47,7 +47,7 @@ def set_markings(obj, selectors, marking): Args: obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in + selectors: string or list of selector strings relative to the TLO in which the field(s) appear(s). marking: identifier or list of marking identifiers that apply to the field(s) selected by `selectors`. @@ -134,7 +134,7 @@ def add_markings(obj, selectors, marking): def clear_markings(obj, selectors): """ - Removes all granular_marking associated with the selectors. + Removes all granular_markings associated with the selectors. Args: obj: A TLO object. diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index da53ce1..f38c0dd 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -1,8 +1,8 @@ -from stix2 import markings - import pytest +from stix2 import markings + """Tests for the Data Markings API.""" @@ -141,8 +141,7 @@ def test_add_marking_bad_selector(data, marking): markings.add_markings(data, marking[0], marking[1]) -GET_MARKINGS_TEST_DATA = \ -{ +GET_MARKINGS_TEST_DATA = { "a": 333, "b": "value", "c": [ @@ -510,8 +509,7 @@ def test_remove_marking_bad_selector(): markings.remove_markings(before, ["title"], ["marking-definition--1", "marking-definition--2"]) -IS_MARKED_TEST_DATA = \ -{ +IS_MARKED_TEST_DATA = { "title": "test title", "description": "test description", "revision": 2, @@ -905,8 +903,7 @@ def test_set_marking_mark_same_property_same_marking(): assert before == after -CLEAR_MARKINGS_TEST_DATA = \ -{ +CLEAR_MARKINGS_TEST_DATA = { "title": "test title", "description": "test description", "revision": 2, diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index 5205ad7..15f35a4 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -1,8 +1,8 @@ -from stix2 import markings - import pytest +from stix2 import markings + """Tests for the Data Markings API.""" From d4edb8b0bc84e8219f2c1121ea6f50feffc80272 Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 18 Aug 2017 14:22:57 -0400 Subject: [PATCH 50/82] Validate patterns when creating Indicators --- setup.py | 1 + stix2/properties.py | 16 ++++++++++++++++ stix2/sdo.py | 6 +++--- stix2/test/test_indicator.py | 20 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index efe754c..5420e04 100644 --- a/setup.py +++ b/setup.py @@ -50,5 +50,6 @@ setup( 'six', 'python-dateutil', 'requests', + 'stix2-patterns', ], ) diff --git a/stix2/properties.py b/stix2/properties.py index f63ec8b..a04294e 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -7,6 +7,8 @@ import uuid from six import string_types, text_type +from stix2patterns.validator import run_validator + from .base import _STIXBase from .exceptions import DictionaryKeyError from .utils import get_dict, parse_into_datetime @@ -370,3 +372,17 @@ class EnumProperty(StringProperty): if value not in self.allowed: raise ValueError("value '%s' is not valid for this enumeration." % value) return self.string_type(value) + + +class PatternProperty(StringProperty): + + def __init__(self, **kwargs): + super(PatternProperty, self).__init__(**kwargs) + + def clean(self, value): + str_value = super(PatternProperty, self).clean(value) + errors = run_validator(str_value) + if errors: + raise ValueError(str(errors[0])) + + return self.string_type(value) diff --git a/stix2/sdo.py b/stix2/sdo.py index 43c8328..86e9f39 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -6,8 +6,8 @@ from .base import _STIXBase from .common import COMMON_PROPERTIES, KillChainPhase from .observables import ObservableProperty from .properties import (IDProperty, IntegerProperty, ListProperty, - ReferenceProperty, StringProperty, TimestampProperty, - TypeProperty) + PatternProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW @@ -77,7 +77,7 @@ class Indicator(_STIXBase): 'labels': ListProperty(StringProperty, required=True), 'name': StringProperty(), 'description': StringProperty(), - 'pattern': StringProperty(required=True), + 'pattern': PatternProperty(required=True), 'valid_from': TimestampProperty(default=lambda: NOW), 'valid_until': TimestampProperty(), 'kill_chain_phases': ListProperty(KillChainPhase), diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py index 5daa0f6..3a486df 100644 --- a/stix2/test/test_indicator.py +++ b/stix2/test/test_indicator.py @@ -174,3 +174,23 @@ def test_parse_indicator(data): assert idctr.valid_from == dt.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc) assert idctr.labels[0] == "malicious-activity" assert idctr.pattern == "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']" + + +def test_invalid_indicator_pattern(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern="file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e'", + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'input is missing square brackets' in excinfo.value.reason + + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.Indicator( + labels=['malicious-activity'], + pattern='[file:hashes.MD5 = "d41d8cd98f00b204e9800998ecf8427e"]', + ) + assert excinfo.value.cls == stix2.Indicator + assert excinfo.value.prop_name == 'pattern' + assert 'mismatched input' in excinfo.value.reason From 868752111148804fe26e2b02f4e104a2cbeb651c Mon Sep 17 00:00:00 2001 From: clenk Date: Mon, 21 Aug 2017 13:57:01 -0400 Subject: [PATCH 51/82] Change object markings for immutable SDOs --- stix2/markings/__init__.py | 12 +- stix2/markings/object_markings.py | 7 +- stix2/properties.py | 1 + stix2/test/constants.py | 9 ++ stix2/test/test_object_markings.py | 189 ++++++++++++++--------------- 5 files changed, 113 insertions(+), 105 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index f543f34..f158723 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -63,9 +63,9 @@ def set_markings(obj, selectors, marking): """ if selectors is None: - object_markings.set_markings(obj, marking) + return object_markings.set_markings(obj, marking) else: - granular_markings.set_markings(obj, selectors, marking) + return granular_markings.set_markings(obj, selectors, marking) def remove_markings(obj, selectors, marking): @@ -89,9 +89,9 @@ def remove_markings(obj, selectors, marking): """ if selectors is None: - object_markings.remove_markings(obj, marking) + return object_markings.remove_markings(obj, marking) else: - granular_markings.remove_markings(obj, selectors, marking) + return granular_markings.remove_markings(obj, selectors, marking) def add_markings(obj, selectors, marking): @@ -134,9 +134,9 @@ def clear_markings(obj, selectors): """ if selectors is None: - object_markings.clear_markings(obj) + return object_markings.clear_markings(obj) else: - granular_markings.clear_markings(obj, selectors) + return granular_markings.clear_markings(obj, selectors) def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index 9d2b2bc..c0d6d72 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -63,8 +63,11 @@ def remove_markings(obj, marking): raise AssertionError("Unable to remove Object Level Marking(s) from " "internal collection. Marking(s) not found...") - return obj.new_version(object_marking_refs=[x for x in object_markings - if x not in marking]) + new_markings = [x for x in object_markings if x not in marking] + if new_markings: + return obj.new_version(object_marking_refs=new_markings) + else: + return obj.new_version(object_marking_refs=None) def set_markings(obj, marking): diff --git a/stix2/properties.py b/stix2/properties.py index f63ec8b..69d2309 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -311,6 +311,7 @@ class ReferenceProperty(Property): def clean(self, value): if isinstance(value, _STIXBase): value = value.id + value = str(value) if self.type: if not value.startswith(self.type): raise ValueError("must start with '{0}'.".format(self.type)) diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 958120b..343488f 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -20,6 +20,15 @@ TOOL_ID = "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" SIGHTING_ID = "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb" VULNERABILITY_ID = "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" +MARKING_IDS = [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "marking-definition--443eb5c3-a76c-4a0a-8caa-e93998e7bc09", + "marking-definition--57fcd772-9c1d-41b0-8d1f-3d47713415d9", + "marking-definition--462bf1a6-03d2-419c-b74e-eee2238b2de4", + "marking-definition--68520ae2-fefe-43a9-84ee-2c2a934d2c7d", + "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f", +] + # Minimum required args for an Identity instance IDENTITY_KWARGS = dict( name="John Smith", diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index 15f35a4..f2d0374 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -1,10 +1,20 @@ import pytest -from stix2 import markings +from stix2 import Malware, exceptions, markings + +from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS +from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST """Tests for the Data Markings API.""" +MALWARE_KWARGS = MALWARE_KWARGS_CONST.copy() +MALWARE_KWARGS.update({ + 'id': MALWARE_ID, + 'created': FAKE_TIME, + 'modified': FAKE_TIME, +}) + def test_add_markings_one_marking(): before = { @@ -15,10 +25,10 @@ def test_add_markings_one_marking(): after = { "title": "test title", "description": "test description", - "object_marking_refs": ["marking-definition--1"] + "object_marking_refs": [MARKING_IDS[0]] } - markings.add_markings(before, None, "marking-definition--1") + markings.add_markings(before, None, MARKING_IDS[0]) assert before == after @@ -32,41 +42,38 @@ def test_add_markings_multiple_marking(): after = { "title": "test title", "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2"] + "object_marking_refs": [MARKING_IDS[0], MARKING_IDS[1]] } - markings.add_markings(before, None, ["marking-definition--1", "marking-definition--2"]) + markings.add_markings(before, None, [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["object_marking_refs"]: assert m in after["object_marking_refs"] def test_add_markings_combination(): - before = { - "title": "test title", - "description": "test description" - } - - after = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2"], - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]], + granular_markings=[ { - "selectors": ["title"], - "marking_ref": "marking-definition--3" + "selectors": ["labels"], + "marking_ref": MARKING_IDS[2] }, { - "selectors": ["description"], - "marking_ref": "marking-definition--4" + "selectors": ["name"], + "marking_ref": MARKING_IDS[3] }, - ] - } + ], + **MALWARE_KWARGS + ) - markings.add_markings(before, None, "marking-definition--1") - markings.add_markings(before, None, "marking-definition--2") - markings.add_markings(before, "title", "marking-definition--3") - markings.add_markings(before, "description", "marking-definition--4") + before = markings.add_markings(before, None, MARKING_IDS[0]) + before = markings.add_markings(before, None, MARKING_IDS[1]) + before = markings.add_markings(before, "labels", MARKING_IDS[2]) + before = markings.add_markings(before, "name", MARKING_IDS[3]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -79,15 +86,14 @@ def test_add_markings_combination(): ([""]), (""), ([]), - (["marking-definition--1", 456]) + ([MARKING_IDS[0], 456]) ]) def test_add_markings_bad_markings(data): - before = { - "title": "test title", - "description": "test description" - } - with pytest.raises(AssertionError): - markings.add_markings(before, None, data) + before = Malware( + **MALWARE_KWARGS + ) + with pytest.raises(exceptions.InvalidValueError): + before = markings.add_markings(before, None, data) assert "object_marking_refs" not in before @@ -240,65 +246,58 @@ def test_get_markings_object_and_granular_combinations(data): def test_remove_markings_object_level(): - after = { - "title": "test title", - "description": "test description" - } + before = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS + ) + after = Malware( + **MALWARE_KWARGS + ) - before = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--1"] - } + before = markings.remove_markings(before, None, MARKING_IDS[0]) - markings.remove_markings(before, None, "marking-definition--1") - - assert before == after + assert 'object_marking_refs' not in before + assert 'object_marking_refs' not in after def test_remove_markings_multiple(): - after = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--2"] - } + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[1]], + **MALWARE_KWARGS + ) - before = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] - } + before = markings.remove_markings(before, None, [MARKING_IDS[0], MARKING_IDS[2]]) - markings.remove_markings(before, None, ["marking-definition--1", "marking-definition--3"]) - - assert before == after + assert before['object_marking_refs'] == after['object_marking_refs'] def test_remove_markings_bad_markings(): before = { "title": "test title", "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] + "object_marking_refs": [MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]] } with pytest.raises(AssertionError): - markings.remove_markings(before, None, ["marking-definition--5"]) + markings.remove_markings(before, None, [MARKING_IDS[4]]) def test_clear_markings(): - after = { - "title": "test title", - "description": "test description" - } + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + after = Malware( + **MALWARE_KWARGS + ) - before = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] - } + before = markings.clear_markings(before, None) - markings.clear_markings(before, None) - - assert before == after + assert 'object_marking_refs' not in before + assert 'object_marking_refs' not in after def test_is_marked_object_and_granular_combinations(): @@ -442,23 +441,21 @@ def test_is_marked_object_and_granular_combinations(): def test_set_marking(): - before = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--1", "marking-definition--2", "marking-definition--3"] - } - after = { - "title": "test title", - "description": "test description", - "object_marking_refs": ["marking-definition--7", "marking-definition--9"] - } + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[4], MARKING_IDS[5]], + **MALWARE_KWARGS + ) - markings.set_markings(before, None, ["marking-definition--7", "marking-definition--9"]) + before = markings.set_markings(before, None, [MARKING_IDS[4], MARKING_IDS[5]]) for m in before["object_marking_refs"]: - assert m in ["marking-definition--7", "marking-definition--9"] + assert m in [MARKING_IDS[4], MARKING_IDS[5]] - assert ["marking-definition--1", "marking-definition--2", "marking-definition--3"] not in before["object_marking_refs"] + assert [MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]] not in before["object_marking_refs"] for x in before["object_marking_refs"]: assert x in after["object_marking_refs"] @@ -468,20 +465,18 @@ def test_set_marking(): ([]), ([""]), (""), - (["marking-definition--7", 687]) + ([MARKING_IDS[4], 687]) ]) def test_set_marking_bad_input(data): - before = { - "description": "test description", - "title": "foo", - "object_marking_refs": ["marking-definition--1"] - } - after = { - "description": "test description", - "title": "foo", - "object_marking_refs": ["marking-definition--1"] - } - with pytest.raises(AssertionError): - markings.set_markings(before, None, data) + before = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS + ) + after = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS + ) + with pytest.raises(exceptions.InvalidValueError): + before = markings.set_markings(before, None, data) assert before == after From 4cd99f04eaf3ccbc467b690d2511957c5651c29a Mon Sep 17 00:00:00 2001 From: Greg Back Date: Mon, 21 Aug 2017 22:16:10 +0000 Subject: [PATCH 52/82] Use version of taxii2-client from PyPI. --- .isort.cfg | 2 +- setup.py | 3 ++- stix2/test/test_data_sources.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index e141924..e8d95f8 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests,taxii2_client +known_third_party=dateutil,pytest,pytz,six,requests,taxii2client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/setup.py b/setup.py index c10da46..29d98f6 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ setup( 'pytz', 'requests', 'simplejson', - 'six' + 'six', + 'taxii2-client', ], ) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index bbb7f31..0095802 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,5 +1,5 @@ import pytest -from taxii2_client import Collection +from taxii2client import Collection from stix2.sources import DataSource, Filter, taxii From d060abbed59434dee6ba7ba18ac9c25f0a8d1e6d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 22 Aug 2017 10:47:13 -0400 Subject: [PATCH 53/82] Updated regex expressions. Thanks to @drothenberg for that contribution! --- stix2/test/test_observed_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index 45610f9..dd8f042 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -173,7 +173,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +194,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -215,7 +215,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +358,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('"objects".+\}', re.DOTALL).sub('"objects": { %s }\n}' % data, EXPECTED) + odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" From 15bff530beadd05b517d4778a832fb3b31d2452c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 23 Aug 2017 13:06:51 -0400 Subject: [PATCH 54/82] Refactor granular_markings.py code and remove unnecessary code in utils.py --- stix2/markings/granular_markings.py | 76 ++++++++++++++++------------ stix2/markings/utils.py | 77 +++++++++-------------------- 2 files changed, 69 insertions(+), 84 deletions(-) diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 903aae9..5c223a3 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -53,8 +53,8 @@ def set_markings(obj, selectors, marking): field(s) selected by `selectors`. """ - clear_markings(obj, selectors) - add_markings(obj, selectors, marking) + obj = clear_markings(obj, selectors) + return add_markings(obj, selectors, marking) def remove_markings(obj, selectors, marking): @@ -76,16 +76,22 @@ def remove_markings(obj, selectors, marking): selectors = utils.fix_value(selectors) utils.validate(obj, selectors, marking) - utils.expand_markings(obj) - granular_markings = obj.get("granular_markings") if not granular_markings: - return + return obj - tlo = utils.build_granular_marking( - {"selectors": selectors, "marking_ref": marking} - ) + granular_markings = utils.expand_markings(granular_markings) + + if isinstance(marking, list): + to_remove = [] + for m in marking: + to_remove.append({"marking_ref": m, "selectors": selectors}) + else: + to_remove = [{"marking_ref": marking, "selectors": selectors}] + + to_remove = utils.expand_markings(to_remove) + tlo = utils.build_granular_marking(to_remove) remove = tlo.get("granular_markings", []) @@ -93,14 +99,16 @@ def remove_markings(obj, selectors, marking): raise AssertionError("Unable to remove Granular Marking(s) from" " internal collection. Marking(s) not found...") - obj["granular_markings"] = [ + granular_markings = [ m for m in granular_markings if m not in remove ] - utils.compress_markings(obj) + granular_markings = utils.compress_markings(granular_markings) - if not obj.get("granular_markings"): - obj.pop("granular_markings") + if not granular_markings: + return obj.new_version(granular_markings=None) + else: + return obj.new_version(granular_markings=granular_markings) def add_markings(obj, selectors, marking): @@ -109,10 +117,10 @@ def add_markings(obj, selectors, marking): Args: obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). - marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + selectors: list of type string, selectors must be relative to the TLO + in which the properties appear. + marking: identifier that apply to the properties selected by + `selectors`. Raises: AssertionError: If `selectors` or `marking` fail data validation. @@ -121,15 +129,19 @@ def add_markings(obj, selectors, marking): selectors = utils.fix_value(selectors) utils.validate(obj, selectors, marking) - granular_marking = {"selectors": sorted(selectors), "marking_ref": marking} + if isinstance(marking, list): + granular_marking = [] + for m in marking: + granular_marking.append({"marking_ref": m, "selectors": sorted(selectors)}) + else: + granular_marking = [{"marking_ref": marking, "selectors": sorted(selectors)}] - if not obj.get("granular_markings"): - obj["granular_markings"] = list() + if obj.get("granular_markings"): + granular_marking.extend(obj.get("granular_markings")) - obj["granular_markings"].append(granular_marking) - - utils.expand_markings(obj) - utils.compress_markings(obj) + granular_marking = utils.expand_markings(granular_marking) + granular_marking = utils.compress_markings(granular_marking) + return obj.new_version(granular_markings=granular_marking) def clear_markings(obj, selectors): @@ -149,15 +161,15 @@ def clear_markings(obj, selectors): selectors = utils.fix_value(selectors) utils.validate(obj, selectors) - utils.expand_markings(obj) - granular_markings = obj.get("granular_markings") if not granular_markings: - return + return obj + + granular_markings = utils.expand_markings(granular_markings) tlo = utils.build_granular_marking( - {"selectors": selectors, "marking_ref": ["N/A"]} + [{"selectors": selectors, "marking_ref": "N/A"}] ) clear = tlo.get("granular_markings", []) @@ -176,12 +188,14 @@ def clear_markings(obj, selectors): marking_refs = granular_marking.get("marking_ref") if marking_refs: - granular_marking["marking_ref"] = list() + granular_marking["marking_ref"] = "" - utils.compress_markings(obj) + granular_markings = utils.compress_markings(granular_markings) - if not obj.get("granular_markings"): - obj.pop("granular_markings") + if not granular_markings: + return obj.new_version(granular_markings=None) + else: + return obj.new_version(granular_markings=granular_markings) def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 9286695..adf3069 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -67,81 +67,51 @@ def fix_value(data): return data -def _fix_markings(markings): +def compress_markings(granular_markings): - for granular_marking in markings: - refs = granular_marking.get("marking_ref", []) - selectors = granular_marking.get("selectors", []) - - if not isinstance(refs, list): - granular_marking["marking_ref"] = [refs] - - if not isinstance(selectors, list): - granular_marking["selectors"] = [selectors] - - -def _group_by(markings): - - key = "marking_ref" - retrieve = "selectors" + if not granular_markings: + return map_ = collections.defaultdict(set) - for granular_marking in markings: - for data in granular_marking.get(key, []): - map_[data].update(granular_marking.get(retrieve)) + for granular_marking in granular_markings: + if granular_marking.get("marking_ref"): + map_[granular_marking.get("marking_ref")].update(granular_marking.get("selectors")) - granular_markings = \ + compressed = \ [ - {"selectors": sorted(selectors), "marking_ref": ref} - for ref, selectors in six.iteritems(map_) + {"marking_ref": marking_ref, "selectors": sorted(selectors)} + for marking_ref, selectors in six.iteritems(map_) ] - return granular_markings + return compressed -def compress_markings(tlo): +def expand_markings(granular_markings): - if not tlo.get("granular_markings"): + if not granular_markings: return - granular_markings = tlo.get("granular_markings") - - _fix_markings(granular_markings) - - tlo["granular_markings"] = _group_by(granular_markings) - - -def expand_markings(tlo): - - if not tlo.get("granular_markings"): - return - - granular_markings = tlo.get("granular_markings") - - _fix_markings(granular_markings) - - expanded = list() + expanded = [] for marking in granular_markings: - selectors = marking.get("selectors", []) - marking_ref = marking.get("marking_ref", []) + selectors = marking.get("selectors") + marking_ref = marking.get("marking_ref") expanded.extend( [ - {"selectors": [sel], "marking_ref": ref} - for sel in selectors - for ref in marking_ref + {"marking_ref": marking_ref, "selectors": [selector]} + for selector in selectors ] ) - tlo["granular_markings"] = expanded + return expanded def build_granular_marking(granular_marking): - tlo = {"granular_markings": [granular_marking]} + tlo = {"granular_markings": granular_marking} - expand_markings(tlo) + expand_markings(tlo["granular_markings"]) return tlo @@ -156,14 +126,15 @@ def iterpath(obj, path=None): path: None, used recursively to store ancestors. Example: - >>> for item in iterpath(tlo): + >>> for item in iterpath(obj): >>> print(item) (['type'], 'campaign') ... (['cybox', 'objects', '[0]', 'hashes', 'sha1'], 'cac35ec206d868b7d7cb0b55f31d9425b075082b') Returns: - tuple: Containing two items: a list of ancestors and the property value. + tuple: Containing two items: a list of ancestors and the + property value. """ if path is None: @@ -209,7 +180,7 @@ def get_selector(obj, prop): location is for now the option to assert the data. Example: - >>> selector = get_selector(tlo, tlo["cybox"]["objects"][0]["file_name"]) + >>> selector = get_selector(obj, obj["cybox"]["objects"][0]["file_name"]) >>> print(selector) ["cybox.objects.[0].file_name"] From f33427328bc86782eda7457f5a3c170a36c13536 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 23 Aug 2017 13:07:22 -0400 Subject: [PATCH 55/82] Update tests for object and granular markings. --- stix2/test/constants.py | 1 + stix2/test/test_bundle.py | 1 + stix2/test/test_granular_markings.py | 580 +++++++++++++-------------- stix2/test/test_object_markings.py | 39 +- 4 files changed, 295 insertions(+), 326 deletions(-) diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 343488f..d62d932 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -45,6 +45,7 @@ INDICATOR_KWARGS = dict( MALWARE_KWARGS = dict( labels=['ransomware'], name="Cryptolocker", + description="A ransomware related to ..." ) # Minimum required args for a Relationship instance diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 0733637..b73345a 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -19,6 +19,7 @@ EXPECTED_BUNDLE = """{ }, { "created": "2017-01-01T12:34:56.000Z", + "description": "A ransomware related to ...", "id": "malware--00000000-0000-0000-0000-000000000002", "labels": [ "ransomware" diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index f38c0dd..50fc27d 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -1,128 +1,134 @@ import pytest -from stix2 import markings +from stix2 import Malware, exceptions, markings + +from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS +from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST """Tests for the Data Markings API.""" +MALWARE_KWARGS = MALWARE_KWARGS_CONST.copy() +MALWARE_KWARGS.update({ + 'id': MALWARE_ID, + 'created': FAKE_TIME, + 'modified': FAKE_TIME, +}) + def test_add_marking_mark_one_selector_multiple_refs(): - before = { - "description": "test description", - "title": "foo", - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, { "selectors": ["description"], - "marking_ref": "marking-definition--2" + "marking_ref": MARKING_IDS[1] }, - ] - } - markings.add_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["granular_markings"]: assert m in after["granular_markings"] def test_add_marking_mark_multiple_selector_one_refs(): - before = { - "description": "test description", - "title": "foo", - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] }, - ] - } - markings.add_markings(before, ["description", "title"], ["marking-definition--1"]) - assert before == after + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, ["description", "name"], [MARKING_IDS[0]]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] def test_add_marking_mark_multiple_selector_multiple_refs(): - before = { - "description": "test description", - "title": "foo", - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.add_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, ["description", "name"], [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["granular_markings"]: assert m in after["granular_markings"] def test_add_marking_mark_another_property_same_marking(): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] }, - ] - } - markings.add_markings(before, ["title"], ["marking-definition--1"]) - assert before == after + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, ["name"], [MARKING_IDS[0]]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] def test_add_marking_mark_same_property_same_marking(): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" - } - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" - } - ] - } - markings.add_markings(before, ["description"], ["marking-definition--1"]) - assert before == after + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, ["description"], [MARKING_IDS[0]]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] @pytest.mark.parametrize("data,marking", [ @@ -329,176 +335,151 @@ def test_get_markings_positional_arguments_combinations(data): def test_remove_marking_remove_one_selector_with_multiple_refs(): - after = { - "description": "test description", - "title": "foo", - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, { "selectors": ["description"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.remove_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) - assert before == after + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) + assert "granular_markings" not in before def test_remove_marking_remove_multiple_selector_one_ref(): - after = { - "description": "test description", - "title": "foo", - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" - }, - ] - } - markings.remove_markings(before, ["description", "title"], ["marking-definition--1"]) - assert before == after + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["description", "modified"], [MARKING_IDS[0]]) + assert "granular_markings" not in before def test_remove_marking_mark_one_selector_from_multiple_ones(): - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" - }, - ] - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" - }, - ] - } - markings.remove_markings(before, ["title"], ["marking-definition--1"]) - assert before == after + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] def test_remove_marking_mark_one_selector_markings_from_multiple_ones(): - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--2" - }, - ] - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.remove_markings(before, ["title"], ["marking-definition--1"]) - + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) for m in before["granular_markings"]: assert m in after["granular_markings"] def test_remove_marking_mark_mutilple_selector_multiple_refs(): - after = { - "description": "test description", - "title": "foo", - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.remove_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) - assert before == after + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["description", "modified"], [MARKING_IDS[0], MARKING_IDS[1]]) + assert "granular_markings" not in before def test_remove_marking_mark_another_property_same_marking(): - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" - }, - ] - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ - { - "selectors": ["description"], - "marking_ref": "marking-definition--1" - }, - { - "selectors": ["title"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - markings.remove_markings(before, ["title"], ["marking-definition--1"]) - assert before == after + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] def test_remove_marking_mark_same_property_same_marking(): - after = { - "description": "test description", - "title": "foo", - } - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - markings.remove_markings(before, ["description"], ["marking-definition--1"]) - assert before == after + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, ["description"], [MARKING_IDS[0]]) + assert "granular_markings" not in before def test_remove_marking_bad_selector(): @@ -741,104 +722,97 @@ def test_is_marked_positional_arguments_combinations(): def test_set_marking_mark_one_selector_multiple_refs(): - before = { - "description": "test description", - "title": "foo", - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, { "selectors": ["description"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.set_markings(before, ["description"], ["marking-definition--1", "marking-definition--2"]) + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["granular_markings"]: assert m in after["granular_markings"] def test_set_marking_mark_multiple_selector_one_refs(): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--3" - }, - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" - }, - ] - } - markings.set_markings(before, ["description", "title"], ["marking-definition--1"]) - assert before == after + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, ["description", "modified"], [MARKING_IDS[0]]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] def test_set_marking_mark_multiple_selector_multiple_refs_from_none(): - before = { - "description": "test description", - "title": "foo", - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--1" + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["description", "title"], - "marking_ref": "marking-definition--2" - }, - ] - } - markings.set_markings(before, ["description", "title"], ["marking-definition--1", "marking-definition--2"]) + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, ["description", "modified"], [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["granular_markings"]: assert m in after["granular_markings"] def test_set_marking_mark_another_property_same_marking(): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--7" + "marking_ref": MARKING_IDS[1] }, { "selectors": ["description"], - "marking_ref": "marking-definition--8" - }, - ] - } - markings.set_markings(before, ["description"], ["marking-definition--7", "marking-definition--8"]) + "marking_ref": MARKING_IDS[2] + } + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, ["description"], [MARKING_IDS[1], MARKING_IDS[2]]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -879,76 +853,72 @@ def test_set_marking_bad_selector(marking): def test_set_marking_mark_same_property_same_marking(): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - markings.set_markings(before, ["description"], ["marking-definition--1"]) - assert before == after + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, ["description"], [MARKING_IDS[0]]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] -CLEAR_MARKINGS_TEST_DATA = { - "title": "test title", - "description": "test description", - "revision": 2, - "type": "test", - "granular_markings": [ +CLEAR_MARKINGS_TEST_DATA = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] }, { - "selectors": ["revision", "description"], - "marking_ref": "marking-definition--2" + "selectors": ["modified", "description"], + "marking_ref": MARKING_IDS[1] }, { - "selectors": ["revision", "description", "type"], - "marking_ref": "marking-definition--3" + "selectors": ["modified", "description", "type"], + "marking_ref": MARKING_IDS[2] }, - ] -} + ], + **MALWARE_KWARGS +) @pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) def test_clear_marking_smoke(data): """Test clear_marking call does not fail.""" - markings.clear_markings(data, "revision") - assert markings.is_marked(data, "revision") is False + data = markings.clear_markings(data, "modified") + assert markings.is_marked(data, "modified") is False @pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) def test_clear_marking_multiple_selectors(data): """Test clearing markings for multiple selectors effectively removes associated markings.""" - markings.clear_markings(data, ["type", "description"]) + data = markings.clear_markings(data, ["type", "description"]) assert markings.is_marked(data, ["type", "description"]) is False @pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) def test_clear_marking_one_selector(data): """Test markings associated with one selector were removed.""" - markings.clear_markings(data, "description") + data = markings.clear_markings(data, "description") assert markings.is_marked(data, "description") is False @pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) def test_clear_marking_all_selectors(data): - markings.clear_markings(data, ["description", "type", "revision"]) + data = markings.clear_markings(data, ["description", "type", "modified"]) assert markings.is_marked(data, "description") is False assert "granular_markings" not in data diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index f2d0374..04c1b97 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -17,35 +17,32 @@ MALWARE_KWARGS.update({ def test_add_markings_one_marking(): - before = { - "title": "test title", - "description": "test description" - } + before = Malware( + **MALWARE_KWARGS + ) - after = { - "title": "test title", - "description": "test description", - "object_marking_refs": [MARKING_IDS[0]] - } + after = Malware( + object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS, + ) - markings.add_markings(before, None, MARKING_IDS[0]) + before = markings.add_markings(before, None, MARKING_IDS[0]) - assert before == after + for m in before["object_marking_refs"]: + assert m in after["object_marking_refs"] def test_add_markings_multiple_marking(): - before = { - "title": "test title", - "description": "test description" - } + before = Malware( + **MALWARE_KWARGS + ) - after = { - "title": "test title", - "description": "test description", - "object_marking_refs": [MARKING_IDS[0], MARKING_IDS[1]] - } + after = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]], + **MALWARE_KWARGS, + ) - markings.add_markings(before, None, [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.add_markings(before, None, [MARKING_IDS[0], MARKING_IDS[1]]) for m in before["object_marking_refs"]: assert m in after["object_marking_refs"] From 8d4c1d55b522b63fdaf0169fc54b843b4b776d89 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 23 Aug 2017 13:12:40 -0400 Subject: [PATCH 56/82] Minor style changes. --- stix2/test/test_granular_markings.py | 2 +- stix2/test/test_markings.py | 4 ++-- stix2/test/test_object_markings.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index 50fc27d..2f0524b 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -1,7 +1,7 @@ import pytest -from stix2 import Malware, exceptions, markings +from stix2 import Malware, markings from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 726cd98..63aecc3 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -155,8 +155,8 @@ def test_campaign_with_granular_markings_example(): marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", selectors=["description"]) ]) - print (markings.get_markings(campaign, None)) - print (markings.add_markings(campaign, None, "marking-definition--00000000-0000-0000-0000-000000000000")) + print(markings.get_markings(campaign, None)) + print(markings.add_markings(campaign, None, "marking-definition--00000000-0000-0000-0000-000000000000")) assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index 04c1b97..25792a7 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -23,7 +23,7 @@ def test_add_markings_one_marking(): after = Malware( object_marking_refs=[MARKING_IDS[0]], - **MALWARE_KWARGS, + **MALWARE_KWARGS ) before = markings.add_markings(before, None, MARKING_IDS[0]) @@ -39,7 +39,7 @@ def test_add_markings_multiple_marking(): after = Malware( object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]], - **MALWARE_KWARGS, + **MALWARE_KWARGS ) before = markings.add_markings(before, None, [MARKING_IDS[0], MARKING_IDS[1]]) @@ -62,7 +62,7 @@ def test_add_markings_combination(): { "selectors": ["name"], "marking_ref": MARKING_IDS[3] - }, + } ], **MALWARE_KWARGS ) From 94ccd422c9ae98f5210ceaf063e426ee80f53e34 Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 23 Aug 2017 13:50:31 -0400 Subject: [PATCH 57/82] Make isort recognize stix2patterns --- .isort.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.isort.cfg b/.isort.cfg index 63f5b73..badf815 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,6 +2,6 @@ check=1 diff=1 known_third_party=dateutil,pytest,pytz,six,requests -known_first_party=stix2 +known_first_party=stix2,stix2patterns not_skip=__init__.py force_sort_within_sections=1 From ee49e78c72163e40c8cdae0b43914434e47d24ca Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 23 Aug 2017 18:36:24 -0400 Subject: [PATCH 58/82] Add custom extensions to cyber observables Fix #31. --- stix2/observables.py | 124 ++++++++++++++++++++++++++++---------- stix2/test/test_custom.py | 57 ++++++++++++++++++ 2 files changed, 150 insertions(+), 31 deletions(-) diff --git a/stix2/observables.py b/stix2/observables.py index 366e007..3c033bd 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -24,7 +24,7 @@ class ObservableProperty(Property): except ValueError: raise ValueError("The observable property must contain a dictionary") if dictified == {}: - raise ValueError("The dictionary property must contain a non-empty dictionary") + raise ValueError("The observable property must contain a non-empty dictionary") valid_refs = dict((k, v['type']) for (k, v) in dictified.items()) @@ -53,7 +53,7 @@ class ExtensionsProperty(DictionaryProperty): except ValueError: raise ValueError("The extensions property must contain a dictionary") if dictified == {}: - raise ValueError("The dictionary property must contain a non-empty dictionary") + raise ValueError("The extensions property must contain a non-empty dictionary") if self.enclosing_type in EXT_MAP: specific_type_map = EXT_MAP[self.enclosing_type] @@ -69,7 +69,7 @@ class ExtensionsProperty(DictionaryProperty): else: raise ValueError("The key used in the extensions dictionary is not an extension type name") else: - raise ValueError("The enclosing type has no extensions defined") + raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type) return dictified @@ -77,6 +77,7 @@ class Artifact(_Observable): _type = 'artifact' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'mime_type': StringProperty(), 'payload_bin': BinaryProperty(), 'url': StringProperty(), @@ -93,6 +94,7 @@ class AutonomousSystem(_Observable): _type = 'autonomous-system' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'number': IntegerProperty(), 'name': StringProperty(), 'rir': StringProperty(), @@ -103,6 +105,7 @@ class Directory(_Observable): _type = 'directory' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'path': StringProperty(required=True), 'path_enc': StringProperty(), # these are not the created/modified timestamps of the object itself @@ -117,6 +120,7 @@ class DomainName(_Observable): _type = 'domain-name' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name'])), } @@ -126,6 +130,7 @@ class EmailAddress(_Observable): _type = 'email-addr' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), 'display_name': StringProperty(), 'belongs_to_ref': ObjectReferenceProperty(valid_types='user-account'), @@ -149,6 +154,7 @@ class EmailMessage(_Observable): _type = 'email-message' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'is_multipart': BooleanProperty(required=True), 'date': TimestampProperty(), 'content_type': StringProperty(), @@ -174,6 +180,7 @@ class EmailMessage(_Observable): class ArchiveExt(_Extension): + _type = 'archive-ext' _properties = { 'contains_refs': ListProperty(ObjectReferenceProperty(valid_types='file'), required=True), 'version': StringProperty(), @@ -190,6 +197,7 @@ class AlternateDataStream(_STIXBase): class NTFSExt(_Extension): + _type = 'ntfs-ext' _properties = { 'sid': StringProperty(), 'alternate_data_streams': ListProperty(EmbeddedObjectProperty(type=AlternateDataStream)), @@ -197,6 +205,7 @@ class NTFSExt(_Extension): class PDFExt(_Extension): + _type = 'pdf-ext' _properties = { 'version': StringProperty(), 'is_optimized': BooleanProperty(), @@ -207,6 +216,7 @@ class PDFExt(_Extension): class RasterImageExt(_Extension): + _type = 'raster-image-ext' _properties = { 'image_height': IntegerProperty(), 'image_weight': IntegerProperty(), @@ -266,6 +276,7 @@ class WindowsPESection(_STIXBase): class WindowsPEBinaryExt(_Extension): + _type = 'windows-pebinary-ext' _properties = { 'pe_type': StringProperty(required=True), # open_vocab 'imphash': StringProperty(), @@ -315,6 +326,7 @@ class IPv4Address(_Observable): _type = 'ipv4-addr' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), @@ -325,6 +337,7 @@ class IPv6Address(_Observable): _type = 'ipv6-addr' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), 'resolves_to_refs': ListProperty(ObjectReferenceProperty(valid_types='mac-addr')), 'belongs_to_refs': ListProperty(ObjectReferenceProperty(valid_types='autonomous-system')), @@ -335,6 +348,7 @@ class MACAddress(_Observable): _type = 'mac-addr' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), } @@ -343,11 +357,13 @@ class Mutex(_Observable): _type = 'mutex' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'name': StringProperty(), } class HTTPRequestExt(_Extension): + _type = 'http-request-ext' _properties = { 'request_method': StringProperty(required=True), 'request_value': StringProperty(required=True), @@ -359,6 +375,7 @@ class HTTPRequestExt(_Extension): class ICMPExt(_Extension): + _type = 'icmp-ext' _properties = { 'icmp_type_hex': HexProperty(required=True), 'icmp_code_hex': HexProperty(required=True), @@ -366,6 +383,7 @@ class ICMPExt(_Extension): class SocketExt(_Extension): + _type = 'socket-ext' _properties = { 'address_family': EnumProperty([ "AF_UNSPEC", @@ -399,6 +417,7 @@ class SocketExt(_Extension): class TCPExt(_Extension): + _type = 'tcp-ext' _properties = { 'src_flags_hex': HexProperty(), 'dst_flags_hex': HexProperty(), @@ -435,6 +454,7 @@ class NetworkTraffic(_Observable): class WindowsProcessExt(_Extension): + _type = 'windows-process-ext' _properties = { 'aslr_enabled': BooleanProperty(), 'dep_enabled': BooleanProperty(), @@ -446,6 +466,7 @@ class WindowsProcessExt(_Extension): class WindowsServiceExt(_Extension): + _type = 'windows-service-ext' _properties = { 'service_name': StringProperty(required=True), 'descriptions': ListProperty(StringProperty), @@ -517,6 +538,7 @@ class Software(_Observable): _type = 'software' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'name': StringProperty(required=True), 'cpe': StringProperty(), 'languages': ListProperty(StringProperty), @@ -529,11 +551,13 @@ class URL(_Observable): _type = 'url' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'value': StringProperty(required=True), } class UNIXAccountExt(_Extension): + _type = 'unix-account-ext' _properties = { 'gid': IntegerProperty(), 'groups': ListProperty(StringProperty), @@ -590,6 +614,7 @@ class WindowsRegistryKey(_Observable): _type = 'windows-registry-key' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'key': StringProperty(required=True), 'values': ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType)), # this is not the modified timestamps of the object itself @@ -630,6 +655,7 @@ class X509Certificate(_Observable): _type = 'x509-certificate' _properties = { 'type': TypeProperty(_type), + 'extensions': ExtensionsProperty(enclosing_type=_type), 'is_self_signed': BooleanProperty(), 'hashes': HashesProperty(), 'version': StringProperty(), @@ -667,36 +693,28 @@ OBJ_MAP_OBSERVABLE = { 'x509-certificate': X509Certificate, } -EXT_MAP_FILE = { - 'archive-ext': ArchiveExt, - 'ntfs-ext': NTFSExt, - 'pdf-ext': PDFExt, - 'raster-image-ext': RasterImageExt, - 'windows-pebinary-ext': WindowsPEBinaryExt -} - -EXT_MAP_NETWORK_TRAFFIC = { - 'http-request-ext': HTTPRequestExt, - 'icmp-ext': ICMPExt, - 'socket-ext': SocketExt, - 'tcp-ext': TCPExt, -} - -EXT_MAP_PROCESS = { - 'windows-process-ext': WindowsProcessExt, - 'windows-service-ext': WindowsServiceExt, -} - -EXT_MAP_USER_ACCOUNT = { - 'unix-account-ext': UNIXAccountExt, -} EXT_MAP = { - 'file': EXT_MAP_FILE, - 'network-traffic': EXT_MAP_NETWORK_TRAFFIC, - 'process': EXT_MAP_PROCESS, - 'user-account': EXT_MAP_USER_ACCOUNT, - + 'file': { + 'archive-ext': ArchiveExt, + 'ntfs-ext': NTFSExt, + 'pdf-ext': PDFExt, + 'raster-image-ext': RasterImageExt, + 'windows-pebinary-ext': WindowsPEBinaryExt + }, + 'network-traffic': { + 'http-request-ext': HTTPRequestExt, + 'icmp-ext': ICMPExt, + 'socket-ext': SocketExt, + 'tcp-ext': TCPExt, + }, + 'process': { + 'windows-process-ext': WindowsProcessExt, + 'windows-service-ext': WindowsServiceExt, + }, + 'user-account': { + 'unix-account-ext': UNIXAccountExt, + }, } @@ -760,3 +778,47 @@ def CustomObservable(type='x-custom-observable', properties={}): return _Custom return custom_builder + + +def _register_extension(observable, new_extension): + """Register a custom extension to a STIX Cyber Observable type. + """ + + try: + observable_type = observable._type + except AttributeError: + raise ValueError("Custom observables must be created with the @CustomObservable decorator.") + + try: + EXT_MAP[observable_type][new_extension._type] = new_extension + except KeyError: + if observable_type not in OBJ_MAP_OBSERVABLE: + raise ValueError("Unknown observable type '%s'" % observable_type) + else: + EXT_MAP[observable_type] = {new_extension._type: new_extension} + + +def CustomExtension(observable=None, type='x-custom-observable', properties={}): + """Decorator for custom extensions to STIX Cyber Observables + """ + + if not observable or not issubclass(observable, _Observable): + raise ValueError("'observable' must be a valid Observable class!") + + def custom_builder(cls): + + class _Custom(cls, _Extension): + _type = type + _properties = { + 'extensions': ExtensionsProperty(enclosing_type=_type), + } + _properties.update(properties) + + def __init__(self, **kwargs): + _Extension.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_extension(observable, _Custom) + return _Custom + + return custom_builder diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 60e982c..804776a 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -180,3 +180,60 @@ def test_observed_data_with_custom_observable_object(): allow_custom=True, ) assert ob_data.objects['0'].property1 == 'something' + + +@stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + 'property2': stix2.properties.IntegerProperty(), +}) +class NewExtension(): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + +def test_custom_extension(): + ext = NewExtension(property1='something') + assert ext.property1 == 'something' + + with pytest.raises(stix2.exceptions.MissingPropertiesError): + NewExtension(property2=42) + + with pytest.raises(ValueError): + NewExtension(property1='something', property2=4) + + +def test_custom_extension_invalid(): + class Foo(object): + pass + with pytest.raises(ValueError): + @stix2.observables.CustomExtension(Foo, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class FooExtension(): + pass # pragma: no cover + + class Bar(stix2.observables._Observable): + pass + with pytest.raises(ValueError): + @stix2.observables.CustomExtension(Bar, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class BarExtension(): + pass + + +def test_parse_observable_with_custom_extension(): + input_str = """{ + "type": "domain-name", + "value": "example.com", + "extensions": { + "x-new-ext": { + "property1": "foo", + "property2": 12 + } + } + }""" + + parsed = stix2.parse_observable(input_str) + assert parsed.extensions['x-new-ext'].property2 == 12 From dd17e88ae07734332a9cf31257b9dfa4c477f4cd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 24 Aug 2017 12:47:14 -0400 Subject: [PATCH 59/82] Code refactor/clean-up, changed some exceptions. Update docstrings. --- stix2/base.py | 2 +- stix2/exceptions.py | 29 ++--- stix2/markings/__init__.py | 85 ++++++++----- stix2/markings/granular_markings.py | 130 +++++++++++--------- stix2/markings/object_markings.py | 26 ++-- stix2/markings/utils.py | 183 ++++++++++++++++------------ stix2/test/test_markings.py | 2 +- 7 files changed, 259 insertions(+), 198 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index d59395c..acac94c 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -81,7 +81,7 @@ class _STIXBase(collections.Mapping): def _check_object_constraints(self): for m in self.get("granular_markings", []): - validate(self, m.get("selectors"), m.get("marking_ref")) + validate(self, m.get("selectors")) def __init__(self, allow_custom=False, **kwargs): cls = self.__class__ diff --git a/stix2/exceptions.py b/stix2/exceptions.py index 864dec8..63ac921 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -159,7 +159,7 @@ class ParseError(STIXError, ValueError): super(ParseError, self).__init__(msg) -class InvalidSelectorError(STIXError, ValueError): +class InvalidSelectorError(STIXError, AssertionError): """Granular Marking selector violation. The selector must resolve into an existing STIX object property.""" def __init__(self, cls, key): @@ -168,31 +168,18 @@ class InvalidSelectorError(STIXError, ValueError): self.key = key def __str__(self): - msg = "Selector '{0}' in '{1}' is not valid!" - return msg.format(self.key, self.__class__.__name__) + msg = "Selector {0} in {1} is not valid!" + return msg.format(self.key, self.cls.__class__.__name__) -class InvalidMarkingError(STIXError, ValueError): - """Marking violation. The marking reference must be a valid identifier.""" +class MarkingNotFoundError(STIXError, AssertionError): + """Marking violation. The marking reference must be present in SDO or SRO.""" def __init__(self, cls, key): - super(InvalidMarkingError, self).__init__() + super(MarkingNotFoundError, self).__init__() self.cls = cls self.key = key def __str__(self): - msg = "Marking '{0}' in '{1}' is not a valid marking reference." - return msg.format(self.key, self.__class__.__name__) - - -class DuplicateMarkingError(STIXError, ValueError): - """Marking violation. The marking reference is a duplicate.""" - - def __init__(self, cls, key): - super(DuplicateMarkingError, self).__init__() - self.cls = cls - self.key = key - - def __str__(self): - msg = "Marking '{0}' in '{1}' is a duplicate marking reference." - return msg.format(self.key, self.__class__.__name__) + msg = "Marking {0} was not found in {1}!" + return msg.format(self.key, self.cls.__class__.__name__) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index f158723..4ef3e3a 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -13,16 +13,16 @@ def get_markings(obj, selectors, inherited=False, descendants=False): Get all markings associated to the field(s). Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. inherited: If True, include object level markings and granular markings - inherited relative to the field(s). + inherited relative to the properties. descendants: If True, include granular markings applied to any children - relative to the field(s). + relative to the properties. Returns: - list: Marking IDs that matched the selectors expression. + list: Marking identifiers that matched the selectors expression. Note: If ``selectors`` is None, operation will be performed only on object @@ -51,11 +51,15 @@ def set_markings(obj, selectors, marking): marking. Refer to `clear_markings` and `add_markings` for details. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. + + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. Note: If ``selectors`` is None, operations will be performed on object level @@ -73,15 +77,19 @@ def remove_markings(obj, selectors, marking): Removes granular_marking from the granular_markings collection. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. Raises: - AssertionError: If `selectors` or `marking` fail data validation. Also - if markings to remove are not found on the provided TLO. + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. Note: If ``selectors`` is None, operations will be performed on object level @@ -99,14 +107,17 @@ def add_markings(obj, selectors, marking): Appends a granular_marking to the granular_markings collection. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. Raises: - AssertionError: If `selectors` or `marking` fail data validation. + InvalidSelectorError: If `selectors` fail validation. + + Returns: + A new version of the given SDO or SRO with specified markings added. Note: If ``selectors`` is None, operations will be performed on object level @@ -124,9 +135,17 @@ def clear_markings(obj, selectors): Removes all granular_marking associated with the selectors. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). + + Raises: + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings cleared. Note: If ``selectors`` is None, operations will be performed on object level @@ -144,23 +163,23 @@ def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): Checks if field(s) is marked by any marking or by specific marking(s). Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. inherited: If True, include object level markings and granular markings - inherited to determine if the field(s) is/are marked. + inherited to determine if the properties is/are marked. descendants: If True, include granular markings applied to any children - of the given selector to determine if the field(s) is/are marked. + of the given selector to determine if the properties is/are marked. Returns: - bool: True if ``selectors`` is found on internal TLO collection. + bool: True if ``selectors`` is found on internal SDO or SRO collection. False otherwise. Note: - When a list of marking IDs is provided, if ANY of the provided marking - IDs matches, True is returned. + When a list of marking identifiers is provided, if ANY of the provided + marking identifiers match, True is returned. If ``selectors`` is None, operation will be performed only on object level markings. diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 5c223a3..6f2933f 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -1,24 +1,29 @@ +from stix2 import exceptions from stix2.markings import utils def get_markings(obj, selectors, inherited=False, descendants=False): """ - Get all markings associated to the field(s). + Get all markings associated to with the properties. Args: - obj: A TLO object. - selectors: string or list of selector strings relative to the TLO in - which the field(s) appear(s). - inherited: If True, include markings inherited relative to the field(s). + obj: An SDO or SRO object. + selectors: string or list of selector strings relative to the SDO or + SRO in which the properties appear. + inherited: If True, include markings inherited relative to the + properties. descendants: If True, include granular markings applied to any children - relative to the field(s). + relative to the properties. + + Raises: + InvalidSelectorError: If `selectors` fail validation. Returns: - list: Marking IDs that matched the selectors expression. + list: Marking identifiers that matched the selectors expression. """ - selectors = utils.fix_value(selectors) + selectors = utils.convert_to_list(selectors) utils.validate(obj, selectors) granular_markings = obj.get("granular_markings", []) @@ -46,11 +51,15 @@ def set_markings(obj, selectors, marking): marking. Refer to `clear_markings` and `add_markings` for details. Args: - obj: A TLO object. - selectors: string or list of selector strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selector strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. + + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. """ obj = clear_markings(obj, selectors) @@ -62,19 +71,23 @@ def remove_markings(obj, selectors, marking): Removes granular_marking from the granular_markings collection. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. Raises: - AssertionError: If `selectors` or `marking` fail data validation. Also - if markings to remove are not found on the provided TLO. + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. """ - selectors = utils.fix_value(selectors) - utils.validate(obj, selectors, marking) + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) granular_markings = obj.get("granular_markings") @@ -90,14 +103,10 @@ def remove_markings(obj, selectors, marking): else: to_remove = [{"marking_ref": marking, "selectors": selectors}] - to_remove = utils.expand_markings(to_remove) - tlo = utils.build_granular_marking(to_remove) - - remove = tlo.get("granular_markings", []) + remove = utils.build_granular_marking(to_remove).get("granular_markings") if not any(marking in granular_markings for marking in remove): - raise AssertionError("Unable to remove Granular Marking(s) from" - " internal collection. Marking(s) not found...") + raise exceptions.MarkingNotFoundError(obj, remove) granular_markings = [ m for m in granular_markings if m not in remove @@ -105,10 +114,10 @@ def remove_markings(obj, selectors, marking): granular_markings = utils.compress_markings(granular_markings) - if not granular_markings: - return obj.new_version(granular_markings=None) - else: + if granular_markings: return obj.new_version(granular_markings=granular_markings) + else: + return obj.new_version(granular_markings=None) def add_markings(obj, selectors, marking): @@ -116,18 +125,21 @@ def add_markings(obj, selectors, marking): Appends a granular_marking to the granular_markings collection. Args: - obj: A TLO object. + obj: An SDO or SRO object. selectors: list of type string, selectors must be relative to the TLO in which the properties appear. - marking: identifier that apply to the properties selected by - `selectors`. + marking: identifier or list of marking identifiers that apply to the + properties selected by `selectors`. Raises: - AssertionError: If `selectors` or `marking` fail data validation. + InvalidSelectorError: If `selectors` fail validation. + + Returns: + A new version of the given SDO or SRO with specified markings added. """ - selectors = utils.fix_value(selectors) - utils.validate(obj, selectors, marking) + selectors = utils.convert_to_list(selectors) + utils.validate(obj, selectors) if isinstance(marking, list): granular_marking = [] @@ -149,16 +161,20 @@ def clear_markings(obj, selectors): Removes all granular_markings associated with the selectors. Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: - AssertionError: If `selectors` or `marking` fail data validation. Also - if markings to remove are not found on the provided TLO. + InvalidSelectorError: If `selectors` fail validation. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings cleared. """ - selectors = utils.fix_value(selectors) + selectors = utils.convert_to_list(selectors) utils.validate(obj, selectors) granular_markings = obj.get("granular_markings") @@ -179,8 +195,7 @@ def clear_markings(obj, selectors): for clear_marking in clear for clear_selector in clear_marking.get("selectors", []) ): - raise AssertionError("Unable to clear Granular Marking(s) from" - " internal collection. Selector(s) not found...") + raise exceptions.MarkingNotFoundError(obj, clear) for granular_marking in granular_markings: for s in selectors: @@ -192,10 +207,10 @@ def clear_markings(obj, selectors): granular_markings = utils.compress_markings(granular_markings) - if not granular_markings: - return obj.new_version(granular_markings=None) - else: + if granular_markings: return obj.new_version(granular_markings=granular_markings) + else: + return obj.new_version(granular_markings=None) def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): @@ -203,27 +218,30 @@ def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): Checks if field is marked by any marking or by specific marking(s). Args: - obj: A TLO object. - selectors: string or list of selectors strings relative to the TLO in - which the field(s) appear(s). + obj: An SDO or SRO object. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the - field(s) selected by `selectors`. + properties selected by `selectors`. inherited: If True, return markings inherited from the given selector. descendants: If True, return granular markings applied to any children of the given selector. + Raises: + InvalidSelectorError: If `selectors` fail validation. + Returns: - bool: True if ``selectors`` is found on internal TLO collection. + bool: True if ``selectors`` is found on internal SDO or SRO collection. False otherwise. Note: - When a list of marking IDs is provided, if ANY of the provided marking - IDs matches, True is returned. + When a list of marking identifiers is provided, if ANY of the provided + marking identifiers match, True is returned. """ - selectors = utils.fix_value(selectors) - marking = utils.fix_value(marking) - utils.validate(obj, selectors, marking) + selectors = utils.convert_to_list(selectors) + marking = utils.convert_to_list(marking) + utils.validate(obj, selectors) granular_markings = obj.get("granular_markings", []) diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index c0d6d72..a3b2586 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -1,4 +1,5 @@ +from stix2 import exceptions from stix2.markings import utils @@ -10,7 +11,7 @@ def get_markings(obj): obj: A SDO or SRO object. Returns: - list: Marking IDs contained in the SDO or SRO. Empty list if no + list: Marking identifiers contained in the SDO or SRO. Empty list if no markings are present in `object_marking_refs`. """ @@ -25,13 +26,12 @@ def add_markings(obj, marking): obj: A SDO or SRO object. marking: identifier or list of identifiers to apply SDO or SRO object. - Raises: - AssertionError: If `marking` fail data validation. + Returns: + A new version of the given SDO or SRO with specified markings added. """ marking = utils.convert_to_list(marking) - # TODO: Remove set for comparison and raise DuplicateMarkingException. object_markings = set(obj.get("object_marking_refs", []) + marking) return obj.new_version(object_marking_refs=list(object_markings)) @@ -47,12 +47,14 @@ def remove_markings(obj, marking): SDO or SRO object. Raises: - AssertionError: If markings to remove are not found on the provided - SDO or SRO. + MarkingNotFoundError: If markings to remove are not found on + the provided SDO or SRO. + + Returns: + A new version of the given SDO or SRO with specified markings removed. """ marking = utils.convert_to_list(marking) - utils.validate(obj, marking=marking) object_markings = obj.get("object_marking_refs", []) @@ -60,8 +62,7 @@ def remove_markings(obj, marking): return obj if any(x not in obj["object_marking_refs"] for x in marking): - raise AssertionError("Unable to remove Object Level Marking(s) from " - "internal collection. Marking(s) not found...") + raise exceptions.MarkingNotFoundError(obj, marking) new_markings = [x for x in object_markings if x not in marking] if new_markings: @@ -80,6 +81,10 @@ def set_markings(obj, marking): marking: identifier or list of identifiers to apply in the SDO or SRO object. + Returns: + A new version of the given SDO or SRO with specified markings removed + and new ones added. + """ return add_markings(clear_markings(obj), marking) @@ -91,6 +96,9 @@ def clear_markings(obj): Args: obj: A SDO or SRO object. + Returns: + A new version of the given SDO or SRO with object_marking_refs cleared. + """ return obj.new_version(object_marking_refs=None) diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index adf3069..4dcbf00 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -3,9 +3,23 @@ import collections import six +from stix2 import exceptions -def evaluate_expression(obj, selector): +def _evaluate_expression(obj, selector): + """ + Walks an SDO or SRO generating selectors to match against ``selector``. If + a match is found and the the value of this property is present in the + objects. Matching value of the property will be returned. + + Args: + obj: An SDO or SRO object. + selector: A string following the selector syntax. + + Returns: + list: Values contained in matching property. Otherwise empty list. + + """ for items, value in iterpath(obj): path = ".".join(items) @@ -15,45 +29,26 @@ def evaluate_expression(obj, selector): return [] -def validate_selector(obj, selector): - results = list(evaluate_expression(obj, selector)) +def _validate_selector(obj, selector): + results = list(_evaluate_expression(obj, selector)) if len(results) >= 1: return True -def validate_markings(marking): - if isinstance(marking, six.string_types): - if not marking: - return False - else: - return True - - elif isinstance(marking, list) and len(marking) >= 1: - for m in marking: - if not m: - return False - elif not isinstance(m, six.string_types): - return False - - return True - else: - return False - - -def validate(obj, selectors=None, marking=None): - - if selectors is not None: - assert selectors - +def validate(obj, selectors): + """Given an SDO or SRO, check that each selector is valid.""" + if selectors: for s in selectors: - assert validate_selector(obj, s) + if not _validate_selector(obj, s): + raise exceptions.InvalidSelectorError(obj, s) + return - if marking is not None: - assert validate_markings(marking) + raise exceptions.InvalidSelectorError(obj, selectors) def convert_to_list(data): + """Convert input into a list for further processing.""" if data is not None: if isinstance(data, list): return data @@ -61,14 +56,47 @@ def convert_to_list(data): return [data] -def fix_value(data): - data = convert_to_list(data) - - return data - - def compress_markings(granular_markings): + """ + Compress granular markings list. If there is more than one marking + identifier matches. It will collapse into a single granular marking. + Examples: + Input: + [ + { + "selectors": [ + "description" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + }, + { + "selectors": [ + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Output: + [ + { + "selectors": [ + "description", + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Args: + granular_markings: The granular markings list property present in a + SDO or SRO. + + Returns: + list: A list with all markings collapsed. + + """ if not granular_markings: return @@ -88,10 +116,46 @@ def compress_markings(granular_markings): def expand_markings(granular_markings): + """ + Expands granular markings list. If there is more than one selector per + granular marking. It will be expanded using the same marking_ref. - if not granular_markings: - return + Examples: + Input: + [ + { + "selectors": [ + "description", + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + Output: + [ + { + "selectors": [ + "description" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + }, + { + "selectors": [ + "name" + ], + "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + } + ] + + Args: + granular_markings: The granular markings list property present in a + SDO or SRO. + + Returns: + list: A list with all markings expanded. + + """ expanded = [] for marking in granular_markings: @@ -109,11 +173,9 @@ def expand_markings(granular_markings): def build_granular_marking(granular_marking): - tlo = {"granular_markings": granular_marking} - - expand_markings(tlo["granular_markings"]) - - return tlo + """Returns a dictionary with the required structure for a granular + marking""" + return {"granular_markings": expand_markings(granular_marking)} def iterpath(obj, path=None): @@ -122,7 +184,7 @@ def iterpath(obj, path=None): tuple containing a list of ancestors and the property value. Args: - obj: A TLO object. + obj: An SDO or SRO object. path: None, used recursively to store ancestors. Example: @@ -164,36 +226,3 @@ def iterpath(obj, path=None): path.pop() path.pop() - - -def get_selector(obj, prop): - """ - Function that creates a selector based on ``prop``. - - Args: - obj: A TLO object. - prop: A property of the TLO object. - - Note: - Must supply the actual value inside the structure. Since some - limitations exist with Python interning methods, checking for object - location is for now the option to assert the data. - - Example: - >>> selector = get_selector(obj, obj["cybox"]["objects"][0]["file_name"]) - >>> print(selector) - ["cybox.objects.[0].file_name"] - - Returns: - list: A list with one selector that asserts the supplied property. - Empty list if it was unable to find the property. - - """ - selector = [] - - for ancestors, value in iterpath(obj): - if value is prop: - path = ".".join(ancestors) - selector.append(path) - - return selector diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 63aecc3..a253a1b 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -102,7 +102,7 @@ def test_marking_def_invalid_type(): stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", - definition_type="my-definiition-type", + definition_type="my-definition-type", definition=stix2.StatementMarking("Copyright 2016, Example Corp") ) From 1dfaaf15ef2501662d277fec858c98c3f439da34 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 24 Aug 2017 12:54:50 -0400 Subject: [PATCH 60/82] Minor change. closes #29 --- stix2/markings/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 4dcbf00..d0d38bb 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -30,6 +30,7 @@ def _evaluate_expression(obj, selector): def _validate_selector(obj, selector): + """Internal method to evaluate each selector.""" results = list(_evaluate_expression(obj, selector)) if len(results) >= 1: From 3822e39243c3fb5054ec1c179ee7eea206b3032c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 24 Aug 2017 15:10:09 -0400 Subject: [PATCH 61/82] Remove COMMON_PROPERTIES. --- stix2/common.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/stix2/common.py b/stix2/common.py index c126d76..a691b3b 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -6,10 +6,9 @@ except ImportError: from ordereddict import OrderedDict from .base import _STIXBase -from .properties import (BooleanProperty, HashesProperty, IDProperty, - ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) +from .properties import (HashesProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) from .utils import NOW, get_dict @@ -148,17 +147,3 @@ TLP_RED = MarkingDefinition( definition_type="tlp", definition=TLPMarking(tlp="red") ) - -COMMON_PROPERTIES = OrderedDict() - -COMMON_PROPERTIES.update([ - # 'type' and 'id' should be defined on each individual type - ('created_by_ref', ReferenceProperty(type="identity")), - ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('revoked', BooleanProperty()), - ('labels', ListProperty(StringProperty)), - ('external_references', ListProperty(ExternalReference)), - ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), - ('granular_markings', ListProperty(GranularMarking)), -]) From 6fa3c84aa3073de13fb64ab02b324fae2837ec43 Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 24 Aug 2017 15:46:36 -0400 Subject: [PATCH 62/82] Improve tests and coverage for custom content In ObservableProperty.clean(), parse_observable() will handle the issue of non-cyber observable objects being presnt in an observable property. The line raising a ValueError would not be reached so it was removed. --- stix2/observables.py | 17 +++--- stix2/test/test_custom.py | 111 +++++++++++++++++++++++++++++++++----- 2 files changed, 106 insertions(+), 22 deletions(-) diff --git a/stix2/observables.py b/stix2/observables.py index 3c033bd..50eb6d1 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -31,9 +31,6 @@ class ObservableProperty(Property): # from .__init__ import parse_observable # avoid circular import for key, obj in dictified.items(): parsed_obj = parse_observable(obj, valid_refs) - if not issubclass(type(parsed_obj), _Observable): - raise ValueError("Objects in an observable property must be " - "Cyber Observable Objects") dictified[key] = parsed_obj return dictified @@ -734,16 +731,17 @@ def parse_observable(data, _valid_refs=[], allow_custom=False): obj['_valid_refs'] = _valid_refs if 'type' not in obj: - raise ParseError("Can't parse object with no 'type' property: %s" % str(obj)) + raise ParseError("Can't parse observable with no 'type' property: %s" % str(obj)) try: obj_class = OBJ_MAP_OBSERVABLE[obj['type']] except KeyError: - raise ParseError("Can't parse unknown object type '%s'! For custom observables, use the CustomObservable decorator." % obj['type']) + raise ParseError("Can't parse unknown observable type '%s'! For custom observables, " + "use the CustomObservable decorator." % obj['type']) if 'extensions' in obj and obj['type'] in EXT_MAP: for name, ext in obj['extensions'].items(): if name not in EXT_MAP[obj['type']]: - raise ParseError("Can't parse Unknown extension type '%s' for object type '%s'!" % (name, obj['type'])) + raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type'])) ext_class = EXT_MAP[obj['type']][name] obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) @@ -787,13 +785,16 @@ def _register_extension(observable, new_extension): try: observable_type = observable._type except AttributeError: - raise ValueError("Custom observables must be created with the @CustomObservable decorator.") + raise ValueError("Unknown observable type. Custom observables must be " + "created with the @CustomObservable decorator.") try: EXT_MAP[observable_type][new_extension._type] = new_extension except KeyError: if observable_type not in OBJ_MAP_OBSERVABLE: - raise ValueError("Unknown observable type '%s'" % observable_type) + raise ValueError("Unknown observable type '%s'. Custom observables " + "must be created with the @CustomObservable decorator." + % observable_type) else: EXT_MAP[observable_type] = {new_extension._type: new_extension} diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 804776a..33d9287 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -6,7 +6,7 @@ from .constants import FAKE_TIME def test_identity_custom_property(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", created="2015-12-21T19:59:11Z", @@ -15,6 +15,7 @@ def test_identity_custom_property(): identity_class="individual", custom_properties="foobar", ) + assert str(excinfo.value) == "'custom_properties' must be a dictionary" identity = stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", @@ -31,7 +32,7 @@ def test_identity_custom_property(): def test_identity_custom_property_invalid(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: stix2.Identity( id="identity--311b2d2d-f010-5473-83ec-1edf84858f4c", created="2015-12-21T19:59:11Z", @@ -40,6 +41,9 @@ def test_identity_custom_property_invalid(): identity_class="individual", x_foo="bar", ) + assert excinfo.value.cls == stix2.Identity + assert excinfo.value.properties == ['x_foo'] + assert "Unexpected properties for" in str(excinfo.value) def test_identity_custom_property_allowed(): @@ -67,8 +71,11 @@ def test_identity_custom_property_allowed(): }""", ]) def test_parse_identity_custom_property(data): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: identity = stix2.parse(data) + assert excinfo.value.cls == stix2.Identity + assert excinfo.value.properties == ['foo'] + assert "Unexpected properties for" in str(excinfo.value) identity = stix2.parse(data, allow_custom=True) assert identity.foo == "bar" @@ -88,11 +95,13 @@ def test_custom_object_type(): nt = NewType(property1='something') assert nt.property1 == 'something' - with pytest.raises(stix2.exceptions.MissingPropertiesError): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewType(property2=42) + assert "No values for required properties" in str(excinfo.value) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewType(property1='something', property2=4) + assert "'property2' is too small." in str(excinfo.value) def test_parse_custom_object_type(): @@ -120,11 +129,14 @@ def test_custom_observable_object(): no = NewObservable(property1='something') assert no.property1 == 'something' - with pytest.raises(stix2.exceptions.MissingPropertiesError): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewObservable(property2=42) + assert excinfo.value.properties == ['property1'] + assert "No values for required properties" in str(excinfo.value) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewObservable(property1='something', property2=4) + assert "'property2' is too small." in str(excinfo.value) def test_parse_custom_observable_object(): @@ -137,12 +149,34 @@ def test_parse_custom_observable_object(): assert nt.property1 == 'something' +def test_parse_unregistered_custom_observable_object(): + nt_string = """{ + "type": "x-foobar-observable", + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse_observable(nt_string) + assert "Can't parse unknown observable type" in str(excinfo.value) + + +def test_parse_invalid_custom_observable_object(): + nt_string = """{ + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse_observable(nt_string) + assert "Can't parse observable with no 'type' property" in str(excinfo.value) + + def test_observable_custom_property(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewObservable( property1='something', custom_properties="foobar", ) + assert "'custom_properties' must be a dictionary" in str(excinfo.value) no = NewObservable( property1='something', @@ -154,11 +188,13 @@ def test_observable_custom_property(): def test_observable_custom_property_invalid(): - with pytest.raises(stix2.exceptions.ExtraPropertiesError): + with pytest.raises(stix2.exceptions.ExtraPropertiesError) as excinfo: NewObservable( property1='something', x_foo="bar", ) + assert excinfo.value.properties == ['x_foo'] + assert "Unexpected properties for" in str(excinfo.value) def test_observable_custom_property_allowed(): @@ -196,31 +232,61 @@ def test_custom_extension(): ext = NewExtension(property1='something') assert ext.property1 == 'something' - with pytest.raises(stix2.exceptions.MissingPropertiesError): + with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewExtension(property2=42) + assert excinfo.value.properties == ['property1'] + assert str(excinfo.value) == "No values for required properties for _Custom: (property1)." - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: NewExtension(property1='something', property2=4) + assert str(excinfo.value) == "'property2' is too small." -def test_custom_extension_invalid(): +def test_custom_extension_wrong_observable_type(): + ext = NewExtension(property1='something') + with pytest.raises(ValueError) as excinfo: + stix2.File(name="abc.txt", + extensions={ + "ntfs-ext": ext, + }) + + assert 'Cannot determine extension type' in excinfo.value.reason + + +def test_custom_extension_invalid_observable(): + # These extensions are being applied to improperly-created Observables. + # The Observable classes should have been created with the CustomObservable decorator. class Foo(object): pass - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: @stix2.observables.CustomExtension(Foo, 'x-new-ext', { 'property1': stix2.properties.StringProperty(required=True), }) class FooExtension(): pass # pragma: no cover + assert str(excinfo.value) == "'observable' must be a valid Observable class!" class Bar(stix2.observables._Observable): pass - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: @stix2.observables.CustomExtension(Bar, 'x-new-ext', { 'property1': stix2.properties.StringProperty(required=True), }) class BarExtension(): pass + assert "Unknown observable type" in str(excinfo.value) + assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) + + class Baz(stix2.observables._Observable): + _type = 'Baz' + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomExtension(Baz, 'x-new-ext', { + 'property1': stix2.properties.StringProperty(required=True), + }) + class BazExtension(): + pass + assert "Unknown observable type" in str(excinfo.value) + assert "Custom observables must be created with the @CustomObservable decorator." in str(excinfo.value) def test_parse_observable_with_custom_extension(): @@ -237,3 +303,20 @@ def test_parse_observable_with_custom_extension(): parsed = stix2.parse_observable(input_str) assert parsed.extensions['x-new-ext'].property2 == 12 + + +def test_parse_observable_with_unregistered_custom_extension(): + input_str = """{ + "type": "domain-name", + "value": "example.com", + "extensions": { + "x-foobar-ext": { + "property1": "foo", + "property2": 12 + } + } + }""" + + with pytest.raises(ValueError) as excinfo: + stix2.parse_observable(input_str) + assert "Can't parse Unknown extension type" in str(excinfo.value) From 0cd75e3fba56f603af150a31f5b7c740e7e1d381 Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 24 Aug 2017 17:53:43 -0400 Subject: [PATCH 63/82] Increase code coverage --- stix2/exceptions.py | 2 +- stix2/test/test_bundle.py | 14 ++++++++ stix2/test/test_custom.py | 43 +++++++++++++++++++++++++ stix2/test/test_observed_data.py | 41 ++++++++++++++++++++++++ stix2/test/test_properties.py | 55 ++++++++++++++++++++++++++++---- stix2/test/test_versioning.py | 2 ++ 6 files changed, 149 insertions(+), 8 deletions(-) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index ef47dd0..8a33afa 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -121,7 +121,7 @@ class DependentPropertiesError(STIXError, TypeError): def __str__(self): msg = "The property dependencies for {0}: ({1}) are not met." return msg.format(self.cls.__name__, - ", ".join(x for x in self.dependencies)) + ", ".join(x for x, y in self.dependencies)) class AtLeastOnePropertyError(STIXError, TypeError): diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 0733637..268c2f7 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -118,6 +118,20 @@ def test_create_bundle_with_arg_listarg_and_kwarg(indicator, malware, relationsh assert str(bundle) == EXPECTED_BUNDLE +def test_create_bundle_invalid(indicator, malware, relationship): + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[1]) + assert excinfo.value.reason == "This property may only contain a dictionary or object" + + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[{}]) + assert excinfo.value.reason == "This property may only contain a non-empty dictionary or object" + + with pytest.raises(ValueError) as excinfo: + stix2.Bundle(objects=[{'type': 'bundle'}]) + assert excinfo.value.reason == 'This property may not contain a Bundle object' + + def test_parse_bundle(): bundle = stix2.parse(EXPECTED_BUNDLE) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 33d9287..d2bb16f 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -115,6 +115,19 @@ def test_parse_custom_object_type(): assert nt.property1 == 'something' +def test_parse_unregistered_custom_object_type(): + nt_string = """{ + "type": "x-foobar-observable", + "created": "2015-12-21T19:59:11Z", + "property1": "something" + }""" + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse(nt_string) + assert "Can't parse unknown object type" in str(excinfo.value) + assert "use the CustomObject decorator." in str(excinfo.value) + + @stix2.observables.CustomObservable('x-new-observable', { 'property1': stix2.properties.StringProperty(required=True), 'property2': stix2.properties.IntegerProperty(), @@ -139,6 +152,36 @@ def test_custom_observable_object(): assert "'property2' is too small." in str(excinfo.value) +def test_custom_observable_object_invalid_ref_property(): + @stix2.observables.CustomObservable('x-new-obs', { + 'property1': stix2.properties.StringProperty(required=True), + 'property_ref': stix2.properties.StringProperty(), + }) + class NewObs(): + pass + + with pytest.raises(ValueError) as excinfo: + NewObs(_valid_refs={'1': 'file'}, + property1='something', + property_ref='1') + assert "is named like an object reference property but is not an ObjectReferenceProperty" in str(excinfo.value) + + +def test_custom_observable_object_invalid_valid_refs(): + @stix2.observables.CustomObservable('x-new-obs', { + 'property1': stix2.properties.StringProperty(required=True), + 'property_ref': stix2.properties.ObjectReferenceProperty(valid_types='email-addr'), + }) + class NewObs(): + pass + + with pytest.raises(Exception) as excinfo: + NewObs(_valid_refs=['1'], + property1='something', + property_ref='1') + assert "must be created with _valid_refs as a dict, not a list" in str(excinfo.value) + + def test_parse_custom_observable_object(): nt_string = """{ "type": "x-new-observable", diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index d5641e7..1c71b22 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -125,6 +125,42 @@ def test_observed_data_example_with_bad_refs(): assert excinfo.value.reason == "Invalid object reference for 'Directory:contains_refs': '2' is not a valid object in local scope" +def test_observed_data_example_with_non_dictionary(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects="file: foo.exe", + ) + + assert excinfo.value.cls == stix2.ObservedData + assert excinfo.value.prop_name == "objects" + assert 'must contain a dictionary' in excinfo.value.reason + + +def test_observed_data_example_with_empty_dictionary(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.ObservedData( + id="observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T19:58:16.000Z", + modified="2016-04-06T19:58:16.000Z", + first_observed="2015-12-21T19:00:00Z", + last_observed="2015-12-21T19:00:00Z", + number_observed=50, + objects={}, + ) + + assert excinfo.value.cls == stix2.ObservedData + assert excinfo.value.prop_name == "objects" + assert 'must contain a non-empty dictionary' in excinfo.value.reason + + @pytest.mark.parametrize("data", [ EXPECTED, { @@ -416,6 +452,8 @@ def test_parse_email_message_with_at_least_one_error(data): assert excinfo.value.cls == stix2.EmailMIMEComponent assert excinfo.value.properties == ["body", "body_raw_ref"] + assert "At least one of the" in str(excinfo.value) + assert "must be populated" in str(excinfo.value) @pytest.mark.parametrize("data", [ @@ -555,6 +593,7 @@ def test_artifact_mutual_exclusion_error(): assert excinfo.value.cls == stix2.Artifact assert excinfo.value.properties == ["payload_bin", "url"] + assert 'are mutually exclusive' in str(excinfo.value) def test_directory_example(): @@ -800,6 +839,8 @@ def test_file_example_encryption_error(): assert excinfo.value.cls == stix2.File assert excinfo.value.dependencies == [("is_encrypted", "encryption_algorithm")] + assert "property dependencies" in str(excinfo.value) + assert "are not met" in str(excinfo.value) with pytest.raises(stix2.exceptions.DependentPropertiesError) as excinfo: stix2.File(name="qwerty.dll", diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 01daebf..4efc51a 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -5,10 +5,10 @@ from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError from stix2.observables import EmailMIMEComponent, ExtensionsProperty from stix2.properties import (BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, - EnumProperty, HashesProperty, HexProperty, - IDProperty, IntegerProperty, ListProperty, - Property, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty) + EnumProperty, FloatProperty, HashesProperty, + HexProperty, IDProperty, IntegerProperty, + ListProperty, Property, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty) from .constants import FAKE_TIME @@ -119,6 +119,27 @@ def test_integer_property_invalid(value): int_prop.clean(value) +@pytest.mark.parametrize("value", [ + 2, + -1, + 3.14, + False, +]) +def test_float_property_valid(value): + int_prop = FloatProperty() + assert int_prop.clean(value) is not None + + +@pytest.mark.parametrize("value", [ + "something", + StringProperty(), +]) +def test_float_property_invalid(value): + int_prop = FloatProperty() + with pytest.raises(ValueError): + int_prop.clean(value) + + @pytest.mark.parametrize("value", [ True, False, @@ -210,10 +231,22 @@ def test_dictionary_property_valid(d): {'a'*300: 'something'}, {'Hey!': 'something'}, ]) +def test_dictionary_property_invalid_key(d): + dict_prop = DictionaryProperty() + + with pytest.raises(DictionaryKeyError) as excinfo: + dict_prop.clean(d) + assert "Invalid dictionary key" in str(excinfo.value) + + +@pytest.mark.parametrize("d", [ + {}, + "{'description': 'something'}", +]) def test_dictionary_property_invalid(d): dict_prop = DictionaryProperty() - with pytest.raises(DictionaryKeyError): + with pytest.raises(ValueError): dict_prop.clean(d) @@ -250,10 +283,18 @@ def test_embedded_property(): emb_prop.clean("string") -def test_enum_property(): - enum_prop = EnumProperty(['a', 'b', 'c']) +@pytest.mark.parametrize("value", [ + ['a', 'b', 'c'], + ('a', 'b', 'c'), + 'b', +]) +def test_enum_property_valid(value): + enum_prop = EnumProperty(value) assert enum_prop.clean('b') + +def test_enum_property_invalid(): + enum_prop = EnumProperty(['a', 'b', 'c']) with pytest.raises(ValueError): enum_prop.clean('z') diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 281ae71..42584ad 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -160,6 +160,7 @@ def test_versioning_error_new_version_of_revoked(): with pytest.raises(stix2.exceptions.RevokeError) as excinfo: campaign_v2.new_version(name="barney") + assert str(excinfo.value) == "Cannot create a new version of a revoked object." assert excinfo.value.called_by == "new_version" @@ -178,5 +179,6 @@ def test_versioning_error_revoke_of_revoked(): with pytest.raises(stix2.exceptions.RevokeError) as excinfo: campaign_v2.revoke() + assert str(excinfo.value) == "Cannot revoke an already revoked object." assert excinfo.value.called_by == "revoke" From 095c5066d5e5109f4c723bb2a59ef54e307a9a9e Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 25 Aug 2017 15:56:39 -0400 Subject: [PATCH 64/82] Touch up markings tests Increase code coverage and s/tlo/sdo/ --- stix2/markings/granular_markings.py | 8 +- stix2/test/test_granular_markings.py | 183 +++++++++++++++------------ stix2/test/test_object_markings.py | 133 ++++++++++--------- 3 files changed, 177 insertions(+), 147 deletions(-) diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 6f2933f..7e2c50a 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -184,14 +184,14 @@ def clear_markings(obj, selectors): granular_markings = utils.expand_markings(granular_markings) - tlo = utils.build_granular_marking( + sdo = utils.build_granular_marking( [{"selectors": selectors, "marking_ref": "N/A"}] ) - clear = tlo.get("granular_markings", []) + clear = sdo.get("granular_markings", []) - if not any(clear_selector in tlo_selectors.get("selectors", []) - for tlo_selectors in granular_markings + if not any(clear_selector in sdo_selectors.get("selectors", []) + for sdo_selectors in granular_markings for clear_marking in clear for clear_selector in clear_marking.get("selectors", []) ): diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index 2f0524b..766348e 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -220,7 +220,10 @@ def test_get_markings_smoke(data): assert markings.get_markings(data, "a") == ["1"] -@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA]) +@pytest.mark.parametrize("data", [ + GET_MARKINGS_TEST_DATA, + {"b": 1234}, +]) def test_get_markings_not_marked(data): """Test selector that is not marked returns empty list.""" results = markings.get_markings(data, "b") @@ -362,7 +365,7 @@ def test_remove_marking_remove_multiple_selector_one_ref(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["description", "modified"], [MARKING_IDS[0]]) + before = markings.remove_markings(before, ["description", "modified"], MARKING_IDS[0]) assert "granular_markings" not in before @@ -385,7 +388,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) + before = markings.remove_markings(before, ["modified"], MARKING_IDS[0]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -482,6 +485,14 @@ def test_remove_marking_mark_same_property_same_marking(): assert "granular_markings" not in before +def test_remove_no_markings(): + before = { + "description": "test description", + } + after = markings.remove_markings(before, ["description"], ["marking-definition--1"]) + assert before == after + + def test_remove_marking_bad_selector(): before = { "description": "test description", @@ -584,7 +595,7 @@ def test_is_marked_no_marking_refs(data): def test_is_marked_positional_arguments_combinations(): """Test multiple combinations for inherited and descendant markings.""" - test_tlo = \ + test_sdo = \ { "a": 333, "b": "value", @@ -650,75 +661,89 @@ def test_is_marked_positional_arguments_combinations(): ] } - assert markings.is_marked(test_tlo, "a", ["1"], False, False) - assert markings.is_marked(test_tlo, "a", ["1"], True, False) - assert markings.is_marked(test_tlo, "a", ["1"], True, True) - assert markings.is_marked(test_tlo, "a", ["1"], False, True) + assert markings.is_marked(test_sdo, "a", ["1"], False, False) + assert markings.is_marked(test_sdo, "a", ["1"], True, False) + assert markings.is_marked(test_sdo, "a", ["1"], True, True) + assert markings.is_marked(test_sdo, "a", ["1"], False, True) - assert markings.is_marked(test_tlo, "b", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "b", inherited=True, descendants=False) is False - assert markings.is_marked(test_tlo, "b", inherited=True, descendants=True) is False - assert markings.is_marked(test_tlo, "b", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "b", inherited=True, descendants=False) is False + assert markings.is_marked(test_sdo, "b", inherited=True, descendants=True) is False + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "c", ["2"], False, False) - assert markings.is_marked(test_tlo, "c", ["2"], True, False) - assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], True, True) - assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], False, True) + assert markings.is_marked(test_sdo, "c", ["2"], False, False) + assert markings.is_marked(test_sdo, "c", ["2"], True, False) + assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], True, True) + assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], False, True) - assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "c.[0]", ["2"], True, False) - assert markings.is_marked(test_tlo, "c.[0]", ["2"], True, True) - assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "c.[0]", ["2"], True, False) + assert markings.is_marked(test_sdo, "c.[0]", ["2"], True, True) + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, False) - assert markings.is_marked(test_tlo, "c.[1]", ["2", "3"], True, False) - assert markings.is_marked(test_tlo, "c.[1]", ["2", "3"], True, True) - assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, True) + assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, False) + assert markings.is_marked(test_sdo, "c.[1]", ["2", "3"], True, False) + assert markings.is_marked(test_sdo, "c.[1]", ["2", "3"], True, True) + assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, True) - assert markings.is_marked(test_tlo, "c.[2]", ["4"], False, False) - assert markings.is_marked(test_tlo, "c.[2]", ["2", "4"], True, False) - assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "5"], True, True) - assert markings.is_marked(test_tlo, "c.[2]", ["4", "5"], False, True) + assert markings.is_marked(test_sdo, "c.[2]", ["4"], False, False) + assert markings.is_marked(test_sdo, "c.[2]", ["2", "4"], True, False) + assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "5"], True, True) + assert markings.is_marked(test_sdo, "c.[2]", ["4", "5"], False, True) - assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, False) - assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5"], True, False) - assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5"], True, True) - assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, True) + assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, False) + assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5"], True, False) + assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5"], True, True) + assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, True) - assert markings.is_marked(test_tlo, "x", ["6"], False, False) - assert markings.is_marked(test_tlo, "x", ["6"], True, False) - assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], True, True) - assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], False, True) + assert markings.is_marked(test_sdo, "x", ["6"], False, False) + assert markings.is_marked(test_sdo, "x", ["6"], True, False) + assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], True, True) + assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], False, True) - assert markings.is_marked(test_tlo, "x.y", ["7"], False, False) - assert markings.is_marked(test_tlo, "x.y", ["6", "7"], True, False) - assert markings.is_marked(test_tlo, "x.y", ["6", "7", "8"], True, True) - assert markings.is_marked(test_tlo, "x.y", ["7", "8"], False, True) + assert markings.is_marked(test_sdo, "x.y", ["7"], False, False) + assert markings.is_marked(test_sdo, "x.y", ["6", "7"], True, False) + assert markings.is_marked(test_sdo, "x.y", ["6", "7", "8"], True, True) + assert markings.is_marked(test_sdo, "x.y", ["7", "8"], False, True) - assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7"], True, False) - assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7"], True, True) - assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7"], True, False) + assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7"], True, True) + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, False) - assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8"], True, False) - assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8"], True, True) - assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, True) + assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, False) + assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8"], True, False) + assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8"], True, True) + assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, True) - assert markings.is_marked(test_tlo, "x.z", ["9"], False, False) - assert markings.is_marked(test_tlo, "x.z", ["6", "9"], True, False) - assert markings.is_marked(test_tlo, "x.z", ["6", "9", "10"], True, True) - assert markings.is_marked(test_tlo, "x.z", ["9", "10"], False, True) + assert markings.is_marked(test_sdo, "x.z", ["9"], False, False) + assert markings.is_marked(test_sdo, "x.z", ["6", "9"], True, False) + assert markings.is_marked(test_sdo, "x.z", ["6", "9", "10"], True, True) + assert markings.is_marked(test_sdo, "x.z", ["9", "10"], False, True) - assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9"], True, False) - assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9"], True, True) - assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9"], True, False) + assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9"], True, True) + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, False) - assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10"], True, False) - assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10"], True, True) - assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, True) + assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, False) + assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10"], True, False) + assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10"], True, True) + assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, True) + + +def test_create_sdo_with_invalid_marking(): + with pytest.raises(AssertionError) as excinfo: + Malware( + granular_markings=[ + { + "selectors": ["foo"], + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) + assert str(excinfo.value) == "Selector foo in Malware is not valid!" def test_set_marking_mark_one_selector_multiple_refs(): @@ -819,35 +844,33 @@ def test_set_marking_mark_another_property_same_marking(): @pytest.mark.parametrize("marking", [ - (["foo"], ["marking-definition--7", "marking-definition--8"]), - ("", ["marking-definition--7", "marking-definition--8"]), - ([], ["marking-definition--7", "marking-definition--8"]), - ([""], ["marking-definition--7", "marking-definition--8"]) + (["foo"], [MARKING_IDS[4], MARKING_IDS[5]]), + ("", [MARKING_IDS[4], MARKING_IDS[5]]), + ([], [MARKING_IDS[4], MARKING_IDS[5]]), + ([""], [MARKING_IDS[4], MARKING_IDS[5]]), ]) def test_set_marking_bad_selector(marking): - before = { - "description": "test description", - "title": "foo", - "granular_markings": [ + before = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" + "marking_ref": MARKING_IDS[0] } - ] - } - after = { - "description": "test description", - "title": "foo", - "granular_markings": [ + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ { "selectors": ["description"], - "marking_ref": "marking-definition--1" - }, - ] - } + "marking_ref": MARKING_IDS[0] + } + ], + **MALWARE_KWARGS + ) with pytest.raises(AssertionError): - markings.set_markings(before, marking[0], marking[1]) + before = markings.set_markings(before, marking[0], marking[1]) assert before == after diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index 25792a7..54c06f4 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -256,6 +256,10 @@ def test_remove_markings_object_level(): assert 'object_marking_refs' not in before assert 'object_marking_refs' not in after + modified = after.modified + after = markings.remove_markings(after, None, MARKING_IDS[0]) + modified == after.modified + def test_remove_markings_multiple(): before = Malware( @@ -273,13 +277,13 @@ def test_remove_markings_multiple(): def test_remove_markings_bad_markings(): - before = { - "title": "test title", - "description": "test description", - "object_marking_refs": [MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]] - } - with pytest.raises(AssertionError): + before = Malware( + object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS + ) + with pytest.raises(AssertionError) as excinfo: markings.remove_markings(before, None, [MARKING_IDS[4]]) + assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] def test_clear_markings(): @@ -299,7 +303,7 @@ def test_clear_markings(): def test_is_marked_object_and_granular_combinations(): """Test multiple combinations for inherited and descendant markings.""" - test_tlo = \ + test_sdo = \ { "a": 333, "b": "value", @@ -366,75 +370,78 @@ def test_is_marked_object_and_granular_combinations(): ] } - assert markings.is_marked(test_tlo, "a", ["1"], False, False) - assert markings.is_marked(test_tlo, "a", ["1", "11"], True, False) - assert markings.is_marked(test_tlo, "a", ["1", "11"], True, True) - assert markings.is_marked(test_tlo, "a", ["1"], False, True) + assert markings.is_marked(test_sdo, "a", ["1"], False, False) + assert markings.is_marked(test_sdo, "a", ["1", "11"], True, False) + assert markings.is_marked(test_sdo, "a", ["1", "11"], True, True) + assert markings.is_marked(test_sdo, "a", ["1"], False, True) - assert markings.is_marked(test_tlo, "b", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "b", ["11"], True, False) - assert markings.is_marked(test_tlo, "b", ["11"], True, True) - assert markings.is_marked(test_tlo, "b", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "b", ["11"], True, False) + assert markings.is_marked(test_sdo, "b", ["11"], True, True) + assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "c", ["2"], False, False) - assert markings.is_marked(test_tlo, "c", ["2", "11"], True, False) - assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5", "11"], True, True) - assert markings.is_marked(test_tlo, "c", ["2", "3", "4", "5"], False, True) + assert markings.is_marked(test_sdo, "c", ["2"], False, False) + assert markings.is_marked(test_sdo, "c", ["2", "11"], True, False) + assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5", "11"], True, True) + assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], False, True) - assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "c.[0]", ["2", "11"], True, False) - assert markings.is_marked(test_tlo, "c.[0]", ["2", "11"], True, True) - assert markings.is_marked(test_tlo, "c.[0]", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "c.[0]", ["2", "11"], True, False) + assert markings.is_marked(test_sdo, "c.[0]", ["2", "11"], True, True) + assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, False) - assert markings.is_marked(test_tlo, "c.[1]", ["2", "3", "11"], True, False) - assert markings.is_marked(test_tlo, "c.[1]", ["2", "3", "11"], True, True) - assert markings.is_marked(test_tlo, "c.[1]", ["3"], False, True) + assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, False) + assert markings.is_marked(test_sdo, "c.[1]", ["2", "3", "11"], True, False) + assert markings.is_marked(test_sdo, "c.[1]", ["2", "3", "11"], True, True) + assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, True) - assert markings.is_marked(test_tlo, "c.[2]", ["4"], False, False) - assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "11"], True, False) - assert markings.is_marked(test_tlo, "c.[2]", ["2", "4", "5", "11"], True, True) - assert markings.is_marked(test_tlo, "c.[2]", ["4", "5"], False, True) + assert markings.is_marked(test_sdo, "c.[2]", ["4"], False, False) + assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "11"], True, False) + assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "5", "11"], True, True) + assert markings.is_marked(test_sdo, "c.[2]", ["4", "5"], False, True) - assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, False) - assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5", "11"], True, False) - assert markings.is_marked(test_tlo, "c.[2].g", ["2", "4", "5", "11"], True, True) - assert markings.is_marked(test_tlo, "c.[2].g", ["5"], False, True) + assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, False) + assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5", "11"], True, False) + assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5", "11"], True, True) + assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, True) - assert markings.is_marked(test_tlo, "x", ["6"], False, False) - assert markings.is_marked(test_tlo, "x", ["6", "11"], True, False) - assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10", "11"], True, True) - assert markings.is_marked(test_tlo, "x", ["6", "7", "8", "9", "10"], False, True) + assert markings.is_marked(test_sdo, "x", ["6"], False, False) + assert markings.is_marked(test_sdo, "x", ["6", "11"], True, False) + assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10", "11"], True, True) + assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], False, True) - assert markings.is_marked(test_tlo, "x.y", ["7"], False, False) - assert markings.is_marked(test_tlo, "x.y", ["6", "7", "11"], True, False) - assert markings.is_marked(test_tlo, "x.y", ["6", "7", "8", "11"], True, True) - assert markings.is_marked(test_tlo, "x.y", ["7", "8"], False, True) + assert markings.is_marked(test_sdo, "x.y", ["7"], False, False) + assert markings.is_marked(test_sdo, "x.y", ["6", "7", "11"], True, False) + assert markings.is_marked(test_sdo, "x.y", ["6", "7", "8", "11"], True, True) + assert markings.is_marked(test_sdo, "x.y", ["7", "8"], False, True) - assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7", "11"], True, False) - assert markings.is_marked(test_tlo, "x.y.[0]", ["6", "7", "11"], True, True) - assert markings.is_marked(test_tlo, "x.y.[0]", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7", "11"], True, False) + assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7", "11"], True, True) + assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, False) - assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8", "11"], True, False) - assert markings.is_marked(test_tlo, "x.y.[1]", ["6", "7", "8", "11"], True, True) - assert markings.is_marked(test_tlo, "x.y.[1]", ["8"], False, True) + assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, False) + assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8", "11"], True, False) + assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8", "11"], True, True) + assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, True) - assert markings.is_marked(test_tlo, "x.z", ["9"], False, False) - assert markings.is_marked(test_tlo, "x.z", ["6", "9", "11"], True, False) - assert markings.is_marked(test_tlo, "x.z", ["6", "9", "10", "11"], True, True) - assert markings.is_marked(test_tlo, "x.z", ["9", "10"], False, True) + assert markings.is_marked(test_sdo, "x.z", ["9"], False, False) + assert markings.is_marked(test_sdo, "x.z", ["6", "9", "11"], True, False) + assert markings.is_marked(test_sdo, "x.z", ["6", "9", "10", "11"], True, True) + assert markings.is_marked(test_sdo, "x.z", ["9", "10"], False, True) - assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=False) is False - assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9", "11"], True, False) - assert markings.is_marked(test_tlo, "x.z.foo1", ["6", "9", "11"], True, True) - assert markings.is_marked(test_tlo, "x.z.foo1", inherited=False, descendants=True) is False + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False + assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9", "11"], True, False) + assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9", "11"], True, True) + assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False - assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, False) - assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10", "11"], True, False) - assert markings.is_marked(test_tlo, "x.z.foo2", ["6", "9", "10", "11"], True, True) - assert markings.is_marked(test_tlo, "x.z.foo2", ["10"], False, True) + assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, False) + assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10", "11"], True, False) + assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10", "11"], True, True) + assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, True) + + assert markings.is_marked(test_sdo, None, ["11"], True, True) + assert markings.is_marked(test_sdo, None, ["2"], True, True) is False def test_set_marking(): From d3f1eb86ab5e8730a18a635cfa7bddb4cbf31acd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:29:10 -0400 Subject: [PATCH 65/82] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b8d544..d9b1edc 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'ordereddict', + 'ordereddict ; python_version<"2.7"', 'python-dateutil', 'pytz', 'requests', From 23f36a9a69d103ca1627800f88fc8694ddfbab0c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:30:53 -0400 Subject: [PATCH 66/82] Minor changes to custom object decorators. Add new marking decorator. --- stix2/__init__.py | 5 +++-- stix2/common.py | 49 ++++++++++++++++++++++++++++++++++++++------ stix2/observables.py | 2 +- stix2/sdo.py | 9 +++----- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index b9b6764..76ca696 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -4,8 +4,9 @@ from . import exceptions from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) + CustomMarking, ExternalReference, GranularMarking, + KillChainPhase, MarkingDefinition, StatementMarking, + TLPMarking) from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, diff --git a/stix2/common.py b/stix2/common.py index a691b3b..6242989 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -109,17 +109,54 @@ class MarkingDefinition(_STIXBase): super(MarkingDefinition, self).__init__(**kwargs) -def register_marking(new_marking): - """Register a custom STIX Marking Definition type. - """ - OBJ_MAP_MARKING[new_marking._type] = new_marking - - OBJ_MAP_MARKING = { 'tlp': TLPMarking, 'statement': StatementMarking, } + +def _register_marking(cls): + """Register a custom STIX Marking Definition type. + """ + OBJ_MAP_MARKING[cls._type] = cls + return cls + + +def CustomMarking(type='x-custom-marking', properties=None): + """ + Custom STIX Marking decorator. + + Examples: + + @CustomMarking('x-custom-marking', [ + ('property1', StringProperty(required=True)), + ('property2', IntegerProperty()), + ]) + class MyNewMarkingObjectType(): + pass + + """ + def custom_builder(cls): + + class _Custom(cls, _STIXBase): + _type = type + _properties = OrderedDict() + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update(properties) + + def __init__(self, **kwargs): + _STIXBase.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_marking(_Custom) + return _Custom + + return custom_builder + + TLP_WHITE = MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", diff --git a/stix2/observables.py b/stix2/observables.py index 4a1319c..6c82e99 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -804,7 +804,7 @@ def CustomObservable(type='x-custom-observable', properties=None): ('type', TypeProperty(_type)), ]) - if not properties: + if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") _properties.update(properties) diff --git a/stix2/sdo.py b/stix2/sdo.py index 7eebf56..8c27d11 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -330,13 +330,10 @@ def CustomObject(type='x-custom-type', properties=None): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ]) - if not properties: + if not properties or not isinstance(properties, list): raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") - normal_properties = [x for x in properties if not x[0].startswith("x_")] - custom_properties = [x for x in properties if x[0].startswith("x_")] - - _properties.update(normal_properties) + _properties.update([x for x in properties if not x[0].startswith("x_")]) # This is to follow the general properties structure. _properties.update([ @@ -348,7 +345,7 @@ def CustomObject(type='x-custom-type', properties=None): ]) # Put all custom properties at the bottom, sorted alphabetically. - _properties.update(sorted(custom_properties, key=lambda x: x[0])) + _properties.update(sorted([x for x in properties if x[0].startswith("x_")], key=lambda x: x[0])) def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) From 07bbe23a98f12dd9f5d8b6012baacd0cc44d4a19 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:32:05 -0400 Subject: [PATCH 67/82] Add some tests for custom objects. --- stix2/test/test_custom.py | 34 ++++++++++++++++++++++ stix2/test/test_markings.py | 56 +++++++++++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 4ce99c6..2b08883 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -128,6 +128,40 @@ def test_custom_observable_object(): NewObservable(property1='something', property2=4) +def test_custom_no_properties_raises_exception(): + with pytest.raises(ValueError): + + @stix2.sdo.CustomObject('x-new-object-type') + class NewObject1(object): + pass + + NewObject1() + + @stix2.sdo.CustomObject('x-new-object-type', ("a", 0)) + class NewObject2(object): + pass + + NewObject2() + + @stix2.observables.CustomObservable('x-new-object-type') + class NewObject3(object): + pass + + NewObject3() + + @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) + class NewObject4(object): + pass + + NewObject4() + + @stix2.common.CustomMarking('x-new-marking-type') + class NewObject5(object): + pass + + NewObject5() + + def test_parse_custom_observable_object(): nt_string = """{ "type": "x-new-observable", diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index f36b032..4e39c54 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -75,7 +75,7 @@ def test_marking_def_example_with_tlp(): assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION -def test_marking_def_example_with_statement(): +def test_marking_def_example_with_statement_positional_argument(): marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", @@ -86,12 +86,13 @@ def test_marking_def_example_with_statement(): assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION -def test_marking_def_example_with_positional_statement(): +def test_marking_def_example_with_kwargs_statement(): + kwargs = dict(statement="Copyright 2016, Example Corp") marking_definition = stix2.MarkingDefinition( id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", created="2017-01-20T00:00:00.000Z", definition_type="statement", - definition=stix2.StatementMarking("Copyright 2016, Example Corp") + definition=stix2.StatementMarking(**kwargs) ) assert str(marking_definition) == EXPECTED_STATEMENT_MARKING_DEFINITION @@ -180,4 +181,53 @@ def test_parse_marking_definition(data): assert gm.definition_type == "tlp" +@stix2.common.CustomMarking('x-new-marking-type', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), +]) +class NewMarking(object): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + +def test_registered_custom_marking(): + nm = NewMarking(property1='something', property2=55) + + marking_def = stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type", + definition=nm + ) + + assert marking_def.type == "marking-definition" + assert marking_def.id == "marking-definition--00000000-0000-0000-0000-000000000012" + assert marking_def.created == dt.datetime(2017, 1, 22, 0, 0, 0, tzinfo=pytz.utc) + assert marking_def.definition.property1 == "something" + assert marking_def.definition.property2 == 55 + assert marking_def.definition_type == "x-new-marking-type" + + +def test_not_registered_marking_raises_exception(): + with pytest.raises(ValueError): + @stix2.sdo.CustomObject('x-new-marking-type2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ('property2', stix2.properties.IntegerProperty()), + ]) + class NewObject2(object): + def __init__(self, property2=None, **kwargs): + if property2 and property2 < 10: + raise ValueError("'property2' is too small.") + + no = NewObject2(property1='something', property2=55) + + stix2.MarkingDefinition( + id="marking-definition--00000000-0000-0000-0000-000000000012", + created="2017-01-22T00:00:00.000Z", + definition_type="x-new-marking-type2", + definition=no + ) + + # TODO: Add other examples From 160d69b03df6e23b6b8c6d7f85df777ef1c010ae Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:32:51 -0400 Subject: [PATCH 68/82] Changes to datastores. --- stix2/sources/__init__.py | 78 ++++++++++++++++++------------------- stix2/sources/filesystem.py | 10 ++--- stix2/sources/memory.py | 10 ++--- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index d8676ca..87ff913 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -72,11 +72,11 @@ class DataStore(object): this abstract class for the specific data store. """ - def __init__(self, name="DataStore"): + def __init__(self, name="DataStore", source=None, sink=None): self.name = name - self.id = make_id() - self.source = None - self.sink = None + self.id_ = make_id() + self.source = source + self.sink = sink def get(self, stix_id): """ @@ -146,14 +146,14 @@ class DataSink(object): different sink components. Attributes: - id (str): A unique UUIDv4 to identify this DataSink. + id_ (str): A unique UUIDv4 to identify this DataSink. name (str): The descriptive name that identifies this DataSink. """ def __init__(self, name="DataSink"): self.name = name - self.id = make_id() + self.id_ = make_id() def add(self, stix_objs): """ @@ -171,17 +171,15 @@ class DataSource(object): different source components. Attributes: - id (str): A unique UUIDv4 to identify this DataSource. + id_ (str): A unique UUIDv4 to identify this DataSource. name (str): The descriptive name that identifies this DataSource. - filters (dict): A collection of filters present in this DataSource. - filter_allowed (dict): A collection of the allowed filters in this - DataSource. + filters (set): A collection of filters present in this DataSource. """ def __init__(self, name="DataSource"): self.name = name - self.id = make_id() + self.id_ = make_id() self.filters = set() def get(self, stix_id, _composite_filters=None): @@ -250,26 +248,26 @@ class DataSource(object): """Add multiple filters to the DataSource. Args: - filter (list): list of filters (dict) to add to the Data Source. + filters (list): list of filters (dict) to add to the Data Source. """ - for filter in filters: - self.add_filter(filter) + for filter_ in filters: + self.add_filter(filter_) - def add_filter(self, filter): + def add_filter(self, filter_): """Add a filter.""" # check filter field is a supported STIX 2.0 common field - if filter.field not in STIX_COMMON_FIELDS: + if filter_.field not in STIX_COMMON_FIELDS: raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") # check filter operator is supported - if filter.op not in FILTER_OPS: + if filter_.op not in FILTER_OPS: raise ValueError("Filter operation(from 'op' field) not supported") # check filter value type is supported - if type(filter.value) not in FILTER_VALUE_TYPES: + if type(filter_.value) not in FILTER_VALUE_TYPES: raise ValueError("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - self.filters.add(filter) + self.filters.add(filter_) # TODO: Do we need a remove_filter function? @@ -398,14 +396,14 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds_id, ds in iteritems(self.data_sources): data = ds.get(stix_id=stix_id, _composite_filters=list(self.filters)) - all_data.extend(data) + all_data.append(data) # remove duplicate versions if len(all_data) > 0: all_data = self.deduplicate(all_data) # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0] return stix_obj @@ -430,10 +428,14 @@ class CompositeDataSource(DataSource): """ all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) # retrieve STIX objects from all configured data sources for ds_id, ds in iteritems(self.data_sources): - data = ds.all_versions(stix_id=stix_id, _composite_filters=list(self.filters)) + data = ds.all_versions(stix_id=stix_id, _composite_filters=list(all_filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -463,11 +465,15 @@ class CompositeDataSource(DataSource): query = [] all_data = [] + all_filters = self.filters + + if _composite_filters: + all_filters = set(self.filters).update(_composite_filters) # federate query to all attached data sources, # pass composite filters to id for ds_id, ds in iteritems(self.data_sources): - data = ds.query(query=query, _composite_filters=list(self.filters)) + data = ds.query(query=query, _composite_filters=list(all_filters)) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 @@ -486,14 +492,14 @@ class CompositeDataSource(DataSource): """ for ds in data_sources: - if issubclass(ds, DataSource): - if self.data_sources[ds['id']] in self.data_sources.keys(): + if issubclass(ds.__class__, DataSource): + if ds.id_ in self.data_sources: # data source already attached to Composite Data Source continue # add data source to Composite Data Source # (its id will be its key identifier) - self.data_sources[ds['id']] = ds + self.data_sources[ds.id_] = ds else: # the Data Source object is not a proper subclass # of DataSource Abstract Class @@ -506,27 +512,21 @@ class CompositeDataSource(DataSource): """Remove/detach Data Source from the Composite Data Source instance Args: - data_source_ids (list): a list of Data Source - id's(which are strings) + data_source_ids (list): a list of Data Source identifiers. """ - for id_ in data_source_ids: - try: - if self.data_sources[id_]: - del self.data_sources[id_] - except KeyError: - # Data Source 'id' was not found in CompositeDataSource's - # list of data sources - pass + if id_ in self.data_sources: + del self.data_sources[id_] + else: + raise ValueError("DataSource 'id' not found in CompositeDataSource collection.") return - @property - def data_sources(self): + def get_all_data_sources(self): """Return all attached Data Sources """ - return copy.deepcopy(self.data_sources.values()) + return self.data_sources.values() class STIXCommonPropertyFilters(object): diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index e16ee0c..61e7c88 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -19,7 +19,7 @@ from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemStore"): + def __init__(self, name="FileSystemStore", stix_dir="stix_data"): super(FileSystemStore, self).__init__(name=name) self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) @@ -28,7 +28,7 @@ class FileSystemStore(DataStore): class FileSystemSink(DataSink): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemSink"): + def __init__(self, name="FileSystemSink", stix_dir="stix_data"): super(FileSystemSink, self).__init__(name=name) self.stix_dir = os.path.abspath(stix_dir) @@ -58,7 +58,7 @@ class FileSystemSink(DataSink): class FileSystemSource(DataSource): """ """ - def __init__(self, stix_dir="stix_data", name="FileSystemSource"): + def __init__(self, name="FileSystemSource", stix_dir="stix_data"): super(FileSystemSource, self).__init__(name=name) self.stix_dir = os.path.abspath(stix_dir) @@ -71,8 +71,8 @@ class FileSystemSource(DataSource): return self.stix_dir @stix_dir.setter - def stix_dir(self, dir): - self.stix_dir = dir + def stix_dir(self, dir_): + self.stix_dir = dir_ def get(self, stix_id, _composite_filters=None): """ diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2f45d68..6460ce3 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -29,7 +29,7 @@ from stix2validator import validate_string class MemoryStore(DataStore): """ """ - def __init__(self, stix_data=None, name="MemoryStore"): + def __init__(self, name="MemoryStore", stix_data=None): """ Notes: It doesn't make sense to create a MemoryStore by passing @@ -75,7 +75,7 @@ class MemoryStore(DataStore): class MemorySink(DataSink): """ """ - def __init__(self, stix_data=None, name="MemorySink", _store=False): + def __init__(self, name="MemorySink", stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -150,7 +150,7 @@ class MemorySink(DataSink): class MemorySource(DataSource): - def __init__(self, stix_data=None, name="MemorySource", _store=False): + def __init__(self, name="MemorySource", stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -177,8 +177,8 @@ class MemorySource(DataSource): for stix_obj in stix_data["objects"]: self.data[stix_obj["id"]] = stix_obj else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) + print("Error: json data passed to MemorySource() was found to not be validated by STIX 2 Validator") + print(r.as_dict()) self.data = {} elif type(stix_data) == list: # STIX objects are in a list From aea076103cdd0b65a89e7708a8527c71d3ba2c93 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 14:33:12 -0400 Subject: [PATCH 69/82] Update tests to improve coverage. --- stix2/test/test_data_sources.py | 241 ++++++++++++++++++++++---------- 1 file changed, 170 insertions(+), 71 deletions(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0095802..df70878 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,19 +1,15 @@ import pytest from taxii2client import Collection -from stix2.sources import DataSource, Filter, taxii +from stix2.sources import CompositeDataSource, DataSink, DataSource, DataStore, Filter, make_id, taxii +from stix2.sources.memory import MemorySource COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' class MockTAXIIClient(object): """Mock for taxii2_client.TAXIIClient""" - - def get(self): - return {} - - def post(self): - return {} + pass @pytest.fixture @@ -21,6 +17,18 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) +def test_ds_smoke(): + ds1 = DataSource() + ds2 = DataSink() + ds3 = DataStore(source=ds1, sink=ds2) + + with pytest.raises(NotImplementedError): + ds3.add(None) + ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + + def test_ds_taxii(collection): ds = taxii.TAXIICollectionSource(collection) assert ds.name == 'TAXIICollectionSource' @@ -104,6 +112,8 @@ def test_add_get_remove_filter(): assert len(ds.filters) == 2 + ds.add_filters(valid_filters) + def test_apply_common_filters(): stix_objs = [ @@ -160,73 +170,112 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] +STIX_OBJS1 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + +STIX_OBJS2 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + def test_deduplicate(): - stix_objs = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } - ] - ds = DataSource() - unique = ds.deduplicate(stix_objs) + unique = ds.deduplicate(STIX_OBJS1) # Only 3 objects are unique # 2 id's vary @@ -243,6 +292,56 @@ def test_deduplicate(): assert "2017-01-27T13:49:53.936Z" in mods +def test_add_remove_composite_datasource(): + cds = CompositeDataSource() + ds1 = DataSource() + ds2 = DataSource() + ds3 = DataSink() + + cds.add_data_source([ds1, ds2, ds1, ds3]) + + assert len(cds.get_all_data_sources()) == 2 + + cds.remove_data_source([ds1.id_, ds2.id_]) + + assert len(cds.get_all_data_sources()) == 0 + + with pytest.raises(ValueError): + cds.remove_data_source([ds3.id_]) + + +def test_composite_datasource_operations(): + BUNDLE1 = dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS1, + spec_version="2.0", + type="bundle") + cds = CompositeDataSource() + ds1 = MemorySource(stix_data=BUNDLE1) + ds2 = MemorySource(stix_data=STIX_OBJS2) + + cds.add_data_source([ds1, ds2]) + + indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + # In STIX_OBJS2 changed the 'modified' property to a later time... + assert len(indicators) == 2 + + indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + query = [ + Filter("type", "=", "indicator") + ] + + results = cds.query(query) + + # STIX_OBJS2 has indicator with later time, one with different id, one with + # original time in STIX_OBJS1 + assert len(results) == 3 + # def test_data_source_file(): # ds = file.FileDataSource() # From 415c53066f99d2d063d5f7d21ec717bd1c223f87 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 28 Aug 2017 15:19:55 -0400 Subject: [PATCH 70/82] Code coverage changes, fix some tests. Add stix2-validator dependency. --- setup.py | 1 + stix2/__init__.py | 7 +++---- stix2/properties.py | 1 - stix2/sources/__init__.py | 1 - stix2/sources/memory.py | 3 ++- stix2/test/test_custom.py | 22 +--------------------- stix2/test/test_data_sources.py | 10 +++++++++- stix2/test/test_markings.py | 13 +++++++++---- 8 files changed, 25 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index d9b1edc..3687e57 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'simplejson', 'six', 'stix2-patterns', + 'stix2-validator', 'taxii2-client', ], ) diff --git a/stix2/__init__.py b/stix2/__init__.py index 76ca696..6e89531 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,10 +3,9 @@ # flake8: noqa from . import exceptions -from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, - CustomMarking, ExternalReference, GranularMarking, - KillChainPhase, MarkingDefinition, StatementMarking, - TLPMarking) +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) from .core import Bundle, _register_type, parse from .environment import ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, diff --git a/stix2/properties.py b/stix2/properties.py index a04294e..6406ad4 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -6,7 +6,6 @@ import re import uuid from six import string_types, text_type - from stix2patterns.validator import run_validator from .base import _STIXBase diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 87ff913..424fc3a 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -17,7 +17,6 @@ Notes: """ import collections -import copy import uuid from six import iteritems diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 6460ce3..28d929d 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -21,9 +21,10 @@ Notes: import json import os +from stix2validator import validate_string + from stix2 import Bundle from stix2.sources import DataSink, DataSource, DataStore, Filter -from stix2validator import validate_string class MemoryStore(DataStore): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 2b08883..0ca6fbd 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -135,32 +135,12 @@ def test_custom_no_properties_raises_exception(): class NewObject1(object): pass - NewObject1() - - @stix2.sdo.CustomObject('x-new-object-type', ("a", 0)) - class NewObject2(object): - pass - - NewObject2() - - @stix2.observables.CustomObservable('x-new-object-type') - class NewObject3(object): - pass - - NewObject3() + with pytest.raises(ValueError): @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) class NewObject4(object): pass - NewObject4() - - @stix2.common.CustomMarking('x-new-marking-type') - class NewObject5(object): - pass - - NewObject5() - def test_parse_custom_observable_object(): nt_string = """{ diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index df70878..10e4cc7 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,8 @@ import pytest from taxii2client import Collection -from stix2.sources import CompositeDataSource, DataSink, DataSource, DataStore, Filter, make_id, taxii +from stix2.sources import (CompositeDataSource, DataSink, DataSource, + DataStore, Filter, make_id, taxii) from stix2.sources.memory import MemorySource COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -24,8 +25,14 @@ def test_ds_smoke(): with pytest.raises(NotImplementedError): ds3.add(None) + + with pytest.raises(NotImplementedError): ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + + with pytest.raises(NotImplementedError): ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) @@ -170,6 +177,7 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] + STIX_OBJS1 = [ { "created": "2017-01-27T13:49:53.935Z", diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 4e39c54..c9d9273 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -187,8 +187,7 @@ def test_parse_marking_definition(data): ]) class NewMarking(object): def __init__(self, property2=None, **kwargs): - if property2 and property2 < 10: - raise ValueError("'property2' is too small.") + return def test_registered_custom_marking(): @@ -217,8 +216,7 @@ def test_not_registered_marking_raises_exception(): ]) class NewObject2(object): def __init__(self, property2=None, **kwargs): - if property2 and property2 < 10: - raise ValueError("'property2' is too small.") + return no = NewObject2(property1='something', property2=55) @@ -230,4 +228,11 @@ def test_not_registered_marking_raises_exception(): ) +def test_bad_marking_construction(): + with pytest.raises(ValueError): + @stix2.sdo.CustomObject('x-new-marking-type2', ("a", "b")) + class NewObject3(object): + pass + + # TODO: Add other examples From b1ac24d46ed2a974e1d7a67175a414dc4210535c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 29 Aug 2017 15:08:26 -0400 Subject: [PATCH 71/82] Minor changes. --- .isort.cfg | 2 +- stix2/base.py | 6 +++--- stix2/exceptions.py | 2 +- stix2/utils.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index e8d95f8..622e7a5 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=dateutil,pytest,pytz,six,requests,taxii2client +known_third_party=ordereddict,dateutil,pytest,pytz,requests,simplejson,six,stix2patterns,stix2validator,taxii2client known_first_party=stix2 not_skip=__init__.py force_sort_within_sections=1 diff --git a/stix2/base.py b/stix2/base.py index a302eb5..60dd78a 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -38,7 +38,7 @@ def get_required_properties(properties): class _STIXBase(collections.Mapping): """Base class for STIX object types""" - def _object_properties(self): + def object_properties(self): return list(self._properties.keys()) def _check_property(self, prop_name, prop, kwargs): @@ -146,7 +146,7 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - properties = self._object_properties() + properties = self.object_properties() def sort_by(element): return find_property_index(self, properties, element) @@ -157,7 +157,7 @@ class _STIXBase(collections.Mapping): separators=(",", ": ")) def __repr__(self): - props = [(k, self[k]) for k in self._object_properties() if self.get(k)] + props = [(k, self[k]) for k in self.object_properties() if self.get(k)] return "{0}({1})".format(self.__class__.__name__, ", ".join(["{0!s}={1!r}".format(k, v) for k, v in props])) diff --git a/stix2/exceptions.py b/stix2/exceptions.py index ef47dd0..5a9e7b2 100644 --- a/stix2/exceptions.py +++ b/stix2/exceptions.py @@ -121,7 +121,7 @@ class DependentPropertiesError(STIXError, TypeError): def __str__(self): msg = "The property dependencies for {0}: ({1}) are not met." return msg.format(self.cls.__name__, - ", ".join(x for x in self.dependencies)) + ", ".join(name for x in self.dependencies for name in x)) class AtLeastOnePropertyError(STIXError, TypeError): diff --git a/stix2/utils.py b/stix2/utils.py index 9e0f33a..de481fc 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -133,7 +133,7 @@ def find_property_index(obj, properties, tuple_to_find): for item in pv: if isinstance(item, _STIXBase): val = find_property_index(item, - item._object_properties(), + item.object_properties(), tuple_to_find) if val is not None: return val @@ -146,7 +146,7 @@ def find_property_index(obj, properties, tuple_to_find): for item in pv.values(): if isinstance(item, _STIXBase): val = find_property_index(item, - item._object_properties(), + item.object_properties(), tuple_to_find) if val is not None: return val From 8ca2c3390b775a99db36f67d8c97bb315140f962 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 29 Aug 2017 15:15:32 -0400 Subject: [PATCH 72/82] Minor changes to DataSource.apply_common_filters(). Improve overall code coverage. --- stix2/sources/__init__.py | 44 +++-- stix2/test/test_bundle.py | 16 ++ stix2/test/test_custom.py | 4 +- stix2/test/test_data_sources.py | 299 ++++++++++++++++++++----------- stix2/test/test_markings.py | 14 +- stix2/test/test_observed_data.py | 16 +- stix2/test/test_properties.py | 17 +- stix2/test/test_versioning.py | 11 ++ 8 files changed, 288 insertions(+), 133 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 424fc3a..6360fde 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -290,28 +290,34 @@ class DataSource(object): for stix_obj in stix_objs: clean = True for filter_ in query: - - # skip filter as filter was identified (when added) as - # not a common filter - if filter_.field not in STIX_COMMON_FIELDS: - continue - - # check filter "field" is in STIX object - if cant be applied - # due to STIX object, STIX object is discarded (i.e. did not - # make it through the filter) - if filter_.field not in stix_obj.keys(): - clean = False - break try: - match = getattr(STIXCommonPropertyFilters, filter_.field)(filter_, stix_obj) + # skip filter as filter was identified (when added) as + # not a common filter + if filter_.field not in STIX_COMMON_FIELDS: + raise Exception("Error, field: {0} is not supported for filtering on.".format(filter_.field)) + + # For properties like granular_markings and external_references + # need to break the first property from the string. + if "." in filter_.field: + field = filter_.field.split(".")[0] + else: + field = filter_.field + + # check filter "field" is in STIX object - if cant be + # applied due to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + if field not in stix_obj.keys(): + clean = False + break + + match = getattr(STIXCommonPropertyFilters, field)(filter_, stix_obj) if not match: clean = False break elif match == -1: - # error, filter operator not supported for specified field: - pass + raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) except Exception as e: - print(e) + raise ValueError(e) # if object unmarked after all filters, add it if clean: @@ -646,11 +652,11 @@ class STIXCommonPropertyFilters(object): @classmethod def modified(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["created"]) + return cls._timestamp(filter_, stix_obj["modified"]) @classmethod - def object_markings_ref(cls, filter_, stix_obj): - for marking_id in stix_obj["object_market_refs"]: + def object_marking_refs(cls, filter_, stix_obj): + for marking_id in stix_obj["object_marking_refs"]: r = cls._id(filter_, marking_id) if r: return r diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b5c1da5..6f56d85 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -128,3 +128,19 @@ def test_parse_bundle(): assert bundle.objects[0].type == 'indicator' assert bundle.objects[1].type == 'malware' assert bundle.objects[2].type == 'relationship' + + +def test_parse_unknown_type(): + unknown = { + "type": "other", + "id": "other--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created": "2016-04-06T20:03:00Z", + "modified": "2016-04-06T20:03:00Z", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "description": "Campaign by Green Group against a series of targets in the financial services sector.", + "name": "Green Group Attacks Against Finance", + } + + with pytest.raises(stix2.exceptions.ParseError) as excinfo: + stix2.parse(unknown) + assert str(excinfo.value) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator." diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 0ca6fbd..f25044e 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -135,10 +135,12 @@ def test_custom_no_properties_raises_exception(): class NewObject1(object): pass + +def test_custom_wrong_properties_arg_raises_exception(): with pytest.raises(ValueError): @stix2.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) - class NewObject4(object): + class NewObject2(object): pass diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 10e4cc7..ee37825 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -18,6 +18,109 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) +STIX_OBJS1 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + +STIX_OBJS2 = [ + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + }, + { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } +] + + def test_ds_smoke(): ds1 = DataSource() ds2 = DataSink() @@ -149,9 +252,21 @@ def test_apply_common_filters(): }, { "created": "2014-05-08T09:00:00.000Z", + "granular_markings": [ + { + "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + "selectors": [ + "relationship_type" + ] + } + ], "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", "modified": "2014-05-08T09:00:00.000Z", + "object_marking_refs": [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + ], "relationship_type": "indicates", + "revoked": True, "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", "type": "relationship" @@ -162,6 +277,13 @@ def test_apply_common_filters(): Filter("type", "!=", "relationship"), Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), Filter("labels", "in", "remote-access-trojan"), + Filter("created", ">", "2015-01-01T01:00:00.000Z"), + Filter("revoked", "=", True), + Filter("revoked", "!=", True), + Filter("revoked", "?", False), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "relationship_type"), + Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), ] ds = DataSource() @@ -177,108 +299,85 @@ def test_apply_common_filters(): resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] + resp = ds.apply_common_filters(stix_objs, [filters[3]]) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 1 -STIX_OBJS1 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] + resp = ds.apply_common_filters(stix_objs, [filters[4]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 -STIX_OBJS2 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-31T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] + # Note that if 'revoked' property is not present in object. + # Currently we can't use such an expression to filter for... + resp = ds.apply_common_filters(stix_objs, [filters[5]]) + assert len(resp) == 0 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(stix_objs, [filters[6]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(filters[6].op, + filters[6].field) + + resp = ds.apply_common_filters(stix_objs, [filters[7]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + resp = ds.apply_common_filters(stix_objs, [filters[8], filters[9]]) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # These are used with STIX_OBJS2 + more_filters = [ + Filter("modified", "<", "2017-01-28T13:49:53.935Z"), + Filter("modified", ">", "2017-01-28T13:49:53.935Z"), + Filter("modified", ">=", "2017-01-27T13:49:53.935Z"), + Filter("modified", "<=", "2017-01-27T13:49:53.935Z"), + Filter("modified", "?", "2017-01-27T13:49:53.935Z"), + Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), + Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), + Filter("notacommonproperty", "=", "bar"), + ] + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[0]]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[1]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[2]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[3]]) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[4]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(more_filters[4].op, + more_filters[4].field) + + resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[5]]) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[6]]) + + assert str(excinfo.value) == ("Error, filter operator: {0} not supported " + "for specified field: {1}").format(more_filters[6].op, + more_filters[6].field) + + with pytest.raises(ValueError) as excinfo: + ds.apply_common_filters(STIX_OBJS2, [more_filters[7]]) + + assert str(excinfo.value) == ("Error, field: {0} is not supported for " + "filtering on.".format(more_filters[7].field)) def test_deduplicate(): diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index c9d9273..9fe51fb 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -209,7 +209,8 @@ def test_registered_custom_marking(): def test_not_registered_marking_raises_exception(): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: + # Used custom object on purpose to demonstrate a not-registered marking @stix2.sdo.CustomObject('x-new-marking-type2', [ ('property1', stix2.properties.StringProperty(required=True)), ('property2', stix2.properties.IntegerProperty()), @@ -227,12 +228,17 @@ def test_not_registered_marking_raises_exception(): definition=no ) + assert str(excinfo.value) == "definition_type must be a valid marking type" -def test_bad_marking_construction(): - with pytest.raises(ValueError): - @stix2.sdo.CustomObject('x-new-marking-type2', ("a", "b")) + +def test_marking_wrong_type_construction(): + with pytest.raises(ValueError) as excinfo: + # Test passing wrong type for properties. + @stix2.CustomMarking('x-new-marking-type2', ("a", "b")) class NewObject3(object): pass + assert str(excinfo.value) == "Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]" + # TODO: Add other examples diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py index dd8f042..fdc66ee 100644 --- a/stix2/test/test_observed_data.py +++ b/stix2/test/test_observed_data.py @@ -8,6 +8,8 @@ import stix2 from .constants import OBSERVED_DATA_ID +OBJECTS_REGEX = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL) + EXPECTED = """{ "type": "observed-data", @@ -173,7 +175,7 @@ def test_parse_observed_data(data): }""", ]) def test_parse_artifact_valid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "artifact" @@ -194,7 +196,7 @@ def test_parse_artifact_valid(data): }""", ]) def test_parse_artifact_invalid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) with pytest.raises(ValueError): stix2.parse(odata_str) @@ -204,6 +206,7 @@ def test_artifact_example_dependency_error(): stix2.Artifact(url="http://example.com/sirvizio.exe") assert excinfo.value.dependencies == [("hashes", "url")] + assert str(excinfo.value) == "The property dependencies for Artifact: (hashes, url) are not met." @pytest.mark.parametrize("data", [ @@ -215,7 +218,7 @@ def test_artifact_example_dependency_error(): }""", ]) def test_parse_autonomous_system_valid(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["0"].type == "autonomous-system" assert odata.objects["0"].number == 15139 @@ -358,7 +361,7 @@ def test_parse_email_message_not_multipart(data): }""", ]) def test_parse_file_archive(data): - odata_str = re.compile('\"objects\": {(?:.*?)(?:(?:[^{]*?)|(?:{[^{]*?}))*}', re.DOTALL).sub('"objects": { %s }' % data, EXPECTED) + odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED) odata = stix2.parse(odata_str) assert odata.objects["3"].extensions['archive-ext'].version == "5.0" @@ -555,6 +558,7 @@ def test_artifact_mutual_exclusion_error(): assert excinfo.value.cls == stix2.Artifact assert excinfo.value.properties == ["payload_bin", "url"] + assert str(excinfo.value) == "The (payload_bin, url) properties for Artifact are mutually exclusive." def test_directory_example(): @@ -925,6 +929,10 @@ def test_process_example_empty_error(): properties_of_process = list(stix2.Process._properties.keys()) properties_of_process.remove("type") assert excinfo.value.properties == sorted(properties_of_process) + msg = "At least one of the ({1}) properties for {0} must be populated." + msg = msg.format(stix2.Process.__name__, + ", ".join(sorted(properties_of_process))) + assert str(excinfo.value) == msg def test_process_example_empty_with_extensions(): diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 01daebf..db1c143 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -206,15 +206,22 @@ def test_dictionary_property_valid(d): @pytest.mark.parametrize("d", [ - {'a': 'something'}, - {'a'*300: 'something'}, - {'Hey!': 'something'}, + [{'a': 'something'}, "Invalid dictionary key a: (shorter than 3 characters)."], + [{'a'*300: 'something'}, "Invalid dictionary key aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaa: (longer than 256 characters)."], + [{'Hey!': 'something'}, "Invalid dictionary key Hey!: (contains characters other thanlowercase a-z, " + "uppercase A-Z, numerals 0-9, hyphen (-), or underscore (_))."], ]) def test_dictionary_property_invalid(d): dict_prop = DictionaryProperty() - with pytest.raises(DictionaryKeyError): - dict_prop.clean(d) + with pytest.raises(DictionaryKeyError) as excinfo: + dict_prop.clean(d[0]) + + assert str(excinfo.value) == d[1] @pytest.mark.parametrize("value", [ diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 281ae71..8506d16 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -128,6 +128,11 @@ def test_versioning_error_bad_modified_value(): assert excinfo.value.prop_name == "modified" assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + msg = "Invalid value for {0} '{1}': {2}" + msg = msg.format(stix2.Campaign.__name__, "modified", + "The new modified datetime cannot be before the current modified datatime.") + assert str(excinfo.value) == msg + def test_versioning_error_usetting_required_property(): campaign_v1 = stix2.Campaign( @@ -145,6 +150,10 @@ def test_versioning_error_usetting_required_property(): assert excinfo.value.cls == stix2.Campaign assert excinfo.value.properties == ["name"] + msg = "No values for required properties for {0}: ({1})." + msg = msg.format(stix2.Campaign.__name__, "name") + assert str(excinfo.value) == msg + def test_versioning_error_new_version_of_revoked(): campaign_v1 = stix2.Campaign( @@ -162,6 +171,7 @@ def test_versioning_error_new_version_of_revoked(): campaign_v2.new_version(name="barney") assert excinfo.value.called_by == "new_version" + assert str(excinfo.value) == "Cannot create a new version of a revoked object." def test_versioning_error_revoke_of_revoked(): @@ -180,3 +190,4 @@ def test_versioning_error_revoke_of_revoked(): campaign_v2.revoke() assert excinfo.value.called_by == "revoke" + assert str(excinfo.value) == "Cannot revoke an already revoked object." From 22c749d0dfb2a07bc003d540ce47c1e06d935f03 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 30 Aug 2017 11:18:11 -0400 Subject: [PATCH 73/82] filter changes --- stix2/sources/__init__.py | 180 +--------------------------- stix2/sources/filesystem.py | 23 +--- stix2/sources/filters.py | 227 ++++++++++++++++++++++++++++++++++++ stix2/sources/memory.py | 23 +--- stix2/sources/taxii.py | 20 +--- 5 files changed, 240 insertions(+), 233 deletions(-) create mode 100644 stix2/sources/filters.py diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index d8676ca..ba5528b 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -16,56 +16,19 @@ Notes: """ -import collections import copy import uuid from six import iteritems - -class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): - __slots__ = () - - def __new__(cls, field, op, value): - # If value is a list, convert it to a tuple so it is hashable. - if isinstance(value, list): - value = tuple(value) - self = super(Filter, cls).__new__(cls, field, op, value) - return self +from filters import (FILTER_OPS, FILTER_VALUE_TYPES, STIX_COMMON_FIELDS, + STIX_COMMON_FILTERS_MAP) def make_id(): return str(uuid.uuid4()) -# Currently, only STIX 2.0 common SDO fields (that are not complex objects) -# are supported for filtering on -STIX_COMMON_FIELDS = [ - "created", - "created_by_ref", - "external_references.source_name", - "external_references.description", - "external_references.url", - "external_references.hashes", - "external_references.external_id", - "granular_markings.marking_ref", - "granular_markings.selectors", - "id", - "labels", - "modified", - "object_marking_refs", - "revoked", - "type", - "granular_markings" -] - -# Supported filter operations -FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] - -# Supported filter value types -FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] - - class DataStore(object): """ An implementer will create a concrete subclass from @@ -306,7 +269,7 @@ class DataSource(object): clean = False break try: - match = getattr(STIXCommonPropertyFilters, filter_.field)(filter_, stix_obj) + match = STIX_COMMON_FILTERS_MAP[filter_.field](filter_, stix_obj) if not match: clean = False break @@ -527,140 +490,3 @@ class CompositeDataSource(DataSource): """ return copy.deepcopy(self.data_sources.values()) - - -class STIXCommonPropertyFilters(object): - """ - """ - @classmethod - def _all(cls, filter_, stix_obj_field): - """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - elif filter_.op == "in": - return stix_obj_field in filter_.value - elif filter_.op == ">": - return stix_obj_field > filter_.value - elif filter_.op == "<": - return stix_obj_field < filter_.value - elif filter_.op == ">=": - return stix_obj_field >= filter_.value - elif filter_.op == "<=": - return stix_obj_field <= filter_.value - else: - return -1 - - @classmethod - def _id(cls, filter_, stix_obj_id): - """base filter types""" - if filter_.op == "=": - return stix_obj_id == filter_.value - elif filter_.op == "!=": - return stix_obj_id != filter_.value - else: - return -1 - - @classmethod - def _boolean(cls, filter_, stix_obj_field): - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - else: - return -1 - - @classmethod - def _string(cls, filter_, stix_obj_field): - return cls._all(filter_, stix_obj_field) - - @classmethod - def _timestamp(cls, filter_, stix_obj_timestamp): - return cls._all(filter_, stix_obj_timestamp) - - # STIX 2.0 Common Property filters - @classmethod - def created(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["created"]) - - @classmethod - def created_by_ref(cls, filter_, stix_obj): - return cls._id(filter_, stix_obj["created_by_ref"]) - - @classmethod - def external_references(cls, filter_, stix_obj): - """ - STIX object's can have a list of external references - - external_references properties: - external_references.source_name (string) - external_references.description (string) - external_references.url (string) - external_references.hashes (hash, but for filtering purposes, a string) - external_references.external_id (string) - - """ - for er in stix_obj["external_references"]: - # grab er property name from filter field - filter_field = filter_.field.split(".")[1] - r = cls._string(filter_, er[filter_field]) - if r: - return r - return False - - @classmethod - def granular_markings(cls, filter_, stix_obj): - """ - STIX object's can have a list of granular marking references - - granular_markings properties: - granular_markings.marking_ref (id) - granular_markings.selectors (string) - - """ - for gm in stix_obj["granular_markings"]: - # grab gm property name from filter field - filter_field = filter_.field.split(".")[1] - - if filter_field == "marking_ref": - return cls._id(filter_, gm[filter_field]) - - elif filter_field == "selectors": - for selector in gm[filter_field]: - r = cls._string(filter_, selector) - if r: - return r - return False - - @classmethod - def id(cls, filter_, stix_obj): - return cls._id(filter_, stix_obj["id"]) - - @classmethod - def labels(cls, filter_, stix_obj): - for label in stix_obj["labels"]: - r = cls._string(filter_, label) - if r: - return r - return False - - @classmethod - def modified(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["created"]) - - @classmethod - def object_markings_ref(cls, filter_, stix_obj): - for marking_id in stix_obj["object_market_refs"]: - r = cls._id(filter_, marking_id) - if r: - return r - return False - - @classmethod - def revoked(cls, filter_, stix_obj): - return cls._boolean(filter_, stix_obj["revoked"]) - - @classmethod - def type(cls, filter_, stix_obj): - return cls._string(filter_, stix_obj["type"]) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 0613ac0..c8ad0b0 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -13,7 +13,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): @@ -77,13 +77,7 @@ class FileSystemSource(DataSource): def get(self, stix_id, _composite_filters=None): """ """ - query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } - ] + query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -95,21 +89,10 @@ class FileSystemSource(DataSource): """ Notes: Since FileSystem sources/sinks don't handle multiple versions - of a STIX object, this operation is futile. Pass call to get(). - (Approved by G.B.) + of a STIX object, this operation is unnecessary. Pass call to get(). """ - # query = [ - # { - # "field": "id", - # "op": "=", - # "value": stix_id - # } - # ] - - # all_data = self.query(query=query, _composite_filters=_composite_filters) - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py new file mode 100644 index 0000000..baa32c8 --- /dev/null +++ b/stix2/sources/filters.py @@ -0,0 +1,227 @@ +""" +Filters for Python STIX 2.0 DataSources, DataSinks, DataStores + +Classes: + Filter + +TODO: The script at the bottom of the module works (to capture +all the callable filter methods), however it causes this module +to be imported by itself twice. Not sure how big of deal that is, +or if cleaner solution possible. +""" + +import collections +import types + +import filters + +# Currently, only STIX 2.0 common SDO fields (that are not complex objects) +# are supported for filtering on +STIX_COMMON_FIELDS = [ + "created", + "created_by_ref", + "external_references.source_name", + "external_references.description", + "external_references.url", + "external_references.hashes", + "external_references.external_id", + "granular_markings.marking_ref", + "granular_markings.selectors", + "id", + "labels", + "modified", + "object_marking_refs", + "revoked", + "type", + "granular_markings" +] + +# Supported filter operations +FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] + +# Supported filter value types +FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] + +# filter lookup map - STIX 2 common fields -> filter method +STIX_COMMON_FILTERS_MAP = {} + + +class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): + __slots__ = () + + def __new__(cls, field, op, value): + # If value is a list, convert it to a tuple so it is hashable. + if isinstance(value, list): + value = tuple(value) + self = super(Filter, cls).__new__(cls, field, op, value) + return self + + +# primitive type filters + +def _all_filter(filter_, stix_obj_field): + """all filter operations (for filters whose value type can be applied to any operation type)""" + if filter_.op == "=": + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value + elif filter_.op == "in": + return stix_obj_field in filter_.value + elif filter_.op == ">": + return stix_obj_field > filter_.value + elif filter_.op == "<": + return stix_obj_field < filter_.value + elif filter_.op == ">=": + return stix_obj_field >= filter_.value + elif filter_.op == "<=": + return stix_obj_field <= filter_.value + else: + return -1 + + +def _id_filter(filter_, stix_obj_id): + """base filter types""" + if filter_.op == "=": + return stix_obj_id == filter_.value + elif filter_.op == "!=": + return stix_obj_id != filter_.value + else: + return -1 + + +def _boolean_filter(filter_, stix_obj_field): + if filter_.op == "=": + return stix_obj_field == filter_.value + elif filter_.op == "!=": + return stix_obj_field != filter_.value + else: + return -1 + + +def _string_filter(filter_, stix_obj_field): + return _all_filter(filter_, stix_obj_field) + + +def _timestamp_filter(filter_, stix_obj_timestamp): + return _all_filter(filter_, stix_obj_timestamp) + +# STIX 2.0 Common Property filters +# The naming of these functions is important as +# they are used to index a mapping dictionary from +# STIX common field names to these filter functions. +# +# REQUIRED naming scheme: +# "check__filter" + + +def check_created_filter(filter_, stix_obj): + return _timestamp_filter(filter_, stix_obj["created"]) + + +def check_created_by_ref_filter(filter_, stix_obj): + return _id_filter(filter_, stix_obj["created_by_ref"]) + + +def check_external_references_filter(filter_, stix_obj): + """ + STIX object's can have a list of external references + + external_references properties: + external_references.source_name (string) + external_references.description (string) + external_references.url (string) + external_references.hashes (hash, but for filtering purposes, a string) + external_references.external_id (string) + + """ + for er in stix_obj["external_references"]: + # grab er property name from filter field + filter_field = filter_.field.split(".")[1] + r = _string_filter(filter_, er[filter_field]) + if r: + return r + return False + + +def check_granular_markings_filter(filter_, stix_obj): + """ + STIX object's can have a list of granular marking references + + granular_markings properties: + granular_markings.marking_ref (id) + granular_markings.selectors (string) + + """ + for gm in stix_obj["granular_markings"]: + # grab gm property name from filter field + filter_field = filter_.field.split(".")[1] + + if filter_field == "marking_ref": + return _id_filter(filter_, gm[filter_field]) + + elif filter_field == "selectors": + for selector in gm[filter_field]: + r = _string_filter(filter_, selector) + if r: + return r + return False + + +def check_id_filter(filter_, stix_obj): + return _id_filter(filter_, stix_obj["id"]) + + +def check_labels_filter(filter_, stix_obj): + for label in stix_obj["labels"]: + r = _string_filter(filter_, label) + if r: + return r + return False + + +def check_modified_filter(filter_, stix_obj): + return _timestamp_filter(filter_, stix_obj["created"]) + + +def check_object_markings_ref_filter(filter_, stix_obj): + for marking_id in stix_obj["object_market_refs"]: + r = _id_filter(filter_, marking_id) + if r: + return r + return False + + +def check_revoked_filter(filter_, stix_obj): + return _boolean_filter(filter_, stix_obj["revoked"]) + + +def check_type_filter(filter_, stix_obj): + return _string_filter(filter_, stix_obj["type"]) + + +# script to collect STIX common field filter +# functions and create mapping to them + +""" +MK: I want to build the filter name -> filter function dictionary +dynamically whenever it is imported. By enumerating the functions +in this module, extracting the "check*" functions and making +pointers to them. But having issues getting an interable of the +modules entities. globals() works but returns an active dictionary +so iterating over it is a no go +""" + +for entity in dir(filters): + if "check_" in str(entity) and isinstance(filters.__dict__.get(entity), types.FunctionType): + field_name = entity.split("_")[1].split("_")[0] + STIX_COMMON_FILTERS_MAP[field_name] = filters.__dict__.get(entity) + +# Tried this to, didnt work ############## +""" +import sys +for entity in dir(sys.modules[__name__]): + print(entity) + if "check_" in str(entity) and type(entity) == "function": + print(sys.modules[__name__].__dict__.get(entity)) + STIX_COMMON_FILTERS_MAP[str(entity)] = sys.modules[__name__].__dict__.get(entity) +""" diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 24f3c1f..bf87517 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -22,7 +22,7 @@ import json import os from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources import DataSink, DataSource, DataStore, Filter from stix2validator import validate_string @@ -204,13 +204,7 @@ class MemorySource(DataSource): return stix_obj # if there are filters from the composite level, process full query - query = [ - { - "field": "id", - "op": "=", - "value": stix_id - } - ] + query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -223,21 +217,10 @@ class MemorySource(DataSource): """ Notes: Since Memory sources/sinks don't handle multiple versions of a - STIX object, this operation is futile. Translate call to get(). - (Approved by G.B.) + STIX object, this operation is unnecessary. Translate call to get(). """ - # query = [ - # { - # "field": "id", - # "op": "=", - # "value": stix_id - # } - # ] - - # all_data = self.query(query=query, _composite_filters=_composite_filters) - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 47ad8ed..b9dc8c4 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -12,7 +12,7 @@ TODO: Test everything import json -from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources import DataSink, DataSource, DataStore, Filter, make_id TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -89,16 +89,8 @@ class TAXIICollectionSource(DataSource): """ # make query in TAXII query format since 'id' is TAXII field query = [ - { - "field": "match[id]", - "op": "=", - "value": stix_id - }, - { - "field": "match[version]", - "op": "=", - "value": "all" - } + Filter("match[id]", "=", stix_id), + Filter("match[version]", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -138,11 +130,7 @@ class TAXIICollectionSource(DataSource): For instance - "?match[type]=indicator,sighting" should be in a query dict as follows: - { - "field": "type" - "op": "=", - "value": "indicator,sighting" - } + Filter("type", "=", "indicator,sighting") Args: query (list): list of filters to extract which ones are TAXII From 0e658255a8d0bfa670a0fec8ab8588ec6b51dfd0 Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 30 Aug 2017 15:33:28 -0400 Subject: [PATCH 74/82] Move check to when custom observable is defined Catch properties named "_ref" or "_refs" that aren't ObjectReferenceProperty when the custom observable is defined, not when it is instantiated. --- stix2/base.py | 16 ++++++---------- stix2/observables.py | 10 ++++++++++ stix2/test/test_custom.py | 35 +++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index 7de193b..185ff35 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -205,18 +205,14 @@ class _Observable(_STIXBase): try: allowed_types = prop.contained.valid_types except AttributeError: - try: - allowed_types = prop.valid_types - except AttributeError: - raise ValueError("'%s' is named like an object reference property but " - "is not an ObjectReferenceProperty or a ListProperty " - "containing ObjectReferenceProperty." % prop_name) + allowed_types = prop.valid_types + + try: + ref_type = self._STIXBase__valid_refs[ref] + except TypeError: + raise ValueError("'%s' must be created with _valid_refs as a dict, not a list." % self.__class__.__name__) if allowed_types: - try: - ref_type = self._STIXBase__valid_refs[ref] - except TypeError: - raise ValueError("'%s' must be created with _valid_refs as a dict, not a list." % self.__class__.__name__) if ref_type not in allowed_types: raise InvalidObjRefError(self.__class__, prop_name, "object reference '%s' is of an invalid type '%s'" % (ref, ref_type)) diff --git a/stix2/observables.py b/stix2/observables.py index 50eb6d1..4c29785 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -766,6 +766,16 @@ def CustomObservable(type='x-custom-observable', properties={}): _properties = { 'type': TypeProperty(_type), } + # Check properties ending in "_ref/s" are ObjectReferenceProperties + for prop_name, prop in properties.items(): + if prop_name.endswith('_ref') and prop.__class__.__name__ != 'ObjectReferenceProperty': + raise ValueError("'%s' is named like an object reference property but " + "is not an ObjectReferenceProperty." % prop_name) + elif (prop_name.endswith('_refs') and (prop.__class__.__name__ != 'ListProperty' + or prop.contained.__class__.__name__ != 'ObjectReferenceProperty')): + raise ValueError("'%s' is named like an object reference property but " + "is not a ListProperty containing ObjectReferenceProperty." % prop_name) + _properties.update(properties) def __init__(self, **kwargs): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index d2bb16f..313a0a8 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -153,20 +153,35 @@ def test_custom_observable_object(): def test_custom_observable_object_invalid_ref_property(): - @stix2.observables.CustomObservable('x-new-obs', { - 'property1': stix2.properties.StringProperty(required=True), - 'property_ref': stix2.properties.StringProperty(), - }) - class NewObs(): - pass - with pytest.raises(ValueError) as excinfo: - NewObs(_valid_refs={'1': 'file'}, - property1='something', - property_ref='1') + @stix2.observables.CustomObservable('x-new-obs', { + 'property_ref': stix2.properties.StringProperty(), + }) + class NewObs(): + pass assert "is named like an object reference property but is not an ObjectReferenceProperty" in str(excinfo.value) +def test_custom_observable_object_invalid_refs_property(): + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomObservable('x-new-obs', { + 'property_refs': stix2.properties.StringProperty(), + }) + class NewObs(): + pass + assert "is named like an object reference property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + + +def test_custom_observable_object_invalid_refs_list_property(): + with pytest.raises(ValueError) as excinfo: + @stix2.observables.CustomObservable('x-new-obs', { + 'property_refs': stix2.properties.ListProperty(stix2.properties.StringProperty), + }) + class NewObs(): + pass + assert "is named like an object reference property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + + def test_custom_observable_object_invalid_valid_refs(): @stix2.observables.CustomObservable('x-new-obs', { 'property1': stix2.properties.StringProperty(required=True), From 8a33cb7716bd8d567dc1e1e66f8c4a6e2164662f Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 30 Aug 2017 16:15:05 -0400 Subject: [PATCH 75/82] Touch up ObjectReferenceProperty checks - reword "_refs" error - check with isinstance instead of __class__.__name__ --- stix2/observables.py | 8 ++++---- stix2/test/test_custom.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stix2/observables.py b/stix2/observables.py index 4c29785..c5290ff 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -768,12 +768,12 @@ def CustomObservable(type='x-custom-observable', properties={}): } # Check properties ending in "_ref/s" are ObjectReferenceProperties for prop_name, prop in properties.items(): - if prop_name.endswith('_ref') and prop.__class__.__name__ != 'ObjectReferenceProperty': + if prop_name.endswith('_ref') and not isinstance(prop, ObjectReferenceProperty): raise ValueError("'%s' is named like an object reference property but " "is not an ObjectReferenceProperty." % prop_name) - elif (prop_name.endswith('_refs') and (prop.__class__.__name__ != 'ListProperty' - or prop.contained.__class__.__name__ != 'ObjectReferenceProperty')): - raise ValueError("'%s' is named like an object reference property but " + elif (prop_name.endswith('_refs') and (not isinstance(prop, ListProperty) + or not isinstance(prop.contained, ObjectReferenceProperty))): + raise ValueError("'%s' is named like an object reference list property but " "is not a ListProperty containing ObjectReferenceProperty." % prop_name) _properties.update(properties) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 313a0a8..266cfd2 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -169,7 +169,7 @@ def test_custom_observable_object_invalid_refs_property(): }) class NewObs(): pass - assert "is named like an object reference property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + assert "is named like an object reference list property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) def test_custom_observable_object_invalid_refs_list_property(): @@ -179,7 +179,7 @@ def test_custom_observable_object_invalid_refs_list_property(): }) class NewObs(): pass - assert "is named like an object reference property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) + assert "is named like an object reference list property but is not a ListProperty containing ObjectReferenceProperty" in str(excinfo.value) def test_custom_observable_object_invalid_valid_refs(): From 15287959a4bdd8ec41899214d0ec3758e7ef1aa0 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 31 Aug 2017 12:28:07 -0400 Subject: [PATCH 76/82] Modify versioning API to work on dictionaries This includes new_version() and revoke(). --- stix2/base.py | 31 ++----- stix2/test/constants.py | 10 +++ stix2/test/test_versioning.py | 157 ++++++++++++++++++---------------- stix2/utils.py | 52 +++++++++++ 4 files changed, 153 insertions(+), 97 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index acac94c..c819340 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -9,10 +9,11 @@ from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, ExtraPropertiesError, ImmutableError, InvalidObjRefError, InvalidValueError, MissingPropertiesError, - MutuallyExclusivePropertiesError, RevokeError, - UnmodifiablePropertyError) + MutuallyExclusivePropertiesError) from .markings.utils import validate -from .utils import NOW, format_datetime, get_timestamp, parse_into_datetime +from .utils import NOW, format_datetime, get_timestamp +from .utils import new_version as _new_version +from .utils import revoke as _revoke __all__ = ['STIXJSONEncoder', '_STIXBase'] @@ -162,30 +163,10 @@ class _STIXBase(collections.Mapping): # Versioning API def new_version(self, **kwargs): - unchangable_properties = [] - if self.get("revoked"): - raise RevokeError("new_version") - new_obj_inner = copy.deepcopy(self._inner) - properties_to_change = kwargs.keys() - for prop in ["created", "created_by_ref", "id", "type"]: - if prop in properties_to_change: - unchangable_properties.append(prop) - if unchangable_properties: - raise UnmodifiablePropertyError(unchangable_properties) - cls = type(self) - if 'modified' not in kwargs: - kwargs['modified'] = get_timestamp() - else: - new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') - if new_modified_property < self.modified: - raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") - new_obj_inner.update(kwargs) - return cls(**new_obj_inner) + return _new_version(self, **kwargs) def revoke(self): - if self.get("revoked"): - raise RevokeError("revoke") - return self.new_version(revoked=True) + return _revoke(self) class _Observable(_STIXBase): diff --git a/stix2/test/constants.py b/stix2/test/constants.py index d62d932..c1789af 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -29,6 +29,16 @@ MARKING_IDS = [ "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f", ] +# All required args for a Campaign instance, plus some optional args +CAMPAIGN_MORE_KWARGS = dict( + id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + created="2016-04-06T20:03:00.000Z", + modified="2016-04-06T20:03:00.000Z", + name="Green Group Attacks Against Finance", + description="Campaign by Green Group against a series of targets in the financial services sector.", +) + # Minimum required args for an Identity instance IDENTITY_KWARGS = dict( name="John Smith", diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 281ae71..c7bd2fc 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -2,16 +2,11 @@ import pytest import stix2 +from .constants import CAMPAIGN_MORE_KWARGS + def test_making_new_version(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.new_version(name="fred") @@ -25,14 +20,7 @@ def test_making_new_version(): def test_making_new_version_with_unset(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.new_version(description=None) @@ -47,16 +35,11 @@ def test_making_new_version_with_unset(): def test_making_new_version_with_embedded_object(): campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", external_references=[{ "source_name": "capec", "external_id": "CAPEC-163" }], - description="Campaign by Green Group against a series of targets in the financial services sector." + **CAMPAIGN_MORE_KWARGS ) campaign_v2 = campaign_v1.new_version(external_references=[{ @@ -74,14 +57,7 @@ def test_making_new_version_with_embedded_object(): def test_revoke(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() @@ -96,14 +72,7 @@ def test_revoke(): def test_versioning_error_invalid_property(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.UnmodifiablePropertyError) as excinfo: campaign_v1.new_version(type="threat-actor") @@ -112,14 +81,7 @@ def test_versioning_error_invalid_property(): def test_versioning_error_bad_modified_value(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: campaign_v1.new_version(modified="2015-04-06T20:03:00.000Z") @@ -130,14 +92,7 @@ def test_versioning_error_bad_modified_value(): def test_versioning_error_usetting_required_property(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: campaign_v1.new_version(name=None) @@ -147,15 +102,7 @@ def test_versioning_error_usetting_required_property(): def test_versioning_error_new_version_of_revoked(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) - + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() with pytest.raises(stix2.exceptions.RevokeError) as excinfo: @@ -165,18 +112,84 @@ def test_versioning_error_new_version_of_revoked(): def test_versioning_error_revoke_of_revoked(): - campaign_v1 = stix2.Campaign( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - created="2016-04-06T20:03:00.000Z", - modified="2016-04-06T20:03:00.000Z", - name="Green Group Attacks Against Finance", - description="Campaign by Green Group against a series of targets in the financial services sector." - ) - + campaign_v1 = stix2.Campaign(**CAMPAIGN_MORE_KWARGS) campaign_v2 = campaign_v1.revoke() with pytest.raises(stix2.exceptions.RevokeError) as excinfo: campaign_v2.revoke() assert excinfo.value.called_by == "revoke" + + +def test_making_new_version_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.new_version(CAMPAIGN_MORE_KWARGS, name="fred") + + assert campaign_v1['id'] == campaign_v2['id'] + assert campaign_v1['created_by_ref'] == campaign_v2['created_by_ref'] + assert campaign_v1['created'] == campaign_v2['created'] + assert campaign_v1['name'] != campaign_v2['name'] + assert campaign_v2['name'] == "fred" + assert campaign_v1['description'] == campaign_v2['description'] + assert stix2.utils.parse_into_datetime(campaign_v1['modified'], precision='millisecond') < campaign_v2['modified'] + + +def test_versioning_error_dict_bad_modified_value(): + with pytest.raises(stix2.exceptions.InvalidValueError) as excinfo: + stix2.utils.new_version(CAMPAIGN_MORE_KWARGS, modified="2015-04-06T20:03:00.000Z") + + assert excinfo.value.cls == dict + assert excinfo.value.prop_name == "modified" + assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + + +def test_versioning_error_dict_no_modified_value(): + campaign_v1 = { + 'type': 'campaign', + 'id': "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + 'created': "2016-04-06T20:03:00.000Z", + 'name': "Green Group Attacks Against Finance", + } + campaign_v2 = stix2.utils.new_version(campaign_v1, modified="2017-04-06T20:03:00.000Z") + + assert str(campaign_v2['modified']) == "2017-04-06T20:03:00.000Z" + + +def test_making_new_version_invalid_cls(): + campaign_v1 = "This is a campaign." + with pytest.raises(ValueError) as excinfo: + stix2.utils.new_version(campaign_v1, name="fred") + + assert 'cannot create new version of object of this type' in str(excinfo.value) + + +def test_revoke_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.revoke(campaign_v1) + + assert campaign_v1['id'] == campaign_v2['id'] + assert campaign_v1['created_by_ref'] == campaign_v2['created_by_ref'] + assert campaign_v1['created'] == campaign_v2['created'] + assert campaign_v1['name'] == campaign_v2['name'] + assert campaign_v1['description'] == campaign_v2['description'] + assert stix2.utils.parse_into_datetime(campaign_v1['modified'], precision='millisecond') < campaign_v2['modified'] + + assert campaign_v2['revoked'] + + +def test_versioning_error_revoke_of_revoked_dict(): + campaign_v1 = CAMPAIGN_MORE_KWARGS + campaign_v2 = stix2.utils.revoke(campaign_v1) + + with pytest.raises(stix2.exceptions.RevokeError) as excinfo: + stix2.utils.revoke(campaign_v2) + + assert excinfo.value.called_by == "revoke" + + +def test_revoke_invalid_cls(): + campaign_v1 = "This is a campaign." + with pytest.raises(ValueError) as excinfo: + stix2.utils.revoke(campaign_v1) + + assert 'cannot revoke object of this type' in str(excinfo.value) diff --git a/stix2/utils.py b/stix2/utils.py index 12b889c..c83741c 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,11 +1,16 @@ """Utility functions and classes for the stix2 library.""" +from collections import Mapping +import copy import datetime as dt import json from dateutil import parser import pytz +from .exceptions import (InvalidValueError, RevokeError, + UnmodifiablePropertyError) + # Sentinel value for properties that should be set to the current time. # We can't use the standard 'default' approach, since if there are multiple # timestamps in a single object, the timestamps will vary by a few microseconds. @@ -112,3 +117,50 @@ def get_dict(data): return dict(data) except (ValueError, TypeError): raise ValueError("Cannot convert '%s' to dictionary." % str(data)) + + +def new_version(data, **kwargs): + """Create a new version of a STIX object, by modifying properties and + updating the `modified` property. + """ + + if not isinstance(data, Mapping): + raise ValueError('cannot create new version of object of this type! ' + 'Try a dictionary or instance of an SDO or SRO class.') + + unchangable_properties = [] + if data.get("revoked"): + raise RevokeError("new_version") + try: + new_obj_inner = copy.deepcopy(data._inner) + except AttributeError: + new_obj_inner = copy.deepcopy(data) + properties_to_change = kwargs.keys() + + # Make sure certain properties aren't trying to change + for prop in ["created", "created_by_ref", "id", "type"]: + if prop in properties_to_change: + unchangable_properties.append(prop) + if unchangable_properties: + raise UnmodifiablePropertyError(unchangable_properties) + + cls = type(data) + if 'modified' not in kwargs: + kwargs['modified'] = get_timestamp() + elif 'modified' in data: + old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond') + new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') + if new_modified_property < old_modified_property: + raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") + new_obj_inner.update(kwargs) + return cls(**new_obj_inner) + + +def revoke(data): + if not isinstance(data, Mapping): + raise ValueError('cannot revoke object of this type! Try a dictionary ' + 'or instance of an SDO or SRO class.') + + if data.get("revoked"): + raise RevokeError("revoke") + return new_version(data, revoked=True) From 9d45a3dca264376197e894f3b1f68f77b4847063 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 31 Aug 2017 12:51:13 -0400 Subject: [PATCH 77/82] Keep MALWARE_KWARGS to a minimal set ...but use an additional set with a description for marking tests. --- stix2/test/constants.py | 6 ++++++ stix2/test/test_bundle.py | 1 - stix2/test/test_granular_markings.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/stix2/test/constants.py b/stix2/test/constants.py index c1789af..c93f803 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -55,6 +55,12 @@ INDICATOR_KWARGS = dict( MALWARE_KWARGS = dict( labels=['ransomware'], name="Cryptolocker", +) + +# All required args for a Malware instance, plus some optional args +MALWARE_MORE_KWARGS = dict( + labels=['ransomware'], + name="Cryptolocker", description="A ransomware related to ..." ) diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b73345a..0733637 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -19,7 +19,6 @@ EXPECTED_BUNDLE = """{ }, { "created": "2017-01-01T12:34:56.000Z", - "description": "A ransomware related to ...", "id": "malware--00000000-0000-0000-0000-000000000002", "labels": [ "ransomware" diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index 766348e..ef6f725 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -4,7 +4,7 @@ import pytest from stix2 import Malware, markings from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS -from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST +from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST """Tests for the Data Markings API.""" From 7b46283a5cc9b19037861726c69960145175338e Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 31 Aug 2017 18:03:12 +0000 Subject: [PATCH 78/82] Build filter function map --- stix2/sources/__init__.py | 6 ++--- stix2/sources/filters.py | 39 +++++++-------------------------- stix2/sources/memory.py | 3 ++- stix2/sources/taxii.py | 3 ++- stix2/test/test_data_sources.py | 3 ++- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 53b005e..7241a0b 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -20,8 +20,8 @@ import uuid from six import iteritems -from filters import (FILTER_OPS, FILTER_VALUE_TYPES, STIX_COMMON_FIELDS, - STIX_COMMON_FILTERS_MAP) +from stix2.sources.filters import (FILTER_OPS, FILTER_VALUE_TYPES, + STIX_COMMON_FIELDS, STIX_COMMON_FILTERS_MAP) def make_id(): @@ -273,7 +273,7 @@ class DataSource(object): clean = False break - match = STIX_COMMON_FILTERS_MAP[filter_.field](filter_, stix_obj) + match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) if not match: clean = False break diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py index baa32c8..7758369 100644 --- a/stix2/sources/filters.py +++ b/stix2/sources/filters.py @@ -13,8 +13,6 @@ or if cleaner solution possible. import collections import types -import filters - # Currently, only STIX 2.0 common SDO fields (that are not complex objects) # are supported for filtering on STIX_COMMON_FIELDS = [ @@ -180,11 +178,11 @@ def check_labels_filter(filter_, stix_obj): def check_modified_filter(filter_, stix_obj): - return _timestamp_filter(filter_, stix_obj["created"]) + return _timestamp_filter(filter_, stix_obj["modified"]) -def check_object_markings_ref_filter(filter_, stix_obj): - for marking_id in stix_obj["object_market_refs"]: +def check_object_marking_refs_filter(filter_, stix_obj): + for marking_id in stix_obj["object_marking_refs"]: r = _id_filter(filter_, marking_id) if r: return r @@ -199,29 +197,8 @@ def check_type_filter(filter_, stix_obj): return _string_filter(filter_, stix_obj["type"]) -# script to collect STIX common field filter -# functions and create mapping to them - -""" -MK: I want to build the filter name -> filter function dictionary -dynamically whenever it is imported. By enumerating the functions -in this module, extracting the "check*" functions and making -pointers to them. But having issues getting an interable of the -modules entities. globals() works but returns an active dictionary -so iterating over it is a no go -""" - -for entity in dir(filters): - if "check_" in str(entity) and isinstance(filters.__dict__.get(entity), types.FunctionType): - field_name = entity.split("_")[1].split("_")[0] - STIX_COMMON_FILTERS_MAP[field_name] = filters.__dict__.get(entity) - -# Tried this to, didnt work ############## -""" -import sys -for entity in dir(sys.modules[__name__]): - print(entity) - if "check_" in str(entity) and type(entity) == "function": - print(sys.modules[__name__].__dict__.get(entity)) - STIX_COMMON_FILTERS_MAP[str(entity)] = sys.modules[__name__].__dict__.get(entity) -""" +# Create mapping of field names to filter functions +for name, obj in dict(globals()).items(): + if "check_" in name and isinstance(obj, types.FunctionType): + field_name = "_".join(name.split("_")[1:-1]) + STIX_COMMON_FILTERS_MAP[field_name] = obj diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 9f2fa49..8cf8e20 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,7 +24,8 @@ import os from stix2validator import validate_string from stix2 import Bundle -from stix2.sources import DataSink, DataSource, DataStore, Filter +from stix2.sources import DataSink, DataSource, DataStore +from stix2.sources.filters import Filter class MemoryStore(DataStore): diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index b9dc8c4..41632ae 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -12,7 +12,8 @@ TODO: Test everything import json -from stix2.sources import DataSink, DataSource, DataStore, Filter, make_id +from stix2.sources import DataSink, DataSource, DataStore, make_id +from stix2.sources.filters import Filter TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index ee37825..79e0c8b 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -2,7 +2,8 @@ import pytest from taxii2client import Collection from stix2.sources import (CompositeDataSource, DataSink, DataSource, - DataStore, Filter, make_id, taxii) + DataStore, make_id, taxii) +from stix2.sources.filters import Filter from stix2.sources.memory import MemorySource COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' From ba846f95016ae3160c904334a3c7ab4b9ccaa3ce Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 31 Aug 2017 18:23:08 +0000 Subject: [PATCH 79/82] Clean up some tests. --- stix2/sources/__init__.py | 47 +++--- stix2/test/test_data_sources.py | 267 ++++++++++++++++---------------- 2 files changed, 157 insertions(+), 157 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 7241a0b..f702748 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -253,34 +253,31 @@ class DataSource(object): for stix_obj in stix_objs: clean = True for filter_ in query: - try: - # skip filter as filter was identified (when added) as - # not a common filter - if filter_.field not in STIX_COMMON_FIELDS: - raise Exception("Error, field: {0} is not supported for filtering on.".format(filter_.field)) + # skip filter as filter was identified (when added) as + # not a common filter + if filter_.field not in STIX_COMMON_FIELDS: + raise ValueError("Error, field: {0} is not supported for filtering on.".format(filter_.field)) - # For properties like granular_markings and external_references - # need to break the first property from the string. - if "." in filter_.field: - field = filter_.field.split(".")[0] - else: - field = filter_.field + # For properties like granular_markings and external_references + # need to break the first property from the string. + if "." in filter_.field: + field = filter_.field.split(".")[0] + else: + field = filter_.field - # check filter "field" is in STIX object - if cant be - # applied due to STIX object, STIX object is discarded - # (i.e. did not make it through the filter) - if field not in stix_obj.keys(): - clean = False - break + # check filter "field" is in STIX object - if cant be + # applied due to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + if field not in stix_obj.keys(): + clean = False + break - match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) - if not match: - clean = False - break - elif match == -1: - raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) - except Exception as e: - raise ValueError(e) + match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) + if not match: + clean = False + break + elif match == -1: + raise ValueError("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) # if object unmarked after all filters, add it if clean: diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 79e0c8b..76934fb 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -19,107 +19,110 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) -STIX_OBJS1 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] +@pytest.fixture +def ds(): + return DataSource() -STIX_OBJS2 = [ - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-31T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - }, - { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" - } -] + +IND1 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND2 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND3 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND4 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND5 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND6 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND7 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND8 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} + +STIX_OBJS2 = [IND6, IND7, IND8] +STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] def test_ds_smoke(): @@ -173,7 +176,7 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -def test_add_get_remove_filter(): +def test_add_get_remove_filter(ds): # First 3 filters are valid, remaining fields are erroneous in some way valid_filters = [ @@ -187,8 +190,6 @@ def test_add_get_remove_filter(): Filter('created', '=', object()), ] - ds = DataSource() - assert len(ds.filters) == 0 ds.add_filter(valid_filters[0]) @@ -226,7 +227,7 @@ def test_add_get_remove_filter(): ds.add_filters(valid_filters) -def test_apply_common_filters(): +def test_apply_common_filters(ds): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -287,8 +288,6 @@ def test_apply_common_filters(): Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), ] - ds = DataSource() - resp = ds.apply_common_filters(stix_objs, [filters[0]]) ids = [r['id'] for r in resp] assert stix_objs[0]['id'] in ids @@ -328,61 +327,65 @@ def test_apply_common_filters(): assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 - # These are used with STIX_OBJS2 - more_filters = [ - Filter("modified", "<", "2017-01-28T13:49:53.935Z"), - Filter("modified", ">", "2017-01-28T13:49:53.935Z"), - Filter("modified", ">=", "2017-01-27T13:49:53.935Z"), - Filter("modified", "<=", "2017-01-27T13:49:53.935Z"), - Filter("modified", "?", "2017-01-27T13:49:53.935Z"), - Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), - Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"), - Filter("notacommonproperty", "=", "bar"), - ] - resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[0]]) +def test_filters0(ds): + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")]) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 - resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[1]]) + +def test_filters1(ds): + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 - resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[2]]) + +def test_filters2(ds): + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 - resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[3]]) + +def test_filters3(ds): + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 + +def test_filters4(ds): + fltr4 = Filter("modified", "?", "2017-01-27T13:49:53.935Z") with pytest.raises(ValueError) as excinfo: - ds.apply_common_filters(STIX_OBJS2, [more_filters[4]]) + ds.apply_common_filters(STIX_OBJS2, [fltr4]) assert str(excinfo.value) == ("Error, filter operator: {0} not supported " - "for specified field: {1}").format(more_filters[4].op, - more_filters[4].field) + "for specified field: {1}").format(fltr4.op, fltr4.field) - resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[5]]) + +def test_filters5(ds): + resp = ds.apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 + +def test_filters6(ds): + fltr6 = Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") with pytest.raises(ValueError) as excinfo: - ds.apply_common_filters(STIX_OBJS2, [more_filters[6]]) + ds.apply_common_filters(STIX_OBJS2, [fltr6]) assert str(excinfo.value) == ("Error, filter operator: {0} not supported " - "for specified field: {1}").format(more_filters[6].op, - more_filters[6].field) + "for specified field: {1}").format(fltr6.op, fltr6.field) + +def test_filters7(ds): + fltr7 = Filter("notacommonproperty", "=", "bar") with pytest.raises(ValueError) as excinfo: - ds.apply_common_filters(STIX_OBJS2, [more_filters[7]]) + ds.apply_common_filters(STIX_OBJS2, [fltr7]) assert str(excinfo.value) == ("Error, field: {0} is not supported for " - "filtering on.".format(more_filters[7].field)) + "filtering on.".format(fltr7.field)) -def test_deduplicate(): - ds = DataSource() +def test_deduplicate(ds): unique = ds.deduplicate(STIX_OBJS1) # Only 3 objects are unique From 13f2810b252a3eae6aaf6a254a16e44a8cf51093 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 1 Sep 2017 08:15:50 -0400 Subject: [PATCH 80/82] Remove name positional argument, clean memory.py redundant codeother minor changes. Increase coverage. --- .gitignore | 5 +- stix2/sources/__init__.py | 208 +++++++++++++++----------------- stix2/sources/filesystem.py | 79 +++++------- stix2/sources/memory.py | 187 +++++++++------------------- stix2/sources/taxii.py | 12 +- stix2/test/test_data_sources.py | 152 ++++++++++++++++------- 6 files changed, 312 insertions(+), 331 deletions(-) diff --git a/.gitignore b/.gitignore index 3b9971a..3b953e1 100644 --- a/.gitignore +++ b/.gitignore @@ -57,9 +57,12 @@ docs/_build/ # PyBuilder target/ +# External data cache +cache.sqlite + # Vim *.swp -# + # PyCharm .idea/ diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 6360fde..529725e 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -70,17 +70,21 @@ class DataStore(object): An implementer will create a concrete subclass from this abstract class for the specific data store. + Attributes: + id (str): A unique UUIDv4 to identify this DataStore. + source (DataStore): An object that implements DataStore class. + sink (DataSink): An object that implements DataSink class. + """ - def __init__(self, name="DataStore", source=None, sink=None): - self.name = name - self.id_ = make_id() + def __init__(self, source=None, sink=None): + self.id = make_id() self.source = source self.sink = sink def get(self, stix_id): """ - Implement: - Translate API get() call to the appropriate DataSource call + Notes: + Translate API get() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX 2.0 object to retrieve. Should @@ -103,9 +107,6 @@ class DataStore(object): return a single object, the most recent version of the object specified by the "id". - _composite_filters (list): list of filters passed along from - the Composite Data Filter. - Returns: stix_objs (list): a list of STIX objects (where each object is a STIX object) @@ -115,9 +116,9 @@ class DataStore(object): def query(self, query): """ - Fill: + Notes: Implement the specific data source API calls, processing, - functionality required for retrieving query from the data source + functionality required for retrieving query from the data source. Args: query (list): a list of filters (which collectively are the query) @@ -132,8 +133,8 @@ class DataStore(object): def add(self, stix_objs): """ - Fill: - -translate add() to the appropriate DataSink call() + Notes: + Translate add() to the appropriate DataSink call(). """ return self.sink.add(stix_objs=stix_objs) @@ -145,18 +146,15 @@ class DataSink(object): different sink components. Attributes: - id_ (str): A unique UUIDv4 to identify this DataSink. - name (str): The descriptive name that identifies this DataSink. + id (str): A unique UUIDv4 to identify this DataSink. """ - - def __init__(self, name="DataSink"): - self.name = name - self.id_ = make_id() + def __init__(self): + self.id = make_id() def add(self, stix_objs): """ - Fill: + Notes: Implement the specific data sink API calls, processing, functionality required for adding data to the sink @@ -170,15 +168,12 @@ class DataSource(object): different source components. Attributes: - id_ (str): A unique UUIDv4 to identify this DataSource. - name (str): The descriptive name that identifies this DataSource. + id (str): A unique UUIDv4 to identify this DataSource. filters (set): A collection of filters present in this DataSource. """ - - def __init__(self, name="DataSource"): - self.name = name - self.id_ = make_id() + def __init__(self): + self.id = make_id() self.filters = set() def get(self, stix_id, _composite_filters=None): @@ -203,12 +198,11 @@ class DataSource(object): def all_versions(self, stix_id, _composite_filters=None): """ - Fill: - -Similar to get() except returns list of all object versions of - the specified "id". - - -implement the specific data source API calls, processing, - functionality required for retrieving data from the data source + Notes: + Similar to get() except returns list of all object versions of + the specified "id". In addition, implement the specific data + source API calls, processing, functionality required for retrieving + data from the data source. Args: stix_id (str): The id of the STIX 2.0 object to retrieve. Should @@ -249,26 +243,24 @@ class DataSource(object): Args: filters (list): list of filters (dict) to add to the Data Source. """ - for filter_ in filters: - self.add_filter(filter_) + for filter in filters: + self.add_filter(filter) - def add_filter(self, filter_): + def add_filter(self, filter): """Add a filter.""" # check filter field is a supported STIX 2.0 common field - if filter_.field not in STIX_COMMON_FIELDS: + if filter.field not in STIX_COMMON_FIELDS: raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") # check filter operator is supported - if filter_.op not in FILTER_OPS: - raise ValueError("Filter operation(from 'op' field) not supported") + if filter.op not in FILTER_OPS: + raise ValueError("Filter operation (from 'op' field) not supported") # check filter value type is supported - if type(filter_.value) not in FILTER_VALUE_TYPES: + if type(filter.value) not in FILTER_VALUE_TYPES: raise ValueError("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") - self.filters.add(filter_) - - # TODO: Do we need a remove_filter function? + self.filters.add(filter) def apply_common_filters(self, stix_objs, query): """Evaluates filters against a set of STIX 2.0 objects @@ -289,19 +281,19 @@ class DataSource(object): # evaluate objects against filter for stix_obj in stix_objs: clean = True - for filter_ in query: + for filter in query: try: # skip filter as filter was identified (when added) as # not a common filter - if filter_.field not in STIX_COMMON_FIELDS: - raise Exception("Error, field: {0} is not supported for filtering on.".format(filter_.field)) + if filter.field not in STIX_COMMON_FIELDS: + raise Exception("Error, field: {0} is not supported for filtering on.".format(filter.field)) # For properties like granular_markings and external_references # need to break the first property from the string. - if "." in filter_.field: - field = filter_.field.split(".")[0] + if "." in filter.field: + field = filter.field.split(".")[0] else: - field = filter_.field + field = filter.field # check filter "field" is in STIX object - if cant be # applied due to STIX object, STIX object is discarded @@ -310,12 +302,12 @@ class DataSource(object): clean = False break - match = getattr(STIXCommonPropertyFilters, field)(filter_, stix_obj) + match = getattr(STIXCommonPropertyFilters, field)(filter, stix_obj) if not match: clean = False break elif match == -1: - raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) + raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter.op, filter.field)) except Exception as e: raise ValueError(e) @@ -361,7 +353,7 @@ class CompositeDataSource(DataSource): controlled and used by the Data Source Controller object. """ - def __init__(self, name="CompositeDataSource"): + def __init__(self): """ Creates a new STIX Data Source. @@ -370,7 +362,7 @@ class CompositeDataSource(DataSource): CompositeDataSource instance. """ - super(CompositeDataSource, self).__init__(name=name) + super(CompositeDataSource, self).__init__() self.data_sources = {} def get(self, stix_id, _composite_filters=None): @@ -498,13 +490,13 @@ class CompositeDataSource(DataSource): """ for ds in data_sources: if issubclass(ds.__class__, DataSource): - if ds.id_ in self.data_sources: + if ds.id in self.data_sources: # data source already attached to Composite Data Source continue # add data source to Composite Data Source # (its id will be its key identifier) - self.data_sources[ds.id_] = ds + self.data_sources[ds.id] = ds else: # the Data Source object is not a proper subclass # of DataSource Abstract Class @@ -520,9 +512,9 @@ class CompositeDataSource(DataSource): data_source_ids (list): a list of Data Source identifiers. """ - for id_ in data_source_ids: - if id_ in self.data_sources: - del self.data_sources[id_] + for id in data_source_ids: + if id in self.data_sources: + del self.data_sources[id] else: raise ValueError("DataSource 'id' not found in CompositeDataSource collection.") return @@ -538,63 +530,63 @@ class STIXCommonPropertyFilters(object): """ """ @classmethod - def _all(cls, filter_, stix_obj_field): + def _all(cls, filter, stix_obj_field): """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - elif filter_.op == "in": - return stix_obj_field in filter_.value - elif filter_.op == ">": - return stix_obj_field > filter_.value - elif filter_.op == "<": - return stix_obj_field < filter_.value - elif filter_.op == ">=": - return stix_obj_field >= filter_.value - elif filter_.op == "<=": - return stix_obj_field <= filter_.value + if filter.op == "=": + return stix_obj_field == filter.value + elif filter.op == "!=": + return stix_obj_field != filter.value + elif filter.op == "in": + return stix_obj_field in filter.value + elif filter.op == ">": + return stix_obj_field > filter.value + elif filter.op == "<": + return stix_obj_field < filter.value + elif filter.op == ">=": + return stix_obj_field >= filter.value + elif filter.op == "<=": + return stix_obj_field <= filter.value else: return -1 @classmethod - def _id(cls, filter_, stix_obj_id): + def _id(cls, filter, stix_obj_id): """base filter types""" - if filter_.op == "=": - return stix_obj_id == filter_.value - elif filter_.op == "!=": - return stix_obj_id != filter_.value + if filter.op == "=": + return stix_obj_id == filter.value + elif filter.op == "!=": + return stix_obj_id != filter.value else: return -1 @classmethod - def _boolean(cls, filter_, stix_obj_field): - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value + def _boolean(cls, filter, stix_obj_field): + if filter.op == "=": + return stix_obj_field == filter.value + elif filter.op == "!=": + return stix_obj_field != filter.value else: return -1 @classmethod - def _string(cls, filter_, stix_obj_field): - return cls._all(filter_, stix_obj_field) + def _string(cls, filter, stix_obj_field): + return cls._all(filter, stix_obj_field) @classmethod - def _timestamp(cls, filter_, stix_obj_timestamp): - return cls._all(filter_, stix_obj_timestamp) + def _timestamp(cls, filter, stix_obj_timestamp): + return cls._all(filter, stix_obj_timestamp) # STIX 2.0 Common Property filters @classmethod - def created(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["created"]) + def created(cls, filter, stix_obj): + return cls._timestamp(filter, stix_obj["created"]) @classmethod - def created_by_ref(cls, filter_, stix_obj): - return cls._id(filter_, stix_obj["created_by_ref"]) + def created_by_ref(cls, filter, stix_obj): + return cls._id(filter, stix_obj["created_by_ref"]) @classmethod - def external_references(cls, filter_, stix_obj): + def external_references(cls, filter, stix_obj): """ STIX object's can have a list of external references @@ -608,14 +600,14 @@ class STIXCommonPropertyFilters(object): """ for er in stix_obj["external_references"]: # grab er property name from filter field - filter_field = filter_.field.split(".")[1] - r = cls._string(filter_, er[filter_field]) + filter_field = filter.field.split(".")[1] + r = cls._string(filter, er[filter_field]) if r: return r return False @classmethod - def granular_markings(cls, filter_, stix_obj): + def granular_markings(cls, filter, stix_obj): """ STIX object's can have a list of granular marking references @@ -626,46 +618,46 @@ class STIXCommonPropertyFilters(object): """ for gm in stix_obj["granular_markings"]: # grab gm property name from filter field - filter_field = filter_.field.split(".")[1] + filter_field = filter.field.split(".")[1] if filter_field == "marking_ref": - return cls._id(filter_, gm[filter_field]) + return cls._id(filter, gm[filter_field]) elif filter_field == "selectors": for selector in gm[filter_field]: - r = cls._string(filter_, selector) + r = cls._string(filter, selector) if r: return r return False @classmethod - def id(cls, filter_, stix_obj): - return cls._id(filter_, stix_obj["id"]) + def id(cls, filter, stix_obj): + return cls._id(filter, stix_obj["id"]) @classmethod - def labels(cls, filter_, stix_obj): + def labels(cls, filter, stix_obj): for label in stix_obj["labels"]: - r = cls._string(filter_, label) + r = cls._string(filter, label) if r: return r return False @classmethod - def modified(cls, filter_, stix_obj): - return cls._timestamp(filter_, stix_obj["modified"]) + def modified(cls, filter, stix_obj): + return cls._timestamp(filter, stix_obj["modified"]) @classmethod - def object_marking_refs(cls, filter_, stix_obj): + def object_marking_refs(cls, filter, stix_obj): for marking_id in stix_obj["object_marking_refs"]: - r = cls._id(filter_, marking_id) + r = cls._id(filter, marking_id) if r: return r return False @classmethod - def revoked(cls, filter_, stix_obj): - return cls._boolean(filter_, stix_obj["revoked"]) + def revoked(cls, filter, stix_obj): + return cls._boolean(filter, stix_obj["revoked"]) @classmethod - def type(cls, filter_, stix_obj): - return cls._string(filter_, stix_obj["type"]) + def type(cls, filter, stix_obj): + return cls._string(filter, stix_obj["type"]) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 61e7c88..cf69675 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -19,8 +19,8 @@ from stix2.sources import DataSink, DataSource, DataStore, Filter class FileSystemStore(DataStore): """ """ - def __init__(self, name="FileSystemStore", stix_dir="stix_data"): - super(FileSystemStore, self).__init__(name=name) + def __init__(self, stix_dir="stix_data"): + super(FileSystemStore, self).__init__() self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir) @@ -28,8 +28,8 @@ class FileSystemStore(DataStore): class FileSystemSink(DataSink): """ """ - def __init__(self, name="FileSystemSink", stix_dir="stix_data"): - super(FileSystemSink, self).__init__(name=name) + def __init__(self, stix_dir="stix_data"): + super(FileSystemSink, self).__init__() self.stix_dir = os.path.abspath(stix_dir) # check directory path exists @@ -58,8 +58,8 @@ class FileSystemSink(DataSink): class FileSystemSource(DataSource): """ """ - def __init__(self, name="FileSystemSource", stix_dir="stix_data"): - super(FileSystemSource, self).__init__(name=name) + def __init__(self, stix_dir="stix_data"): + super(FileSystemSource, self).__init__() self.stix_dir = os.path.abspath(stix_dir) # check directory path exists @@ -71,15 +71,13 @@ class FileSystemSource(DataSource): return self.stix_dir @stix_dir.setter - def stix_dir(self, dir_): - self.stix_dir = dir_ + def stix_dir(self, dir): + self.stix_dir = dir def get(self, stix_id, _composite_filters=None): """ """ - query = [ - Filter("id", "=", stix_id) - ] + query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -95,17 +93,6 @@ class FileSystemSource(DataSource): (Approved by G.B.) """ - - # query = [ - # { - # "field": "id", - # "op": "=", - # "value": stix_id - # } - # ] - - # all_data = self.query(query=query, _composite_filters=_composite_filters) - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): @@ -134,13 +121,13 @@ class FileSystemSource(DataSource): # the corresponding subdirectories as well include_paths = [] declude_paths = [] - if "type" in [filter_.field for filter_ in file_filters]: - for filter_ in file_filters: - if filter_.field == "type": - if filter_.op == "=": - include_paths.append(os.path.join(self.stix_dir, filter_.value)) - elif filter_.op == "!=": - declude_paths.append(os.path.join(self.stix_dir, filter_.value)) + if "type" in [filter.field for filter in file_filters]: + for filter in file_filters: + if filter.field == "type": + if filter.op == "=": + include_paths.append(os.path.join(self.stix_dir, filter.value)) + elif filter.op == "!=": + declude_paths.append(os.path.join(self.stix_dir, filter.value)) else: # have to walk entire STIX directory include_paths.append(self.stix_dir) @@ -157,35 +144,35 @@ class FileSystemSource(DataSource): # user has specified types that are not wanted (i.e. "!=") # so query will look in all STIX directories that are not # the specified type. Compile correct dir paths - for dir_ in os.listdir(self.stix_dir): - if os.path.abspath(dir_) not in declude_paths: - include_paths.append(os.path.abspath(dir_)) + for dir in os.listdir(self.stix_dir): + if os.path.abspath(dir) not in declude_paths: + include_paths.append(os.path.abspath(dir)) # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory - if "id" in [filter_.field for filter_ in file_filters]: - for filter_ in file_filters: - if filter_.field == "id" and filter_.op == "=": - id_ = filter_.value + if "id" in [filter.field for filter in file_filters]: + for filter in file_filters: + if filter.field == "id" and filter.op == "=": + id = filter.value break else: - id_ = None + id = None else: - id_ = None + id = None # now iterate through all STIX objs for path in include_paths: for root, dirs, files in os.walk(path): - for file_ in files: - if id_: - if id_ == file_.split(".")[0]: + for file in files: + if id: + if id == file.split(".")[0]: # since ID is specified in one of filters, can evaluate against filename first without loading - stix_obj = json.load(file_)["objects"] + stix_obj = json.load(file)["objects"] # check against other filters, add if match all_data.extend(self.apply_common_filters([stix_obj], query)) else: # have to load into memory regardless to evaluate other filters - stix_obj = json.load(file_)["objects"] + stix_obj = json.load(file)["objects"] all_data.extend(self.apply_common_filters([stix_obj], query)) all_data = self.deduplicate(all_data) @@ -195,7 +182,7 @@ class FileSystemSource(DataSource): """ """ file_filters = [] - for filter_ in query: - if filter_.field == "id" or filter_.field == "type": - file_filters.append(filter_) + for filter in query: + if filter.field == "id" or filter.field == "type": + file_filters.append(filter) return file_filters diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 28d929d..696bdd1 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -18,19 +18,44 @@ Notes: """ +import collections import json import os -from stix2validator import validate_string +from stix2validator import validate_instance from stix2 import Bundle from stix2.sources import DataSink, DataSource, DataStore, Filter +def _add(store, stix_data): + """Adds stix objects to MemoryStore/Source/Sink.""" + if isinstance(stix_data, collections.Mapping): + # stix objects are in a bundle + # verify STIX json data + r = validate_instance(stix_data) + # make dictionary of the objects for easy lookup + if r.is_valid: + for stix_obj in stix_data["objects"]: + store.data[stix_obj["id"]] = stix_obj + else: + raise ValueError("Error: data passed was found to not be valid by the STIX 2 Validator: \n%s", r.as_dict()) + elif isinstance(stix_data, list): + # stix objects are in a list + for stix_obj in stix_data: + r = validate_instance(stix_obj) + if r.is_valid: + store.data[stix_obj["id"]] = stix_obj + else: + raise ValueError("Error: STIX object %s is not valid under STIX 2 validator.\n%s", stix_obj["id"], r) + else: + raise ValueError("stix_data must be in bundle format or raw list") + + class MemoryStore(DataStore): """ """ - def __init__(self, name="MemoryStore", stix_data=None): + def __init__(self, stix_data): """ Notes: It doesn't make sense to create a MemoryStore by passing @@ -38,30 +63,11 @@ class MemoryStore(DataStore): be data concurrency issues. Just as easy to create new MemoryStore. """ - super(MemoryStore, self).__init__(name=name) + super(MemoryStore, self).__init__() self.data = {} if stix_data: - if type(stix_data) == dict: - # stix objects are in a bundle - # verify STIX json data - r = validate_string(json.dumps(stix_data)) - # make dictionary of the objects for easy lookup - if r.is_valid: - for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) - elif type(stix_data) == list: - # stix objects are in a list - for stix_obj in stix_data: - r = validate_string(json.dumps(stix_obj)) - if r.is_valid: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) - print(r) + _add(self, stix_data) self.source = MemorySource(stix_data=self.data, _store=True) self.sink = MemorySink(stix_data=self.data, _store=True) @@ -76,72 +82,28 @@ class MemoryStore(DataStore): class MemorySink(DataSink): """ """ - def __init__(self, name="MemorySink", stix_data=None, _store=False): + def __init__(self, stix_data, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in bundle or a list. - name (string): optional name tag of the data source _store (bool): if the MemorySink is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSource. """ - super(MemorySink, self).__init__(name=name) + super(MemorySink, self).__init__() + self.data = {} if _store: self.data = stix_data - else: - self.data = {} - if stix_data: - if type(stix_data) == dict: - # stix objects are in a bundle - # verify STIX json data - r = validate_string(json.dumps(stix_data)) - # make dictionary of the objects for easy lookup - if r.is_valid: - for stix_obj in stix_data["objects"]: - - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) - self.data = {} - elif type(stix_data) == list: - # stix objects are in a list - for stix_obj in stix_data: - r = validate_string(json.dumps(stix_obj)) - if r.is_valid: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) - print(r) - else: - raise ValueError("stix_data must be in bundle format or raw list") + elif stix_data: + self.add(stix_data) def add(self, stix_data): """ """ - if type(stix_data) == dict: - # stix data is in bundle - r = validate_string(json.dumps(stix_data)) - if r.is_valid: - for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") - print(r) - elif type(stix_data) == list: - # stix data is in list - for stix_obj in stix_data: - r = validate_string(json.dumps(stix_obj)) - if r.is_valid: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) - print(r) - else: - raise ValueError("stix_data must be in bundle format or raw list") + _add(self, stix_data) def save_to_file(self, file_path): """ @@ -151,47 +113,23 @@ class MemorySink(DataSink): class MemorySource(DataSource): - def __init__(self, name="MemorySource", stix_data=None, _store=False): + def __init__(self, stix_data, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in bundle or list. - name (string): optional name tag of the data source. _store (bool): if the MemorySource is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSink. """ - super(MemorySource, self).__init__(name=name) + super(MemorySource, self).__init__() + self.data = {} if _store: self.data = stix_data - else: - self.data = {} - if stix_data: - if type(stix_data) == dict: - # STIX objects are in a bundle - # verify STIX json data - r = validate_string(json.dumps(stix_data)) - # make dictionary of the objects for easy lookup - if r.is_valid: - for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: json data passed to MemorySource() was found to not be validated by STIX 2 Validator") - print(r.as_dict()) - self.data = {} - elif type(stix_data) == list: - # STIX objects are in a list - for stix_obj in stix_data: - r = validate_string(json.dumps(stix_obj)) - if r.is_valid: - self.data[stix_obj["id"]] = stix_obj - else: - print("Error: STIX object %s is not valid under STIX 2 validator." % stix_obj["id"]) - print(r) - else: - raise ValueError("stix_data must be in bundle format or raw list") + elif stix_data: + _add(self, stix_data) def get(self, stix_id, _composite_filters=None): """ @@ -205,9 +143,7 @@ class MemorySource(DataSource): return stix_obj # if there are filters from the composite level, process full query - query = [ - Filter("id", "=", stix_id) - ] + query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -219,22 +155,21 @@ class MemorySource(DataSource): def all_versions(self, stix_id, _composite_filters=None): """ Notes: - Since Memory sources/sinks don't handle multiple versions of a - STIX object, this operation is futile. Translate call to get(). - (Approved by G.B.) + Similar to get() except returns list of all object versions of + the specified "id". + + Args: + stix_id (str): The id of the STIX 2.0 object to retrieve. Should + return a list of objects, all the versions of the object + specified by the "id". + + _composite_filters (list): list of filters passed from the + Composite Data Source. + + Returns: + stix_objs (list): STIX objects that matched ``stix_id``. """ - - # query = [ - # { - # "field": "id", - # "op": "=", - # "value": stix_id - # } - # ] - - # all_data = self.query(query=query, _composite_filters=_composite_filters) - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): @@ -245,14 +180,11 @@ class MemorySource(DataSource): # combine all query filters if self.filters: - query.extend(self.filters.values()) + query.extend(list(self.filters)) if _composite_filters: query.extend(_composite_filters) - # deduplicate data before filtering -> Deduplication is not required as Memory only ever holds one version of an object - # all_data = self.deduplicate(all_data) - - # apply STIX common property filters + # Apply STIX common property filters. all_data = self.apply_common_filters(self.data.values(), query) return all_data @@ -263,11 +195,10 @@ class MemorySource(DataSource): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - r = validate_string(json.dumps(stix_data)) + r = validate_instance(stix_data) if r.is_valid: for stix_obj in stix_data["objects"]: self.data[stix_obj["id"]] = stix_obj - else: - print("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator" % file_path) - print(r) + + raise ValueError("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator.\n%s", file_path, r) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 4edeeed..eec5296 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -20,7 +20,7 @@ TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] class TAXIICollectionStore(DataStore): """ """ - def __init__(self, collection, name="TAXIICollectionStore"): + def __init__(self, collection): """ Create a new TAXII Collection Data store @@ -28,7 +28,7 @@ class TAXIICollectionStore(DataStore): collection (taxii2.Collection): Collection instance """ - super(TAXIICollectionStore, self).__init__(name=name) + super(TAXIICollectionStore, self).__init__() self.source = TAXIICollectionSource(collection) self.sink = TAXIICollectionSink(collection) @@ -36,8 +36,8 @@ class TAXIICollectionStore(DataStore): class TAXIICollectionSink(DataSink): """ """ - def __init__(self, collection, name="TAXIICollectionSink"): - super(TAXIICollectionSink, self).__init__(name=name) + def __init__(self, collection): + super(TAXIICollectionSink, self).__init__() self.collection = collection def add(self, stix_obj): @@ -56,8 +56,8 @@ class TAXIICollectionSink(DataSink): class TAXIICollectionSource(DataSource): """ """ - def __init__(self, collection, name="TAXIICollectionSource"): - super(TAXIICollectionSource, self).__init__(name=name) + def __init__(self, collection): + super(TAXIICollectionSource, self).__init__() self.collection = collection def get(self, stix_id, _composite_filters=None): diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index ee37825..a11e02d 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,9 +1,10 @@ import pytest from taxii2client import Collection +import stix2 from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, Filter, make_id, taxii) -from stix2.sources.memory import MemorySource +from stix2.sources.memory import MemorySource, MemoryStore COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -121,7 +122,7 @@ STIX_OBJS2 = [ ] -def test_ds_smoke(): +def test_ds_abstract_class_smoke(): ds1 = DataSource() ds2 = DataSink() ds3 = DataStore(source=ds1, sink=ds2) @@ -139,14 +140,36 @@ def test_ds_smoke(): ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) +def test_memory_store_smoke(): + # Initialize MemoryStore with dict + ms = MemoryStore(STIX_OBJS1) + + # Add item to sink + ms.add(dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS2, + spec_version="2.0", + type="bundle")) + + resp = ms.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(resp) == 1 + + resp = ms.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + + query = [Filter('type', '=', 'malware')] + + resp = ms.query(query) + assert len(resp) == 0 + + def test_ds_taxii(collection): ds = taxii.TAXIICollectionSource(collection) - assert ds.name == 'TAXIICollectionSource' + assert ds.collection is not None def test_ds_taxii_name(collection): - ds = taxii.TAXIICollectionSource(collection, name='My Data Source Name') - assert ds.name == "My Data Source Name" + ds = taxii.TAXIICollectionSource(collection) + assert ds.collection is not None def test_parse_taxii_filters(): @@ -209,7 +232,7 @@ def test_add_get_remove_filter(): with pytest.raises(ValueError) as excinfo: ds.add_filter(invalid_filters[1]) - assert str(excinfo.value) == "Filter operation(from 'op' field) not supported" + assert str(excinfo.value) == "Filter operation (from 'op' field) not supported" with pytest.raises(ValueError) as excinfo: ds.add_filter(invalid_filters[2]) @@ -270,6 +293,22 @@ def test_apply_common_filters(): "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", "type": "relationship" + }, + { + "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", + "created": "2016-02-14T00:00:00.000Z", + "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", + "modified": "2016-02-14T00:00:00.000Z", + "type": "vulnerability", + "name": "CVE-2014-0160", + "description": "The (1) TLS...", + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2014-0160" + } + ], + "labels": ["heartbleed", "has-logo"] } ] @@ -284,50 +323,90 @@ def test_apply_common_filters(): Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "relationship_type"), Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), + Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), + Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "description"), + Filter("external_references.source_name", "=", "CVE"), ] ds = DataSource() + # "Return any object whose type is not relationship" resp = ds.apply_common_filters(stix_objs, [filters[0]]) ids = [r['id'] for r in resp] assert stix_objs[0]['id'] in ids assert stix_objs[1]['id'] in ids + assert stix_objs[3]['id'] in ids + assert len(ids) == 3 + # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" resp = ds.apply_common_filters(stix_objs, [filters[1]]) assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + # "Return any object that contains remote-access-trojan in labels" resp = ds.apply_common_filters(stix_objs, [filters[2]]) assert resp[0]['id'] == stix_objs[0]['id'] - - resp = ds.apply_common_filters(stix_objs, [filters[3]]) - assert resp[0]['id'] == stix_objs[0]['id'] assert len(resp) == 1 + # "Return any object created after 2015-01-01T01:00:00.000Z" + resp = ds.apply_common_filters(stix_objs, [filters[3]]) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 2 + + # "Return any revoked object" resp = ds.apply_common_filters(stix_objs, [filters[4]]) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + # "Return any object whose not revoked" # Note that if 'revoked' property is not present in object. - # Currently we can't use such an expression to filter for... + # Currently we can't use such an expression to filter for... :( resp = ds.apply_common_filters(stix_objs, [filters[5]]) assert len(resp) == 0 + # Assert unknown operator for _boolean() raises exception. with pytest.raises(ValueError) as excinfo: ds.apply_common_filters(stix_objs, [filters[6]]) assert str(excinfo.value) == ("Error, filter operator: {0} not supported " - "for specified field: {1}").format(filters[6].op, - filters[6].field) + "for specified field: {1}" + .format(filters[6].op, filters[6].field)) + # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs" resp = ds.apply_common_filters(stix_objs, [filters[7]]) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + # "Return any object that contains relationship_type in their selectors AND + # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref" resp = ds.apply_common_filters(stix_objs, [filters[8], filters[9]]) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 - # These are used with STIX_OBJS2 + # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id" + resp = ds.apply_common_filters(stix_objs, [filters[10]]) + assert resp[0]['id'] == stix_objs[3]['id'] + assert len(resp) == 1 + + # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9" + resp = ds.apply_common_filters(stix_objs, [filters[11]]) + assert len(resp) == 1 + + # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None) + resp = ds.apply_common_filters(stix_objs, [filters[12]]) + assert len(resp) == 0 + + # "Return any object that contains description in its selectors" (None) + resp = ds.apply_common_filters(stix_objs, [filters[13]]) + assert len(resp) == 0 + + # "Return any object that object that matches CVE in source_name" (None, case sensitive) + resp = ds.apply_common_filters(stix_objs, [filters[14]]) + assert len(resp) == 0 + + # These filters are used with STIX_OBJS2 object collection. more_filters = [ Filter("modified", "<", "2017-01-28T13:49:53.935Z"), Filter("modified", ">", "2017-01-28T13:49:53.935Z"), @@ -339,45 +418,56 @@ def test_apply_common_filters(): Filter("notacommonproperty", "=", "bar"), ] + # "Return any object modified before 2017-01-28T13:49:53.935Z" resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[0]]) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 + # "Return any object modified after 2017-01-28T13:49:53.935Z" resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[1]]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 + # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[2]]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 + # "Return any object modified before or on 2017-01-28T13:49:53.935Z" resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[3]]) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 + # Assert unknown operator for _all() raises exception. with pytest.raises(ValueError) as excinfo: ds.apply_common_filters(STIX_OBJS2, [more_filters[4]]) assert str(excinfo.value) == ("Error, filter operator: {0} not supported " - "for specified field: {1}").format(more_filters[4].op, - more_filters[4].field) + "for specified field: {1}" + .format(more_filters[4].op, + more_filters[4].field)) + # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[5]]) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 + # Assert unknown operator for _id() raises exception. with pytest.raises(ValueError) as excinfo: ds.apply_common_filters(STIX_OBJS2, [more_filters[6]]) assert str(excinfo.value) == ("Error, filter operator: {0} not supported " - "for specified field: {1}").format(more_filters[6].op, - more_filters[6].field) + "for specified field: {1}" + .format(more_filters[6].op, + more_filters[6].field)) + # Assert unknown field raises exception. with pytest.raises(ValueError) as excinfo: ds.apply_common_filters(STIX_OBJS2, [more_filters[7]]) assert str(excinfo.value) == ("Error, field: {0} is not supported for " - "filtering on.".format(more_filters[7].field)) + "filtering on." + .format(more_filters[7].field)) def test_deduplicate(): @@ -409,12 +499,12 @@ def test_add_remove_composite_datasource(): assert len(cds.get_all_data_sources()) == 2 - cds.remove_data_source([ds1.id_, ds2.id_]) + cds.remove_data_source([ds1.id, ds2.id]) assert len(cds.get_all_data_sources()) == 0 with pytest.raises(ValueError): - cds.remove_data_source([ds3.id_]) + cds.remove_data_source([ds3.id]) def test_composite_datasource_operations(): @@ -448,25 +538,3 @@ def test_composite_datasource_operations(): # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 - -# def test_data_source_file(): -# ds = file.FileDataSource() -# -# assert ds.name == "DataSource" -# -# -# def test_data_source_name(): -# ds = file.FileDataSource(name="My File Data Source") -# -# assert ds.name == "My File Data Source" -# -# -# def test_data_source_get(): -# ds = file.FileDataSource(name="My File Data Source") -# -# with pytest.raises(NotImplementedError): -# ds.get("foo") -# -# #filter testing -# def test_add_filter(): -# ds = file.FileDataSource() From cc66e1a492c75c1004288fb4392f6f0bf27dbc9b Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 1 Sep 2017 10:50:01 -0400 Subject: [PATCH 81/82] Make markings function signatures consistent Put `marking` before `selectors`. The granular marking version of `is_marked` requires selectors to not be None. --- stix2/markings/__init__.py | 22 ++-- stix2/markings/granular_markings.py | 13 +- stix2/test/test_granular_markings.py | 172 +++++++++++++-------------- stix2/test/test_markings.py | 4 +- stix2/test/test_object_markings.py | 126 ++++++++++---------- 5 files changed, 169 insertions(+), 168 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 4ef3e3a..4f72e4c 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -8,7 +8,7 @@ granular markings unless otherwise noted in each of the functions. from stix2.markings import granular_markings, object_markings -def get_markings(obj, selectors, inherited=False, descendants=False): +def get_markings(obj, selectors=None, inherited=False, descendants=False): """ Get all markings associated to the field(s). @@ -45,7 +45,7 @@ def get_markings(obj, selectors, inherited=False, descendants=False): return list(set(results)) -def set_markings(obj, selectors, marking): +def set_markings(obj, marking, selectors=None): """ Removes all markings associated with selectors and appends a new granular marking. Refer to `clear_markings` and `add_markings` for details. @@ -69,10 +69,10 @@ def set_markings(obj, selectors, marking): if selectors is None: return object_markings.set_markings(obj, marking) else: - return granular_markings.set_markings(obj, selectors, marking) + return granular_markings.set_markings(obj, marking, selectors) -def remove_markings(obj, selectors, marking): +def remove_markings(obj, marking, selectors=None): """ Removes granular_marking from the granular_markings collection. @@ -99,10 +99,10 @@ def remove_markings(obj, selectors, marking): if selectors is None: return object_markings.remove_markings(obj, marking) else: - return granular_markings.remove_markings(obj, selectors, marking) + return granular_markings.remove_markings(obj, marking, selectors) -def add_markings(obj, selectors, marking): +def add_markings(obj, marking, selectors=None): """ Appends a granular_marking to the granular_markings collection. @@ -127,10 +127,10 @@ def add_markings(obj, selectors, marking): if selectors is None: return object_markings.add_markings(obj, marking) else: - return granular_markings.add_markings(obj, selectors, marking) + return granular_markings.add_markings(obj, marking, selectors) -def clear_markings(obj, selectors): +def clear_markings(obj, selectors=None): """ Removes all granular_marking associated with the selectors. @@ -158,7 +158,7 @@ def clear_markings(obj, selectors): return granular_markings.clear_markings(obj, selectors) -def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): +def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ Checks if field(s) is marked by any marking or by specific marking(s). @@ -190,8 +190,8 @@ def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): result = granular_markings.is_marked( obj, - selectors, marking, + selectors, inherited, descendants ) @@ -203,8 +203,8 @@ def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): if granular_marks: result = granular_markings.is_marked( obj, - selectors, granular_marks, + selectors, inherited, descendants ) diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 7e2c50a..3ccc1f0 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -45,7 +45,7 @@ def get_markings(obj, selectors, inherited=False, descendants=False): return list(results) -def set_markings(obj, selectors, marking): +def set_markings(obj, marking, selectors): """ Removes all markings associated with selectors and appends a new granular marking. Refer to `clear_markings` and `add_markings` for details. @@ -63,10 +63,10 @@ def set_markings(obj, selectors, marking): """ obj = clear_markings(obj, selectors) - return add_markings(obj, selectors, marking) + return add_markings(obj, marking, selectors) -def remove_markings(obj, selectors, marking): +def remove_markings(obj, marking, selectors): """ Removes granular_marking from the granular_markings collection. @@ -120,7 +120,7 @@ def remove_markings(obj, selectors, marking): return obj.new_version(granular_markings=None) -def add_markings(obj, selectors, marking): +def add_markings(obj, marking, selectors): """ Appends a granular_marking to the granular_markings collection. @@ -213,7 +213,7 @@ def clear_markings(obj, selectors): return obj.new_version(granular_markings=None) -def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): +def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ Checks if field is marked by any marking or by specific marking(s). @@ -239,6 +239,9 @@ def is_marked(obj, selectors, marking=None, inherited=False, descendants=False): marking identifiers match, True is returned. """ + if selectors is None: + raise TypeError("Required argument 'selectors' must be provided") + selectors = utils.convert_to_list(selectors) marking = utils.convert_to_list(marking) utils.validate(obj, selectors) diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index ef6f725..2d3aa6a 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -33,7 +33,7 @@ def test_add_marking_mark_one_selector_multiple_refs(): ], **MALWARE_KWARGS ) - before = markings.add_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -52,7 +52,7 @@ def test_add_marking_mark_multiple_selector_one_refs(): ], **MALWARE_KWARGS ) - before = markings.add_markings(before, ["description", "name"], [MARKING_IDS[0]]) + before = markings.add_markings(before, [MARKING_IDS[0]], ["description", "name"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -75,7 +75,7 @@ def test_add_marking_mark_multiple_selector_multiple_refs(): ], **MALWARE_KWARGS ) - before = markings.add_markings(before, ["description", "name"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "name"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -100,7 +100,7 @@ def test_add_marking_mark_another_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.add_markings(before, ["name"], [MARKING_IDS[0]]) + before = markings.add_markings(before, [MARKING_IDS[0]], ["name"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -125,7 +125,7 @@ def test_add_marking_mark_same_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.add_markings(before, ["description"], [MARKING_IDS[0]]) + before = markings.add_markings(before, [MARKING_IDS[0]], ["description"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -351,7 +351,7 @@ def test_remove_marking_remove_one_selector_with_multiple_refs(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) assert "granular_markings" not in before @@ -365,7 +365,7 @@ def test_remove_marking_remove_multiple_selector_one_ref(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["description", "modified"], MARKING_IDS[0]) + before = markings.remove_markings(before, MARKING_IDS[0], ["description", "modified"]) assert "granular_markings" not in before @@ -388,7 +388,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["modified"], MARKING_IDS[0]) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -420,7 +420,7 @@ def test_remove_marking_mark_one_selector_markings_from_multiple_ones(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -439,7 +439,7 @@ def test_remove_marking_mark_mutilple_selector_multiple_refs(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["description", "modified"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"]) assert "granular_markings" not in before @@ -466,7 +466,7 @@ def test_remove_marking_mark_another_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["modified"], [MARKING_IDS[0]]) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -481,7 +481,7 @@ def test_remove_marking_mark_same_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.remove_markings(before, ["description"], [MARKING_IDS[0]]) + before = markings.remove_markings(before, [MARKING_IDS[0]], ["description"]) assert "granular_markings" not in before @@ -489,7 +489,7 @@ def test_remove_no_markings(): before = { "description": "test description", } - after = markings.remove_markings(before, ["description"], ["marking-definition--1"]) + after = markings.remove_markings(before, ["marking-definition--1"], ["description"]) assert before == after @@ -498,7 +498,7 @@ def test_remove_marking_bad_selector(): "description": "test description", } with pytest.raises(AssertionError): - markings.remove_markings(before, ["title"], ["marking-definition--1", "marking-definition--2"]) + markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"]) IS_MARKED_TEST_DATA = { @@ -526,8 +526,8 @@ IS_MARKED_TEST_DATA = { @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) def test_is_marked_smoke(data): """Smoke test is_marked call does not fail.""" - assert markings.is_marked(data, ["description"]) - assert markings.is_marked(data, ["title"]) is False + assert markings.is_marked(data, selectors=["description"]) + assert markings.is_marked(data, selectors=["title"]) is False @pytest.mark.parametrize("data,selector", [ @@ -548,40 +548,40 @@ def test_is_marked_smoke(data): def test_is_marked_invalid_selector(data, selector): """Test invalid selector raises an error.""" with pytest.raises(AssertionError): - markings.is_marked(data, selector) + markings.is_marked(data, selectors=selector) @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) def test_is_marked_mix_selector(data): """Test valid selector, one marked and one not marked returns True.""" - assert markings.is_marked(data, ["description", "revision"]) - assert markings.is_marked(data, ["description"]) + assert markings.is_marked(data, selectors=["description", "revision"]) + assert markings.is_marked(data, selectors=["description"]) @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) def test_is_marked_valid_selector_no_refs(data): """Test that a valid selector return True when it has marking refs and False when not.""" - assert markings.is_marked(data, ["description"]) - assert markings.is_marked(data, ["description"], ["marking-definition--2", "marking-definition--3"]) - assert markings.is_marked(data, ["description"], ["marking-definition--2"]) - assert markings.is_marked(data, ["description"], ["marking-definition--2", "marking-definition--8"]) is False + assert markings.is_marked(data, selectors=["description"]) + assert markings.is_marked(data, ["marking-definition--2", "marking-definition--3"], ["description"]) + assert markings.is_marked(data, ["marking-definition--2"], ["description"]) + assert markings.is_marked(data, ["marking-definition--2", "marking-definition--8"], ["description"]) is False @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) def test_is_marked_valid_selector_and_refs(data): """Test that a valid selector returns True when marking_refs match.""" - assert markings.is_marked(data, ["description"], ["marking-definition--1"]) - assert markings.is_marked(data, ["title"], ["marking-definition--1"]) is False + assert markings.is_marked(data, ["marking-definition--1"], ["description"]) + assert markings.is_marked(data, ["marking-definition--1"], ["title"]) is False @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) def test_is_marked_valid_selector_multiple_refs(data): """Test that a valid selector returns True if aall marking_refs match. Otherwise False.""" - assert markings.is_marked(data, ["revision"], ["marking-definition--2", "marking-definition--3"]) - assert markings.is_marked(data, ["revision"], ["marking-definition--2", "marking-definition--1"]) is False - assert markings.is_marked(data, ["revision"], "marking-definition--2") - assert markings.is_marked(data, ["revision"], ["marking-definition--1234"]) is False + assert markings.is_marked(data, ["marking-definition--2", "marking-definition--3"], ["revision"]) + assert markings.is_marked(data, ["marking-definition--2", "marking-definition--1"], ["revision"]) is False + assert markings.is_marked(data, "marking-definition--2", ["revision"]) + assert markings.is_marked(data, ["marking-definition--1234"], ["revision"]) is False @pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) @@ -589,8 +589,8 @@ def test_is_marked_no_marking_refs(data): """Test that a valid content selector with no marking_refs returns True if there is a granular_marking that asserts that field, False otherwise.""" - assert markings.is_marked(data, ["type"]) is False - assert markings.is_marked(data, ["revision"]) + assert markings.is_marked(data, selectors=["type"]) is False + assert markings.is_marked(data, selectors=["revision"]) def test_is_marked_positional_arguments_combinations(): @@ -661,75 +661,75 @@ def test_is_marked_positional_arguments_combinations(): ] } - assert markings.is_marked(test_sdo, "a", ["1"], False, False) - assert markings.is_marked(test_sdo, "a", ["1"], True, False) - assert markings.is_marked(test_sdo, "a", ["1"], True, True) - assert markings.is_marked(test_sdo, "a", ["1"], False, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, False) + assert markings.is_marked(test_sdo, ["1"], "a", True, False) + assert markings.is_marked(test_sdo, ["1"], "a", True, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, True) assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False assert markings.is_marked(test_sdo, "b", inherited=True, descendants=False) is False assert markings.is_marked(test_sdo, "b", inherited=True, descendants=True) is False assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "c", ["2"], False, False) - assert markings.is_marked(test_sdo, "c", ["2"], True, False) - assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], True, True) - assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], False, True) + assert markings.is_marked(test_sdo, ["2"], "c", False, False) + assert markings.is_marked(test_sdo, ["2"], "c", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", True, True) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", False, True) assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "c.[0]", ["2"], True, False) - assert markings.is_marked(test_sdo, "c.[0]", ["2"], True, True) + assert markings.is_marked(test_sdo, ["2"], "c.[0]", True, False) + assert markings.is_marked(test_sdo, ["2"], "c.[0]", True, True) assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, False) - assert markings.is_marked(test_sdo, "c.[1]", ["2", "3"], True, False) - assert markings.is_marked(test_sdo, "c.[1]", ["2", "3"], True, True) - assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, False) + assert markings.is_marked(test_sdo, ["2", "3"], "c.[1]", True, False) + assert markings.is_marked(test_sdo, ["2", "3"], "c.[1]", True, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, True) - assert markings.is_marked(test_sdo, "c.[2]", ["4"], False, False) - assert markings.is_marked(test_sdo, "c.[2]", ["2", "4"], True, False) - assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "5"], True, True) - assert markings.is_marked(test_sdo, "c.[2]", ["4", "5"], False, True) + assert markings.is_marked(test_sdo, ["4"], "c.[2]", False, False) + assert markings.is_marked(test_sdo, ["2", "4"], "c.[2]", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2]", True, True) + assert markings.is_marked(test_sdo, ["4", "5"], "c.[2]", False, True) - assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, False) - assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5"], True, False) - assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5"], True, True) - assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2].g", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5"], "c.[2].g", True, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, True) - assert markings.is_marked(test_sdo, "x", ["6"], False, False) - assert markings.is_marked(test_sdo, "x", ["6"], True, False) - assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], True, True) - assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], False, True) + assert markings.is_marked(test_sdo, ["6"], "x", False, False) + assert markings.is_marked(test_sdo, ["6"], "x", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", True, True) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", False, True) - assert markings.is_marked(test_sdo, "x.y", ["7"], False, False) - assert markings.is_marked(test_sdo, "x.y", ["6", "7"], True, False) - assert markings.is_marked(test_sdo, "x.y", ["6", "7", "8"], True, True) - assert markings.is_marked(test_sdo, "x.y", ["7", "8"], False, True) + assert markings.is_marked(test_sdo, ["7"], "x.y", False, False) + assert markings.is_marked(test_sdo, ["6", "7"], "x.y", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y", True, True) + assert markings.is_marked(test_sdo, ["7", "8"], "x.y", False, True) assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7"], True, False) - assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7"], True, True) + assert markings.is_marked(test_sdo, ["6", "7"], "x.y.[0]", True, False) + assert markings.is_marked(test_sdo, ["6", "7"], "x.y.[0]", True, True) assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, False) - assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8"], True, False) - assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8"], True, True) - assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y.[1]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8"], "x.y.[1]", True, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, True) - assert markings.is_marked(test_sdo, "x.z", ["9"], False, False) - assert markings.is_marked(test_sdo, "x.z", ["6", "9"], True, False) - assert markings.is_marked(test_sdo, "x.z", ["6", "9", "10"], True, True) - assert markings.is_marked(test_sdo, "x.z", ["9", "10"], False, True) + assert markings.is_marked(test_sdo, ["9"], "x.z", False, False) + assert markings.is_marked(test_sdo, ["6", "9"], "x.z", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z", True, True) + assert markings.is_marked(test_sdo, ["9", "10"], "x.z", False, True) assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9"], True, False) - assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9"], True, True) + assert markings.is_marked(test_sdo, ["6", "9"], "x.z.foo1", True, False) + assert markings.is_marked(test_sdo, ["6", "9"], "x.z.foo1", True, True) assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, False) - assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10"], True, False) - assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10"], True, True) - assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z.foo2", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10"], "x.z.foo2", True, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, True) def test_create_sdo_with_invalid_marking(): @@ -763,7 +763,7 @@ def test_set_marking_mark_one_selector_multiple_refs(): ], **MALWARE_KWARGS ) - before = markings.set_markings(before, ["description"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -787,7 +787,7 @@ def test_set_marking_mark_multiple_selector_one_refs(): ], **MALWARE_KWARGS ) - before = markings.set_markings(before, ["description", "modified"], [MARKING_IDS[0]]) + before = markings.set_markings(before, [MARKING_IDS[0]], ["description", "modified"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -809,7 +809,7 @@ def test_set_marking_mark_multiple_selector_multiple_refs_from_none(): ], **MALWARE_KWARGS ) - before = markings.set_markings(before, ["description", "modified"], [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -837,17 +837,17 @@ def test_set_marking_mark_another_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.set_markings(before, ["description"], [MARKING_IDS[1], MARKING_IDS[2]]) + before = markings.set_markings(before, [MARKING_IDS[1], MARKING_IDS[2]], ["description"]) for m in before["granular_markings"]: assert m in after["granular_markings"] @pytest.mark.parametrize("marking", [ - (["foo"], [MARKING_IDS[4], MARKING_IDS[5]]), - ("", [MARKING_IDS[4], MARKING_IDS[5]]), - ([], [MARKING_IDS[4], MARKING_IDS[5]]), - ([""], [MARKING_IDS[4], MARKING_IDS[5]]), + ([MARKING_IDS[4], MARKING_IDS[5]], ["foo"]), + ([MARKING_IDS[4], MARKING_IDS[5]], ""), + ([MARKING_IDS[4], MARKING_IDS[5]], []), + ([MARKING_IDS[4], MARKING_IDS[5]], [""]), ]) def test_set_marking_bad_selector(marking): before = Malware( @@ -894,7 +894,7 @@ def test_set_marking_mark_same_property_same_marking(): ], **MALWARE_KWARGS ) - before = markings.set_markings(before, ["description"], [MARKING_IDS[0]]) + before = markings.set_markings(before, [MARKING_IDS[0]], ["description"]) for m in before["granular_markings"]: assert m in after["granular_markings"] diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index d7e3d31..0c6069a 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -4,7 +4,7 @@ import pytest import pytz import stix2 -from stix2 import TLP_WHITE, markings +from stix2 import TLP_WHITE from .constants import MARKING_DEFINITION_ID @@ -156,8 +156,6 @@ def test_campaign_with_granular_markings_example(): marking_ref="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", selectors=["description"]) ]) - print(markings.get_markings(campaign, None)) - print(markings.add_markings(campaign, None, "marking-definition--00000000-0000-0000-0000-000000000000")) assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index 54c06f4..f949042 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -26,7 +26,7 @@ def test_add_markings_one_marking(): **MALWARE_KWARGS ) - before = markings.add_markings(before, None, MARKING_IDS[0]) + before = markings.add_markings(before, MARKING_IDS[0], None) for m in before["object_marking_refs"]: assert m in after["object_marking_refs"] @@ -42,7 +42,7 @@ def test_add_markings_multiple_marking(): **MALWARE_KWARGS ) - before = markings.add_markings(before, None, [MARKING_IDS[0], MARKING_IDS[1]]) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], None) for m in before["object_marking_refs"]: assert m in after["object_marking_refs"] @@ -67,10 +67,10 @@ def test_add_markings_combination(): **MALWARE_KWARGS ) - before = markings.add_markings(before, None, MARKING_IDS[0]) - before = markings.add_markings(before, None, MARKING_IDS[1]) - before = markings.add_markings(before, "labels", MARKING_IDS[2]) - before = markings.add_markings(before, "name", MARKING_IDS[3]) + before = markings.add_markings(before, MARKING_IDS[0], None) + before = markings.add_markings(before, MARKING_IDS[1], None) + before = markings.add_markings(before, MARKING_IDS[2], "labels") + before = markings.add_markings(before, MARKING_IDS[3], "name") for m in before["granular_markings"]: assert m in after["granular_markings"] @@ -90,7 +90,7 @@ def test_add_markings_bad_markings(data): **MALWARE_KWARGS ) with pytest.raises(exceptions.InvalidValueError): - before = markings.add_markings(before, None, data) + before = markings.add_markings(before, data, None) assert "object_marking_refs" not in before @@ -251,13 +251,13 @@ def test_remove_markings_object_level(): **MALWARE_KWARGS ) - before = markings.remove_markings(before, None, MARKING_IDS[0]) + before = markings.remove_markings(before, MARKING_IDS[0], None) assert 'object_marking_refs' not in before assert 'object_marking_refs' not in after modified = after.modified - after = markings.remove_markings(after, None, MARKING_IDS[0]) + after = markings.remove_markings(after, MARKING_IDS[0], None) modified == after.modified @@ -271,7 +271,7 @@ def test_remove_markings_multiple(): **MALWARE_KWARGS ) - before = markings.remove_markings(before, None, [MARKING_IDS[0], MARKING_IDS[2]]) + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[2]], None) assert before['object_marking_refs'] == after['object_marking_refs'] @@ -282,7 +282,7 @@ def test_remove_markings_bad_markings(): **MALWARE_KWARGS ) with pytest.raises(AssertionError) as excinfo: - markings.remove_markings(before, None, [MARKING_IDS[4]]) + markings.remove_markings(before, [MARKING_IDS[4]], None) assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] @@ -370,78 +370,78 @@ def test_is_marked_object_and_granular_combinations(): ] } - assert markings.is_marked(test_sdo, "a", ["1"], False, False) - assert markings.is_marked(test_sdo, "a", ["1", "11"], True, False) - assert markings.is_marked(test_sdo, "a", ["1", "11"], True, True) - assert markings.is_marked(test_sdo, "a", ["1"], False, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, False) + assert markings.is_marked(test_sdo, ["1", "11"], "a", True, False) + assert markings.is_marked(test_sdo, ["1", "11"], "a", True, True) + assert markings.is_marked(test_sdo, ["1"], "a", False, True) assert markings.is_marked(test_sdo, "b", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "b", ["11"], True, False) - assert markings.is_marked(test_sdo, "b", ["11"], True, True) + assert markings.is_marked(test_sdo, ["11"], "b", True, False) + assert markings.is_marked(test_sdo, ["11"], "b", True, True) assert markings.is_marked(test_sdo, "b", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "c", ["2"], False, False) - assert markings.is_marked(test_sdo, "c", ["2", "11"], True, False) - assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5", "11"], True, True) - assert markings.is_marked(test_sdo, "c", ["2", "3", "4", "5"], False, True) + assert markings.is_marked(test_sdo, ["2"], "c", False, False) + assert markings.is_marked(test_sdo, ["2", "11"], "c", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5", "11"], "c", True, True) + assert markings.is_marked(test_sdo, ["2", "3", "4", "5"], "c", False, True) assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "c.[0]", ["2", "11"], True, False) - assert markings.is_marked(test_sdo, "c.[0]", ["2", "11"], True, True) + assert markings.is_marked(test_sdo, ["2", "11"], "c.[0]", True, False) + assert markings.is_marked(test_sdo, ["2", "11"], "c.[0]", True, True) assert markings.is_marked(test_sdo, "c.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, False) - assert markings.is_marked(test_sdo, "c.[1]", ["2", "3", "11"], True, False) - assert markings.is_marked(test_sdo, "c.[1]", ["2", "3", "11"], True, True) - assert markings.is_marked(test_sdo, "c.[1]", ["3"], False, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, False) + assert markings.is_marked(test_sdo, ["2", "3", "11"], "c.[1]", True, False) + assert markings.is_marked(test_sdo, ["2", "3", "11"], "c.[1]", True, True) + assert markings.is_marked(test_sdo, ["3"], "c.[1]", False, True) - assert markings.is_marked(test_sdo, "c.[2]", ["4"], False, False) - assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "11"], True, False) - assert markings.is_marked(test_sdo, "c.[2]", ["2", "4", "5", "11"], True, True) - assert markings.is_marked(test_sdo, "c.[2]", ["4", "5"], False, True) + assert markings.is_marked(test_sdo, ["4"], "c.[2]", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "11"], "c.[2]", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2]", True, True) + assert markings.is_marked(test_sdo, ["4", "5"], "c.[2]", False, True) - assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, False) - assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5", "11"], True, False) - assert markings.is_marked(test_sdo, "c.[2].g", ["2", "4", "5", "11"], True, True) - assert markings.is_marked(test_sdo, "c.[2].g", ["5"], False, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2].g", True, False) + assert markings.is_marked(test_sdo, ["2", "4", "5", "11"], "c.[2].g", True, True) + assert markings.is_marked(test_sdo, ["5"], "c.[2].g", False, True) - assert markings.is_marked(test_sdo, "x", ["6"], False, False) - assert markings.is_marked(test_sdo, "x", ["6", "11"], True, False) - assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10", "11"], True, True) - assert markings.is_marked(test_sdo, "x", ["6", "7", "8", "9", "10"], False, True) + assert markings.is_marked(test_sdo, ["6"], "x", False, False) + assert markings.is_marked(test_sdo, ["6", "11"], "x", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10", "11"], "x", True, True) + assert markings.is_marked(test_sdo, ["6", "7", "8", "9", "10"], "x", False, True) - assert markings.is_marked(test_sdo, "x.y", ["7"], False, False) - assert markings.is_marked(test_sdo, "x.y", ["6", "7", "11"], True, False) - assert markings.is_marked(test_sdo, "x.y", ["6", "7", "8", "11"], True, True) - assert markings.is_marked(test_sdo, "x.y", ["7", "8"], False, True) + assert markings.is_marked(test_sdo, ["7"], "x.y", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y", True, True) + assert markings.is_marked(test_sdo, ["7", "8"], "x.y", False, True) assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7", "11"], True, False) - assert markings.is_marked(test_sdo, "x.y.[0]", ["6", "7", "11"], True, True) + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y.[0]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "11"], "x.y.[0]", True, True) assert markings.is_marked(test_sdo, "x.y.[0]", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, False) - assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8", "11"], True, False) - assert markings.is_marked(test_sdo, "x.y.[1]", ["6", "7", "8", "11"], True, True) - assert markings.is_marked(test_sdo, "x.y.[1]", ["8"], False, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y.[1]", True, False) + assert markings.is_marked(test_sdo, ["6", "7", "8", "11"], "x.y.[1]", True, True) + assert markings.is_marked(test_sdo, ["8"], "x.y.[1]", False, True) - assert markings.is_marked(test_sdo, "x.z", ["9"], False, False) - assert markings.is_marked(test_sdo, "x.z", ["6", "9", "11"], True, False) - assert markings.is_marked(test_sdo, "x.z", ["6", "9", "10", "11"], True, True) - assert markings.is_marked(test_sdo, "x.z", ["9", "10"], False, True) + assert markings.is_marked(test_sdo, ["9"], "x.z", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z", True, True) + assert markings.is_marked(test_sdo, ["9", "10"], "x.z", False, True) assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=False) is False - assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9", "11"], True, False) - assert markings.is_marked(test_sdo, "x.z.foo1", ["6", "9", "11"], True, True) + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z.foo1", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "11"], "x.z.foo1", True, True) assert markings.is_marked(test_sdo, "x.z.foo1", inherited=False, descendants=True) is False - assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, False) - assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10", "11"], True, False) - assert markings.is_marked(test_sdo, "x.z.foo2", ["6", "9", "10", "11"], True, True) - assert markings.is_marked(test_sdo, "x.z.foo2", ["10"], False, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z.foo2", True, False) + assert markings.is_marked(test_sdo, ["6", "9", "10", "11"], "x.z.foo2", True, True) + assert markings.is_marked(test_sdo, ["10"], "x.z.foo2", False, True) - assert markings.is_marked(test_sdo, None, ["11"], True, True) - assert markings.is_marked(test_sdo, None, ["2"], True, True) is False + assert markings.is_marked(test_sdo, ["11"], None, True, True) + assert markings.is_marked(test_sdo, ["2"], None, True, True) is False def test_set_marking(): @@ -454,7 +454,7 @@ def test_set_marking(): **MALWARE_KWARGS ) - before = markings.set_markings(before, None, [MARKING_IDS[4], MARKING_IDS[5]]) + before = markings.set_markings(before, [MARKING_IDS[4], MARKING_IDS[5]], None) for m in before["object_marking_refs"]: assert m in [MARKING_IDS[4], MARKING_IDS[5]] @@ -481,6 +481,6 @@ def test_set_marking_bad_input(data): **MALWARE_KWARGS ) with pytest.raises(exceptions.InvalidValueError): - before = markings.set_markings(before, None, data) + before = markings.set_markings(before, data, None) assert before == after From 42962e6675b05e4c838cd67e6485a11c3072b7ab Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 1 Sep 2017 16:37:49 -0400 Subject: [PATCH 82/82] Improve markings tests - Test markings functions with both dictionaries and our _STIXBase-derived classes as input. - Slightly improve test coverage. - Drop Python 2.6 support. --- .travis.yml | 1 - setup.py | 1 - stix2/__init__.py | 2 +- stix2/common.py | 5 +- stix2/core.py | 5 +- stix2/markings/granular_markings.py | 11 +- stix2/markings/object_markings.py | 9 +- stix2/observables.py | 5 +- stix2/sdo.py | 5 +- stix2/sro.py | 5 +- stix2/test/constants.py | 7 +- stix2/test/test_granular_markings.py | 266 +++++++++++++++++---------- stix2/test/test_object_markings.py | 120 ++++++++---- stix2/utils.py | 3 +- tox.ini | 3 +- 15 files changed, 277 insertions(+), 171 deletions(-) diff --git a/.travis.yml b/.travis.yml index fdbb686..aba764d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ sudo: false language: python cache: pip python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/setup.py b/setup.py index 3687e57..e359147 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ setup( keywords="stix stix2 json cti cyber threat intelligence", packages=find_packages(), install_requires=[ - 'ordereddict ; python_version<"2.7"', 'python-dateutil', 'pytz', 'requests', diff --git a/stix2/__init__.py b/stix2/__init__.py index 6e89531..c2aae2e 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -42,5 +42,5 @@ from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, Vulnerability) from .sro import Relationship, Sighting -from .utils import get_dict +from .utils import get_dict, new_version, revoke from .version import __version__ diff --git a/stix2/common.py b/stix2/common.py index 6242989..a2e6918 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -1,9 +1,6 @@ """STIX 2 Common Data Types and Properties""" -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from .base import _STIXBase from .properties import (HashesProperty, IDProperty, ListProperty, Property, diff --git a/stix2/core.py b/stix2/core.py index 0d0d1d2..be2a53d 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -1,9 +1,6 @@ """STIX 2.0 Objects that are neither SDOs nor SROs""" -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from . import exceptions from .base import _STIXBase diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 3ccc1f0..7e9ccc7 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -1,6 +1,7 @@ from stix2 import exceptions from stix2.markings import utils +from stix2.utils import new_version def get_markings(obj, selectors, inherited=False, descendants=False): @@ -115,9 +116,9 @@ def remove_markings(obj, marking, selectors): granular_markings = utils.compress_markings(granular_markings) if granular_markings: - return obj.new_version(granular_markings=granular_markings) + return new_version(obj, granular_markings=granular_markings) else: - return obj.new_version(granular_markings=None) + return new_version(obj, granular_markings=None) def add_markings(obj, marking, selectors): @@ -153,7 +154,7 @@ def add_markings(obj, marking, selectors): granular_marking = utils.expand_markings(granular_marking) granular_marking = utils.compress_markings(granular_marking) - return obj.new_version(granular_markings=granular_marking) + return new_version(obj, granular_markings=granular_marking) def clear_markings(obj, selectors): @@ -208,9 +209,9 @@ def clear_markings(obj, selectors): granular_markings = utils.compress_markings(granular_markings) if granular_markings: - return obj.new_version(granular_markings=granular_markings) + return new_version(obj, granular_markings=granular_markings) else: - return obj.new_version(granular_markings=None) + return new_version(obj, granular_markings=None) def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index a3b2586..c39c036 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -1,6 +1,7 @@ from stix2 import exceptions from stix2.markings import utils +from stix2.utils import new_version def get_markings(obj): @@ -34,7 +35,7 @@ def add_markings(obj, marking): object_markings = set(obj.get("object_marking_refs", []) + marking) - return obj.new_version(object_marking_refs=list(object_markings)) + return new_version(obj, object_marking_refs=list(object_markings)) def remove_markings(obj, marking): @@ -66,9 +67,9 @@ def remove_markings(obj, marking): new_markings = [x for x in object_markings if x not in marking] if new_markings: - return obj.new_version(object_marking_refs=new_markings) + return new_version(obj, object_marking_refs=new_markings) else: - return obj.new_version(object_marking_refs=None) + return new_version(obj, object_marking_refs=None) def set_markings(obj, marking): @@ -100,7 +101,7 @@ def clear_markings(obj): A new version of the given SDO or SRO with object_marking_refs cleared. """ - return obj.new_version(object_marking_refs=None) + return new_version(obj, object_marking_refs=None) def is_marked(obj, marking=None): diff --git a/stix2/observables.py b/stix2/observables.py index b9dcf7f..4caaaa5 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -5,10 +5,7 @@ embedded in Email Message objects, inherit from _STIXBase instead of Observable and do not have a '_type' attribute. """ -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from .base import _Extension, _Observable, _STIXBase from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, diff --git a/stix2/sdo.py b/stix2/sdo.py index 8c27d11..77c781a 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -1,9 +1,6 @@ """STIX 2.0 Domain Objects""" -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict import stix2 diff --git a/stix2/sro.py b/stix2/sro.py index 05a29ed..af483bc 100644 --- a/stix2/sro.py +++ b/stix2/sro.py @@ -1,9 +1,6 @@ """STIX 2.0 Relationship Objects.""" -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from .base import _STIXBase from .common import ExternalReference, GranularMarking diff --git a/stix2/test/constants.py b/stix2/test/constants.py index c93f803..839b547 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -31,7 +31,8 @@ MARKING_IDS = [ # All required args for a Campaign instance, plus some optional args CAMPAIGN_MORE_KWARGS = dict( - id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + type='campaign', + id=CAMPAIGN_ID, created_by_ref="identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", created="2016-04-06T20:03:00.000Z", modified="2016-04-06T20:03:00.000Z", @@ -59,6 +60,10 @@ MALWARE_KWARGS = dict( # All required args for a Malware instance, plus some optional args MALWARE_MORE_KWARGS = dict( + type='malware', + id=MALWARE_ID, + created="2016-04-06T20:03:00.000Z", + modified="2016-04-06T20:03:00.000Z", labels=['ransomware'], name="Cryptolocker", description="A ransomware related to ..." diff --git a/stix2/test/test_granular_markings.py b/stix2/test/test_granular_markings.py index 2d3aa6a..e910ad3 100644 --- a/stix2/test/test_granular_markings.py +++ b/stix2/test/test_granular_markings.py @@ -3,17 +3,12 @@ import pytest from stix2 import Malware, markings -from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST +from .constants import MARKING_IDS """Tests for the Data Markings API.""" MALWARE_KWARGS = MALWARE_KWARGS_CONST.copy() -MALWARE_KWARGS.update({ - 'id': MALWARE_ID, - 'created': FAKE_TIME, - 'modified': FAKE_TIME, -}) def test_add_marking_mark_one_selector_multiple_refs(): @@ -39,19 +34,34 @@ def test_add_marking_mark_one_selector_multiple_refs(): assert m in after["granular_markings"] -def test_add_marking_mark_multiple_selector_one_refs(): - before = Malware( - **MALWARE_KWARGS - ) - after = Malware( - granular_markings=[ - { - "selectors": ["description", "name"], - "marking_ref": MARKING_IDS[0] - }, - ], - **MALWARE_KWARGS - ) +@pytest.mark.parametrize("data", [ + ( + Malware(**MALWARE_KWARGS), + Malware( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS), + ), + ( + MALWARE_KWARGS, + dict( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0] + }, + ], + **MALWARE_KWARGS), + ), +]) +def test_add_marking_mark_multiple_selector_one_refs(data): + before = data[0] + after = data[1] + before = markings.add_markings(before, [MARKING_IDS[0]], ["description", "name"]) for m in before["granular_markings"]: @@ -337,8 +347,8 @@ def test_get_markings_positional_arguments_combinations(data): assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) -def test_remove_marking_remove_one_selector_with_multiple_refs(): - before = Malware( +@pytest.mark.parametrize("before", [ + Malware( granular_markings=[ { "selectors": ["description"], @@ -347,10 +357,25 @@ def test_remove_marking_remove_one_selector_with_multiple_refs(): { "selectors": ["description"], "marking_ref": MARKING_IDS[1] - } + }, ], **MALWARE_KWARGS - ) + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + ], + **MALWARE_KWARGS + ), +]) +def test_remove_marking_remove_one_selector_with_multiple_refs(before): before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"]) assert "granular_markings" not in before @@ -501,49 +526,65 @@ def test_remove_marking_bad_selector(): markings.remove_markings(before, ["marking-definition--1", "marking-definition--2"], ["title"]) -IS_MARKED_TEST_DATA = { - "title": "test title", - "description": "test description", - "revision": 2, - "type": "test", - "granular_markings": [ - { - "selectors": ["description"], - "marking_ref": "marking-definition--1" - }, - { - "selectors": ["revision", "description"], - "marking_ref": "marking-definition--2" - }, - { - "selectors": ["revision", "description"], - "marking_ref": "marking-definition--3" - }, - ] -} +IS_MARKED_TEST_DATA = [ + Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[2] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[3] + }, + ], + **MALWARE_KWARGS + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[2] + }, + { + "selectors": ["labels", "description"], + "marking_ref": MARKING_IDS[3] + }, + ], + **MALWARE_KWARGS + ), +] -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_smoke(data): """Smoke test is_marked call does not fail.""" assert markings.is_marked(data, selectors=["description"]) - assert markings.is_marked(data, selectors=["title"]) is False + assert markings.is_marked(data, selectors=["modified"]) is False @pytest.mark.parametrize("data,selector", [ - (IS_MARKED_TEST_DATA, "foo"), - (IS_MARKED_TEST_DATA, ""), - (IS_MARKED_TEST_DATA, []), - (IS_MARKED_TEST_DATA, [""]), - (IS_MARKED_TEST_DATA, "x.z.[-2]"), - (IS_MARKED_TEST_DATA, "c.f"), - (IS_MARKED_TEST_DATA, "c.[2].i"), - (IS_MARKED_TEST_DATA, "c.[3]"), - (IS_MARKED_TEST_DATA, "d"), - (IS_MARKED_TEST_DATA, "x.[0]"), - (IS_MARKED_TEST_DATA, "z.y.w"), - (IS_MARKED_TEST_DATA, "x.z.[1]"), - (IS_MARKED_TEST_DATA, "x.z.foo3") + (IS_MARKED_TEST_DATA[0], "foo"), + (IS_MARKED_TEST_DATA[0], ""), + (IS_MARKED_TEST_DATA[0], []), + (IS_MARKED_TEST_DATA[0], [""]), + (IS_MARKED_TEST_DATA[0], "x.z.[-2]"), + (IS_MARKED_TEST_DATA[0], "c.f"), + (IS_MARKED_TEST_DATA[0], "c.[2].i"), + (IS_MARKED_TEST_DATA[1], "c.[3]"), + (IS_MARKED_TEST_DATA[1], "d"), + (IS_MARKED_TEST_DATA[1], "x.[0]"), + (IS_MARKED_TEST_DATA[1], "z.y.w"), + (IS_MARKED_TEST_DATA[1], "x.z.[1]"), + (IS_MARKED_TEST_DATA[1], "x.z.foo3") ]) def test_is_marked_invalid_selector(data, selector): """Test invalid selector raises an error.""" @@ -551,46 +592,54 @@ def test_is_marked_invalid_selector(data, selector): markings.is_marked(data, selectors=selector) -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_mix_selector(data): """Test valid selector, one marked and one not marked returns True.""" - assert markings.is_marked(data, selectors=["description", "revision"]) + assert markings.is_marked(data, selectors=["description", "labels"]) assert markings.is_marked(data, selectors=["description"]) -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_valid_selector_no_refs(data): """Test that a valid selector return True when it has marking refs and False when not.""" assert markings.is_marked(data, selectors=["description"]) - assert markings.is_marked(data, ["marking-definition--2", "marking-definition--3"], ["description"]) - assert markings.is_marked(data, ["marking-definition--2"], ["description"]) - assert markings.is_marked(data, ["marking-definition--2", "marking-definition--8"], ["description"]) is False + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[3]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[2]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[5]], ["description"]) is False -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_valid_selector_and_refs(data): """Test that a valid selector returns True when marking_refs match.""" - assert markings.is_marked(data, ["marking-definition--1"], ["description"]) - assert markings.is_marked(data, ["marking-definition--1"], ["title"]) is False + assert markings.is_marked(data, [MARKING_IDS[1]], ["description"]) + assert markings.is_marked(data, [MARKING_IDS[1]], ["modified"]) is False -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_valid_selector_multiple_refs(data): """Test that a valid selector returns True if aall marking_refs match. Otherwise False.""" - assert markings.is_marked(data, ["marking-definition--2", "marking-definition--3"], ["revision"]) - assert markings.is_marked(data, ["marking-definition--2", "marking-definition--1"], ["revision"]) is False - assert markings.is_marked(data, "marking-definition--2", ["revision"]) - assert markings.is_marked(data, ["marking-definition--1234"], ["revision"]) is False + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[3]], ["labels"]) + assert markings.is_marked(data, [MARKING_IDS[2], MARKING_IDS[1]], ["labels"]) is False + assert markings.is_marked(data, MARKING_IDS[2], ["labels"]) + assert markings.is_marked(data, ["marking-definition--1234"], ["labels"]) is False -@pytest.mark.parametrize("data", [IS_MARKED_TEST_DATA]) +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) def test_is_marked_no_marking_refs(data): """Test that a valid content selector with no marking_refs returns True if there is a granular_marking that asserts that field, False otherwise.""" assert markings.is_marked(data, selectors=["type"]) is False - assert markings.is_marked(data, selectors=["revision"]) + assert markings.is_marked(data, selectors=["labels"]) + + +@pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) +def test_is_marked_no_selectors(data): + """Test that we're ensuring 'selectors' is provided.""" + with pytest.raises(TypeError) as excinfo: + markings.granular_markings.is_marked(data) + assert "'selectors' must be provided" in str(excinfo.value) def test_is_marked_positional_arguments_combinations(): @@ -899,47 +948,66 @@ def test_set_marking_mark_same_property_same_marking(): assert m in after["granular_markings"] -CLEAR_MARKINGS_TEST_DATA = Malware( - granular_markings=[ - { - "selectors": ["description"], - "marking_ref": MARKING_IDS[0] - }, - { - "selectors": ["modified", "description"], - "marking_ref": MARKING_IDS[1] - }, - { - "selectors": ["modified", "description", "type"], - "marking_ref": MARKING_IDS[2] - }, - ], - **MALWARE_KWARGS -) +CLEAR_MARKINGS_TEST_DATA = [ + Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified", "description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["modified", "description", "type"], + "marking_ref": MARKING_IDS[2] + }, + ], + **MALWARE_KWARGS + ), + dict( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0] + }, + { + "selectors": ["modified", "description"], + "marking_ref": MARKING_IDS[1] + }, + { + "selectors": ["modified", "description", "type"], + "marking_ref": MARKING_IDS[2] + }, + ], + **MALWARE_KWARGS + ) +] -@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) def test_clear_marking_smoke(data): """Test clear_marking call does not fail.""" data = markings.clear_markings(data, "modified") assert markings.is_marked(data, "modified") is False -@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) def test_clear_marking_multiple_selectors(data): """Test clearing markings for multiple selectors effectively removes associated markings.""" data = markings.clear_markings(data, ["type", "description"]) assert markings.is_marked(data, ["type", "description"]) is False -@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) def test_clear_marking_one_selector(data): """Test markings associated with one selector were removed.""" data = markings.clear_markings(data, "description") assert markings.is_marked(data, "description") is False -@pytest.mark.parametrize("data", [CLEAR_MARKINGS_TEST_DATA]) +@pytest.mark.parametrize("data", CLEAR_MARKINGS_TEST_DATA) def test_clear_marking_all_selectors(data): data = markings.clear_markings(data, ["description", "type", "modified"]) assert markings.is_marked(data, "description") is False @@ -947,10 +1015,10 @@ def test_clear_marking_all_selectors(data): @pytest.mark.parametrize("data,selector", [ - (CLEAR_MARKINGS_TEST_DATA, "foo"), - (CLEAR_MARKINGS_TEST_DATA, ""), - (CLEAR_MARKINGS_TEST_DATA, []), - (CLEAR_MARKINGS_TEST_DATA, [""]), + (CLEAR_MARKINGS_TEST_DATA[0], "foo"), + (CLEAR_MARKINGS_TEST_DATA[0], ""), + (CLEAR_MARKINGS_TEST_DATA[1], []), + (CLEAR_MARKINGS_TEST_DATA[1], [""]), ]) def test_clear_marking_bad_selector(data, selector): """Test bad selector raises exception.""" diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py index f949042..36e8e4d 100644 --- a/stix2/test/test_object_markings.py +++ b/stix2/test/test_object_markings.py @@ -16,15 +16,21 @@ MALWARE_KWARGS.update({ }) -def test_add_markings_one_marking(): - before = Malware( - **MALWARE_KWARGS - ) - - after = Malware( - object_marking_refs=[MARKING_IDS[0]], - **MALWARE_KWARGS - ) +@pytest.mark.parametrize("data", [ + ( + Malware(**MALWARE_KWARGS), + Malware(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + ), + ( + MALWARE_KWARGS, + dict(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + ), +]) +def test_add_markings_one_marking(data): + before = data[0] + after = data[1] before = markings.add_markings(before, MARKING_IDS[0], None) @@ -242,34 +248,49 @@ def test_get_markings_object_and_granular_combinations(data): assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) -def test_remove_markings_object_level(): - before = Malware( - object_marking_refs=[MARKING_IDS[0]], - **MALWARE_KWARGS - ) - after = Malware( - **MALWARE_KWARGS - ) +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_remove_markings_object_level(data): + before = data[0] + after = data[1] before = markings.remove_markings(before, MARKING_IDS[0], None) assert 'object_marking_refs' not in before assert 'object_marking_refs' not in after - modified = after.modified + modified = after['modified'] after = markings.remove_markings(after, MARKING_IDS[0], None) - modified == after.modified + modified == after['modified'] -def test_remove_markings_multiple(): - before = Malware( - object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], - **MALWARE_KWARGS - ) - after = Malware( - object_marking_refs=[MARKING_IDS[1]], - **MALWARE_KWARGS - ) +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(object_marking_refs=[MARKING_IDS[1]], + **MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + dict(object_marking_refs=[MARKING_IDS[1]], + **MALWARE_KWARGS), + ), +]) +def test_remove_markings_multiple(data): + before = data[0] + after = data[1] before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[2]], None) @@ -286,14 +307,21 @@ def test_remove_markings_bad_markings(): assert str(excinfo.value) == "Marking ['%s'] was not found in Malware!" % MARKING_IDS[4] -def test_clear_markings(): - before = Malware( - object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], - **MALWARE_KWARGS - ) - after = Malware( - **MALWARE_KWARGS - ) +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_clear_markings(data): + before = data[0] + after = data[1] before = markings.clear_markings(before, None) @@ -444,6 +472,26 @@ def test_is_marked_object_and_granular_combinations(): assert markings.is_marked(test_sdo, ["2"], None, True, True) is False +@pytest.mark.parametrize("data", [ + ( + Malware(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + Malware(**MALWARE_KWARGS), + ), + ( + dict(object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], + **MALWARE_KWARGS), + MALWARE_KWARGS, + ), +]) +def test_is_marked_no_markings(data): + marked = data[0] + nonmarked = data[1] + + assert markings.is_marked(marked) + assert markings.is_marked(nonmarked) is False + + def test_set_marking(): before = Malware( object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]], diff --git a/stix2/utils.py b/stix2/utils.py index 72a9c8e..ca195f6 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -191,7 +191,8 @@ def new_version(data, **kwargs): if new_modified_property < old_modified_property: raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") new_obj_inner.update(kwargs) - return cls(**new_obj_inner) + # Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass + return cls(**{k: v for k, v in new_obj_inner.items() if v is not None}) def revoke(data): diff --git a/tox.ini b/tox.ini index b1265ec..bca18c3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,py35,py36,pycodestyle,isort-check +envlist = py27,py33,py34,py35,py36,pycodestyle,isort-check [testenv] deps = @@ -36,7 +36,6 @@ commands = [travis] python = - 2.6: py26 2.7: py27, pycodestyle 3.3: py33, pycodestyle 3.4: py34, pycodestyle