From 4bbabaecb2a2275a891c91f77cd94828be7ba9a3 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 22 Apr 2019 15:25:46 -0400 Subject: [PATCH 1/7] update marking API methods to allow/use the 'lang' property including utility methods that expand collapse markings --- stix2/markings/__init__.py | 42 ++++++++++----- stix2/markings/granular_markings.py | 84 ++++++++++++++++++++--------- stix2/markings/utils.py | 31 ++++++++--- 3 files changed, 110 insertions(+), 47 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 79d1012..b0dba6b 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -22,7 +22,7 @@ Note: from stix2.markings import granular_markings, object_markings -def get_markings(obj, selectors=None, inherited=False, descendants=False): +def get_markings(obj, selectors=None, inherited=False, descendants=False, marking_ref=True, lang=True): """ Get all markings associated to the field(s) specified by selectors. @@ -30,10 +30,13 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False): obj: An SDO or SRO object. selectors: string or list of selectors strings relative to the SDO or SRO in which the properties appear. - inherited: If True, include object level markings and granular markings - inherited relative to the properties. - descendants: If True, include granular markings applied to any children - relative to the properties. + inherited (bool): If True, include object level markings and granular + markings inherited relative to the properties. + descendants (bool): If True, include granular markings applied to any + children relative to the properties. + marking_ref (bool): If False, excludes markings that use + ``marking_ref`` property. + lang (bool): If False, excludes markings that use ``lang`` property. Returns: list: Marking identifiers that matched the selectors expression. @@ -51,6 +54,8 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False): selectors, inherited, descendants, + marking_ref, + lang ) if inherited: @@ -59,7 +64,7 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False): return list(set(results)) -def set_markings(obj, marking, selectors=None): +def set_markings(obj, marking, selectors=None, marking_ref=True, lang=True): """ Remove all markings associated with selectors and appends a new granular marking. Refer to `clear_markings` and `add_markings` for details. @@ -70,6 +75,10 @@ def set_markings(obj, marking, selectors=None): properties selected by `selectors`. selectors: string or list of selectors strings relative to the SDO or SRO in which the properties appear. + marking_ref (bool): If False, markings that use the ``marking_ref`` + property will not be removed. + lang (bool): If False, markings that use the ``lang`` property + will not be removed. Returns: A new version of the given SDO or SRO with specified markings removed @@ -83,7 +92,7 @@ def set_markings(obj, marking, selectors=None): if selectors is None: return object_markings.set_markings(obj, marking) else: - return granular_markings.set_markings(obj, marking, selectors) + return granular_markings.set_markings(obj, marking, selectors, marking_ref, lang) def remove_markings(obj, marking, selectors=None): @@ -144,7 +153,7 @@ def add_markings(obj, marking, selectors=None): return granular_markings.add_markings(obj, marking, selectors) -def clear_markings(obj, selectors=None): +def clear_markings(obj, selectors=None, marking_ref=True, lang=True): """ Remove all markings associated with the selectors. @@ -152,6 +161,10 @@ def clear_markings(obj, selectors=None): obj: An SDO or SRO object. selectors: string or list of selectors strings relative to the SDO or SRO in which the field(s) appear(s). + marking_ref (bool): If False, markings that use the ``marking_ref`` + property will not be removed. + lang (bool): If False, markings that use the ``lang`` property + will not be removed. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -169,7 +182,7 @@ def clear_markings(obj, selectors=None): if selectors is None: return object_markings.clear_markings(obj) else: - return granular_markings.clear_markings(obj, selectors) + return granular_markings.clear_markings(obj, selectors, marking_ref, lang) def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): @@ -182,10 +195,11 @@ def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=Fa properties selected by `selectors`. selectors: string or list of selectors strings relative to the SDO or SRO in which the field(s) appear(s). - inherited: If True, include object level markings and granular markings - inherited to determine if the properties is/are marked. - descendants: If True, include granular markings applied to any children - of the given selector to determine if the properties is/are marked. + inherited (bool): If True, include object level markings and granular + markings inherited to determine if the properties is/are marked. + descendants (bool): If True, include granular markings applied to any + children of the given selector to determine if the properties + is/are marked. Returns: bool: True if ``selectors`` is found on internal SDO or SRO collection. @@ -228,7 +242,7 @@ def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=Fa return result -class _MarkingsMixin(): +class _MarkingsMixin(object): pass diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 09c3d37..a1c8479 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -2,10 +2,10 @@ from stix2 import exceptions from stix2.markings import utils -from stix2.utils import new_version +from stix2.utils import new_version, is_marking -def get_markings(obj, selectors, inherited=False, descendants=False): +def get_markings(obj, selectors, inherited=False, descendants=False, marking_ref=True, lang=True): """ Get all granular markings associated to with the properties. @@ -13,10 +13,13 @@ def get_markings(obj, selectors, inherited=False, descendants=False): obj: An SDO or SRO object. selectors: string or list of selector strings relative to the SDO or SRO in which the properties appear. - inherited: If True, include markings inherited relative to the + inherited (bool): If True, include markings inherited relative to the properties. - descendants: If True, include granular markings applied to any children - relative to the properties. + descendants (bool): If True, include granular markings applied to any + children relative to the properties. + marking_ref (bool): If False, excludes markings that use + ``marking_ref`` property. + lang (bool): If False, excludes markings that use ``lang`` property. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -43,13 +46,18 @@ def get_markings(obj, selectors, inherited=False, descendants=False): (user_selector.startswith(marking_selector) and inherited), # Catch inherited selectors. (marking_selector.startswith(user_selector) and descendants), ]): # Catch descendants selectors - refs = marking.get('marking_ref', []) - results.update([refs]) + ref = marking.get('marking_ref') + lng = marking.get('lang') + + if ref and marking_ref: + results.add(ref) + if lng and lang: + results.add(lng) return list(results) -def set_markings(obj, marking, selectors): +def set_markings(obj, marking, selectors, marking_ref=True, lang=True): """ Remove all granular markings associated with selectors and append a new granular marking. Refer to `clear_markings` and `add_markings` for details. @@ -60,19 +68,25 @@ def set_markings(obj, marking, selectors): SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + marking_ref (bool): If False, markings that use the ``marking_ref`` + property will not be removed. + lang (bool): If False, markings that use the ``lang`` property + will not be removed. Returns: A new version of the given SDO or SRO with specified markings removed and new ones added. """ - obj = clear_markings(obj, selectors) + obj = clear_markings(obj, selectors, marking_ref, lang) return add_markings(obj, marking, selectors) def remove_markings(obj, marking, selectors): """ - Remove a granular marking from the granular_markings collection. + Remove a granular marking from the granular_markings collection. The method + makes a best-effort attempt to distinguish between a marking-definition + or language granular marking. Args: obj: An SDO or SRO object. @@ -103,7 +117,10 @@ def remove_markings(obj, marking, selectors): to_remove = [] for m in marking: - to_remove.append({'marking_ref': m, 'selectors': selectors}) + if is_marking(m): + to_remove.append({'marking_ref': m, 'selectors': selectors}) + else: + to_remove.append({'lang': m, 'selectors': selectors}) remove = utils.build_granular_marking(to_remove).get('granular_markings') @@ -124,7 +141,9 @@ def remove_markings(obj, marking, selectors): def add_markings(obj, marking, selectors): """ - Append a granular marking to the granular_markings collection. + Append a granular marking to the granular_markings collection. The method + makes a best-effort attempt to distinguish between a marking-definition + or language granular marking. Args: obj: An SDO or SRO object. @@ -146,7 +165,10 @@ def add_markings(obj, marking, selectors): granular_marking = [] for m in marking: - granular_marking.append({'marking_ref': m, 'selectors': sorted(selectors)}) + if is_marking(m): + granular_marking.append({'marking_ref': m, 'selectors': sorted(selectors)}) + else: + granular_marking.append({'lang': m, 'selectors': sorted(selectors)}) if obj.get('granular_markings'): granular_marking.extend(obj.get('granular_markings')) @@ -156,7 +178,7 @@ def add_markings(obj, marking, selectors): return new_version(obj, granular_markings=granular_marking, allow_custom=True) -def clear_markings(obj, selectors): +def clear_markings(obj, selectors, marking_ref=True, lang=True): """ Remove all granular markings associated with the selectors. @@ -164,6 +186,10 @@ def clear_markings(obj, selectors): obj: An SDO or SRO object. selectors: string or list of selectors strings relative to the SDO or SRO in which the properties appear. + marking_ref (bool): If False, markings that use the ``marking_ref`` + property will not be removed. + lang (bool): If False, markings that use the ``lang`` property + will not be removed. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -184,11 +210,12 @@ def clear_markings(obj, selectors): granular_markings = utils.expand_markings(granular_markings) - sdo = utils.build_granular_marking( - [{'selectors': selectors, 'marking_ref': 'N/A'}], - ) + granular_dict = utils.build_granular_marking([ + {'selectors': selectors, 'marking_ref': 'N/A'}, + {'selectors': selectors, 'lang': 'N/A'} + ]) - clear = sdo.get('granular_markings', []) + clear = granular_dict.get('granular_markings', []) if not any( clear_selector in sdo_selectors.get('selectors', []) @@ -201,10 +228,13 @@ def clear_markings(obj, selectors): for granular_marking in granular_markings: for s in selectors: if s in granular_marking.get('selectors', []): - marking_refs = granular_marking.get('marking_ref') + ref = granular_marking.get('marking_ref') + lng = granular_marking.get('lang') - if marking_refs: + if ref and marking_ref: granular_marking['marking_ref'] = '' + if lng and lang: + granular_marking['lang'] = '' granular_markings = utils.compress_markings(granular_markings) @@ -222,11 +252,12 @@ def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=Fa obj: An SDO or SRO object. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. - inherited: If True, return markings inherited from the given selector. - descendants: If True, return granular markings applied to any children - of the given selector. + selectors (bool): string or list of selectors strings relative to the + SDO or SRO in which the properties appear. + inherited (bool): If True, return markings inherited from the given + selector. + descendants (bool): If True, return granular markings applied to any + children of the given selector. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -262,9 +293,12 @@ def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=Fa (marking_selector.startswith(user_selector) and descendants), ]): # Catch descendants selectors marking_ref = granular_marking.get('marking_ref', '') + lang = granular_marking.get('lang', '') if marking and any(x == marking_ref for x in marking): markings.update([marking_ref]) + if marking and any(x == lang for x in marking): + markings.update([lang]) marked = True diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index d8bbf1d..7a6aae1 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -5,6 +5,7 @@ import collections import six from stix2 import exceptions +from stix2.utils import is_marking def _evaluate_expression(obj, selector): @@ -121,10 +122,15 @@ def compress_markings(granular_markings): if granular_marking.get('marking_ref'): map_[granular_marking.get('marking_ref')].update(granular_marking.get('selectors')) + if granular_marking.get('lang'): + map_[granular_marking.get('lang')].update(granular_marking.get('selectors')) + compressed = \ [ - {'marking_ref': marking_ref, 'selectors': sorted(selectors)} - for marking_ref, selectors in six.iteritems(map_) + {'marking_ref': item, 'selectors': sorted(selectors)} + if is_marking(item) else + {'lang': item, 'selectors': sorted(selectors)} + for item, selectors in six.iteritems(map_) ] return compressed @@ -174,13 +180,22 @@ def expand_markings(granular_markings): for marking in granular_markings: selectors = marking.get('selectors') marking_ref = marking.get('marking_ref') + lang = marking.get('lang') - expanded.extend( - [ - {'marking_ref': marking_ref, 'selectors': [selector]} - for selector in selectors - ], - ) + if marking_ref: + expanded.extend( + [ + {'marking_ref': marking_ref, 'selectors': [selector]} + for selector in selectors + ], + ) + if lang: + expanded.extend( + [ + {'lang': lang, 'selectors': [selector]} + for selector in selectors + ], + ) return expanded From 0c78acafd0d1e5c90088085767b3317c19d5e8fe Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 22 Apr 2019 15:26:21 -0400 Subject: [PATCH 2/7] add tests to cover the language aspect of the markings --- stix2/test/v21/constants.py | 6 + stix2/test/v21/test_granular_markings.py | 221 ++++++++++++++++++++++- stix2/test/v21/test_markings.py | 48 ++++- 3 files changed, 272 insertions(+), 3 deletions(-) diff --git a/stix2/test/v21/constants.py b/stix2/test/v21/constants.py index bbce32c..b0ba1ef 100644 --- a/stix2/test/v21/constants.py +++ b/stix2/test/v21/constants.py @@ -31,6 +31,12 @@ MARKING_IDS = [ "marking-definition--68520ae2-fefe-43a9-84ee-2c2a934d2c7d", "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f", ] +MARKING_LANGS = [ + "en", + "es", + "de", + "ja", +] RELATIONSHIP_IDS = [ 'relationship--06520621-5352-4e6a-b976-e8fa3d437ffd', 'relationship--181c9c09-43e6-45dd-9374-3bec192f05ef', diff --git a/stix2/test/v21/test_granular_markings.py b/stix2/test/v21/test_granular_markings.py index 9f7234e..5d70063 100644 --- a/stix2/test/v21/test_granular_markings.py +++ b/stix2/test/v21/test_granular_markings.py @@ -5,7 +5,7 @@ from stix2.exceptions import MarkingNotFoundError from stix2.v21 import TLP_RED, Malware from .constants import MALWARE_MORE_KWARGS as MALWARE_KWARGS_CONST -from .constants import MARKING_IDS +from .constants import MARKING_IDS, MARKING_LANGS """Tests for the Data Markings API.""" @@ -111,6 +111,37 @@ def test_add_marking_mark_multiple_selector_multiple_refs(): assert m in after["granular_markings"] +def test_add_marking_mark_multiple_selector_multiple_refs_mixed(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[0], + }, + { + "selectors": ["description", "name"], + "marking_ref": MARKING_IDS[1], + }, + { + "selectors": ["description", "name"], + "lang": MARKING_LANGS[0], + }, + { + "selectors": ["description", "name"], + "lang": MARKING_LANGS[1], + }, + ], + **MALWARE_KWARGS + ) + before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1], MARKING_LANGS[0], MARKING_LANGS[1]], ["description", "name"]) + + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + def test_add_marking_mark_another_property_same_marking(): before = Malware( granular_markings=[ @@ -376,6 +407,98 @@ def test_get_markings_positional_arguments_combinations(data): assert set(markings.get_markings(data, "x.z.foo2", False, True)) == set(["10"]) +GET_MARKINGS_TEST_DATA_LANGS = { + "a": 333, + "b": "value", + "c": [ + 17, + "list value", + { + "g": "nested", + "h": 45, + }, + ], + "x": { + "y": [ + "hello", + 88, + ], + "z": { + "foo1": "bar", + "foo2": 65, + }, + }, + "granular_markings": [ + { + "marking_ref": "m1", + "selectors": ["a"], + }, + { + "marking_ref": "m2", + "selectors": ["c"], + }, + { + "marking_ref": "m3", + "selectors": ["c.[1]"], + }, + { + "marking_ref": "m4", + "selectors": ["c.[2]"], + }, + { + "marking_ref": "m5", + "selectors": ["c.[2].g"], + }, + { + "marking_ref": "m6", + "selectors": ["x"], + }, + { + "lang": "l7", + "selectors": ["x.y"], + }, + { + "marking_ref": "m8", + "selectors": ["x.y.[1]"], + }, + { + "lang": "l9", + "selectors": ["x.z"], + }, + { + "marking_ref": "m9", + "selectors": ["x.z"], + }, + { + "marking_ref": "m10", + "selectors": ["x.z.foo2"], + }, + ], +} + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA_LANGS]) +def test_get_markings_multiple_selectors(data): + """Test multiple selectors return combination of markings.""" + total = markings.get_markings(data, ["x.y", "x.z"]) + xy_markings = markings.get_markings(data, ["x.y"]) + xz_markings = markings.get_markings(data, ["x.z"]) + + assert set(xy_markings).issubset(total) + assert set(xz_markings).issubset(total) + assert set(xy_markings).union(xz_markings).issuperset(total) + + +@pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA_LANGS]) +def test_get_markings_multiple_selectors_with_options(data): + """Test multiple selectors return combination of markings.""" + total = markings.get_markings(data, ["x.y", "x.z"], lang=False) + xz_markings = markings.get_markings(data, ["x.z"], marking_ref=False) + + assert len(total) == 1 + assert len(xz_markings) == 1 + + @pytest.mark.parametrize( "data", [ ( @@ -455,6 +578,38 @@ def test_remove_marking_mark_one_selector_from_multiple_ones(): assert m in after["granular_markings"] +def test_remove_marking_mark_one_selector_from_multiple_ones_mixed(): + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "marking_ref": MARKING_IDS[0], + }, + { + "selectors": ["description"], + "lang": MARKING_LANGS[0], + }, + ], + **MALWARE_KWARGS + ) + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[0], + }, + { + "selectors": ["description", "modified"], + "lang": MARKING_LANGS[0], + }, + ], + **MALWARE_KWARGS + ) + before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_LANGS[0]], ["modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + def test_remove_marking_mark_one_selector_markings_from_multiple_ones(): after = Malware( granular_markings=[ @@ -592,6 +747,10 @@ IS_MARKED_TEST_DATA = [ "selectors": ["malware_types", "description"], "marking_ref": MARKING_IDS[3], }, + { + "selectors": ["name"], + "lang": MARKING_LANGS[1], + }, ], **MALWARE_KWARGS ), @@ -609,6 +768,10 @@ IS_MARKED_TEST_DATA = [ "selectors": ["malware_types", "description"], "marking_ref": MARKING_IDS[3], }, + { + "selectors": ["name"], + "lang": MARKING_LANGS[1], + }, ], **MALWARE_KWARGS ), @@ -620,6 +783,7 @@ def test_is_marked_smoke(data): """Smoke test is_marked call does not fail.""" assert markings.is_marked(data, selectors=["description"]) assert markings.is_marked(data, selectors=["modified"]) is False + assert markings.is_marked(data, selectors=["name"]) @pytest.mark.parametrize( @@ -666,6 +830,7 @@ def test_is_marked_valid_selector_and_refs(data): """Test that a valid selector returns True when marking_refs match.""" assert markings.is_marked(data, [MARKING_IDS[1]], ["description"]) assert markings.is_marked(data, [MARKING_IDS[1]], ["modified"]) is False + assert markings.is_marked(data, [MARKING_LANGS[1]], ["name"]) @pytest.mark.parametrize("data", IS_MARKED_TEST_DATA) @@ -870,6 +1035,28 @@ def test_set_marking_mark_one_selector_multiple_refs(): assert m in after["granular_markings"] +def test_set_marking_mark_one_selector_multiple_lang_refs(): + before = Malware( + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description"], + "lang": MARKING_LANGS[0], + }, + { + "selectors": ["description"], + "lang": MARKING_LANGS[1], + }, + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_LANGS[0], MARKING_LANGS[1]], ["description"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + def test_set_marking_mark_multiple_selector_one_refs(): before = Malware( granular_markings=[ @@ -894,6 +1081,38 @@ def test_set_marking_mark_multiple_selector_one_refs(): assert m in after["granular_markings"] +def test_set_marking_mark_multiple_mixed_markings(): + before = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[1], + }, + { + "selectors": ["description", "modified"], + "lang": MARKING_LANGS[2], + }, + ], + **MALWARE_KWARGS + ) + after = Malware( + granular_markings=[ + { + "selectors": ["description", "modified"], + "marking_ref": MARKING_IDS[2], + }, + { + "selectors": ["description", "modified"], + "lang": MARKING_LANGS[3], + }, + ], + **MALWARE_KWARGS + ) + before = markings.set_markings(before, [MARKING_IDS[2], MARKING_LANGS[3]], ["description", "modified"]) + for m in before["granular_markings"]: + assert m in after["granular_markings"] + + def test_set_marking_mark_multiple_selector_multiple_refs_from_none(): before = Malware( **MALWARE_KWARGS diff --git a/stix2/test/v21/test_markings.py b/stix2/test/v21/test_markings.py index 7793889..7e0cf91 100644 --- a/stix2/test/v21/test_markings.py +++ b/stix2/test/v21/test_markings.py @@ -54,7 +54,7 @@ EXPECTED_GRANULAR_MARKING = """{ ] }""" -EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ +EXPECTED_CAMPAIGN_WITH_GRANULAR_REF_MARKINGS = """{ "type": "campaign", "spec_version": "2.1", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", @@ -74,6 +74,27 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS = """{ }""" +EXPECTED_CAMPAIGN_WITH_GRANULAR_LANG_MARKINGS = """{ + "type": "campaign", + "spec_version": "2.1", + "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + "created_by_ref": "identity--311b2d2d-f010-4473-83ec-1edf84858f4c", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "name": "Bank Attack", + "description": "Weitere Informationen über Banküberfall", + "lang": "en", + "granular_markings": [ + { + "lang": "de", + "selectors": [ + "description" + ] + } + ] +}""" + + def test_marking_def_example_with_tlp(): assert str(TLP_WHITE) == EXPECTED_TLP_MARKING_DEFINITION @@ -161,7 +182,7 @@ def test_campaign_with_granular_markings_example(): ), ], ) - assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_MARKINGS + assert str(campaign) == EXPECTED_CAMPAIGN_WITH_GRANULAR_REF_MARKINGS @pytest.mark.parametrize( @@ -273,3 +294,26 @@ def test_campaign_add_markings(): ) campaign = campaign.add_markings(TLP_WHITE) assert campaign.object_marking_refs[0] == TLP_WHITE.id + + +def test_campaign_with_granular_lang_markings_example(): + campaign = stix2.v21.Campaign( + id="campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", + created_by_ref=IDENTITY_ID, + created="2016-04-06T20:03:00Z", + modified="2016-04-06T20:03:00Z", + name="Bank Attack", + lang="en", + description="Weitere Informationen über Banküberfall", + granular_markings=[ + stix2.v21.GranularMarking( + lang="de", + selectors=["description"], + ), + ], + ) + + # In order to provide the same representation, we need to disable escaping + # in json.dumps(). https://docs.python.org/3/library/json.html#json.dumps + # or https://docs.python.org/2/library/json.html#json.dumps + assert campaign.serialize(pretty=True, ensure_ascii=False) == EXPECTED_CAMPAIGN_WITH_GRANULAR_LANG_MARKINGS From dbc63b7b9fcb71b9b9f0fbeee1591e363a660736 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 23 Apr 2019 07:43:56 -0400 Subject: [PATCH 3/7] pre-commit changes --- stix2/markings/__init__.py | 2 +- stix2/markings/granular_markings.py | 4 ++-- stix2/test/v21/test_granular_markings.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index b0dba6b..6d09f81 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -55,7 +55,7 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False, markin inherited, descendants, marking_ref, - lang + lang, ) if inherited: diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index a1c8479..5456f83 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -2,7 +2,7 @@ from stix2 import exceptions from stix2.markings import utils -from stix2.utils import new_version, is_marking +from stix2.utils import is_marking, new_version def get_markings(obj, selectors, inherited=False, descendants=False, marking_ref=True, lang=True): @@ -212,7 +212,7 @@ def clear_markings(obj, selectors, marking_ref=True, lang=True): granular_dict = utils.build_granular_marking([ {'selectors': selectors, 'marking_ref': 'N/A'}, - {'selectors': selectors, 'lang': 'N/A'} + {'selectors': selectors, 'lang': 'N/A'}, ]) clear = granular_dict.get('granular_markings', []) diff --git a/stix2/test/v21/test_granular_markings.py b/stix2/test/v21/test_granular_markings.py index 5d70063..e178f86 100644 --- a/stix2/test/v21/test_granular_markings.py +++ b/stix2/test/v21/test_granular_markings.py @@ -478,7 +478,7 @@ GET_MARKINGS_TEST_DATA_LANGS = { @pytest.mark.parametrize("data", [GET_MARKINGS_TEST_DATA_LANGS]) -def test_get_markings_multiple_selectors(data): +def test_get_markings_multiple_selectors_langs(data): """Test multiple selectors return combination of markings.""" total = markings.get_markings(data, ["x.y", "x.z"]) xy_markings = markings.get_markings(data, ["x.y"]) From f8857569d5c189c8d8267e8290052b927a97d7c9 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 23 Apr 2019 07:48:51 -0400 Subject: [PATCH 4/7] Add header to test file --- stix2/test/v21/test_markings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stix2/test/v21/test_markings.py b/stix2/test/v21/test_markings.py index 7e0cf91..a7dff22 100644 --- a/stix2/test/v21/test_markings.py +++ b/stix2/test/v21/test_markings.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import datetime as dt import pytest From c3aecd76ba37f7c1d3fbc33f8b59175a578df2f8 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 23 Apr 2019 09:27:21 -0400 Subject: [PATCH 5/7] update unnecesary property clean-up and add tests --- stix2/properties.py | 20 +++++++++++--------- stix2/test/v21/test_markings.py | 2 +- stix2/test/v21/test_properties.py | 5 +++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 5de7a1e..f4060e5 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -183,11 +183,12 @@ class CallableValues(list): class StringProperty(Property): def __init__(self, **kwargs): - self.string_type = text_type super(StringProperty, self).__init__(**kwargs) def clean(self, value): - return self.string_type(value) + if not isinstance(value, string_types): + return text_type(value) + return value class TypeProperty(Property): @@ -439,21 +440,22 @@ class EnumProperty(StringProperty): super(EnumProperty, self).__init__(**kwargs) def clean(self, value): - value = super(EnumProperty, self).clean(value) - if value not in self.allowed: - raise ValueError("value '{}' is not valid for this enumeration.".format(value)) - return self.string_type(value) + cleaned_value = super(EnumProperty, self).clean(value) + if cleaned_value not in self.allowed: + raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value)) + + return cleaned_value class PatternProperty(StringProperty): def clean(self, value): - str_value = super(PatternProperty, self).clean(value) - errors = run_validator(str_value) + cleaned_value = super(PatternProperty, self).clean(value) + errors = run_validator(cleaned_value) if errors: raise ValueError(str(errors[0])) - return self.string_type(value) + return cleaned_value class ObservableProperty(Property): diff --git a/stix2/test/v21/test_markings.py b/stix2/test/v21/test_markings.py index a7dff22..bd247e6 100644 --- a/stix2/test/v21/test_markings.py +++ b/stix2/test/v21/test_markings.py @@ -76,7 +76,7 @@ EXPECTED_CAMPAIGN_WITH_GRANULAR_REF_MARKINGS = """{ }""" -EXPECTED_CAMPAIGN_WITH_GRANULAR_LANG_MARKINGS = """{ +EXPECTED_CAMPAIGN_WITH_GRANULAR_LANG_MARKINGS = u"""{ "type": "campaign", "spec_version": "2.1", "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py index 298a8df..e4fa4a0 100644 --- a/stix2/test/v21/test_properties.py +++ b/stix2/test/v21/test_properties.py @@ -450,6 +450,11 @@ def test_enum_property_valid(value): assert enum_prop.clean('b') +def test_enum_property_clean(): + enum_prop = EnumProperty(['1']) + assert enum_prop.clean(1) == '1' + + def test_enum_property_invalid(): enum_prop = EnumProperty(['a', 'b', 'c']) with pytest.raises(ValueError): From 8d842aeb94e626ea60a43c3bd12bfd5c005e2548 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 May 2019 14:48:16 -0400 Subject: [PATCH 6/7] update user guide on marking extraction via API --- docs/guide/markings.ipynb | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 8230daf..270bfcd 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -1310,6 +1310,85 @@ "source": [ "malware.is_marked(TLP_WHITE.id, 'description')" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Extracting lang Data Markings or marking-definition Data Markings\n", + "\n", + "If you need a specific kind of marking, you can also filter them using the API. By default the library will get both types of markings by default. You can choose between `lang=True/False` or `marking_ref=True/False` depending on your use-case." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"spec_version\": \"2.1\",\n", + " \"id\": \"indicator--634ef462-d6b5-48bc-9d9f-b46a6919227c\",\n", + " \"created\": \"2019-05-03T18:36:44.354Z\",\n", + " \"modified\": \"2019-05-03T18:36:44.354Z\",\n", + " \"description\": \"Una descripcion sobre este indicador\",\n", + " \"indicator_types\": [\n", + " \"malware\"\n", + " ],\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2019-05-03T18:36:44.354443Z\",\n", + " \"object_marking_refs\": [\n", + " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", + " ],\n", + " \"granular_markings\": [\n", + " {\n", + " \"lang\": \"es\",\n", + " \"selectors\": [\n", + " \"description\"\n", + " ]\n", + " },\n", + " {\n", + " \"marking_ref\": \"marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da\",\n", + " \"selectors\": [\n", + " \"description\"\n", + " ]\n", + " }\n", + " ]\n", + "}\n", + "['es', 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']\n", + "['marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da']\n", + "['es']\n" + ] + } + ], + "source": [ + "from stix2 import v21\n", + "\n", + "v21_indicator = v21.Indicator(\n", + " description=\"Una descripcion sobre este indicador\",\n", + " pattern=\"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " object_marking_refs=['marking-definition--f88d31f6-486f-44da-b317-01333bde0b82'],\n", + " indicator_types=['malware'],\n", + " granular_markings=[\n", + " {\n", + " 'selectors': ['description'],\n", + " 'lang': 'es'\n", + " },\n", + " {\n", + " 'selectors': ['description'],\n", + " 'marking_ref': 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da'\n", + " }\n", + " ]\n", + ")\n", + "print(v21_indicator)\n", + "print(v21_indicator.get_markings('description')) # Gets both lang and marking_ref markings for 'description'\n", + "print(v21_indicator.get_markings('description', lang=False)) # Exclude lang markings from results\n", + "print(v21_indicator.get_markings('description', marking_ref=False)) # Exclude marking-definition markings from results" + ] } ], "metadata": { From 851ed3e85ad09538f46478de247bd681b83d8680 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 May 2019 15:41:58 -0400 Subject: [PATCH 7/7] marking-definition --- docs/guide/markings.ipynb | 135 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 4 deletions(-) diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 270bfcd..44e023a 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -1315,7 +1315,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Extracting lang Data Markings or marking-definition Data Markings\n", + "# Extracting Lang Data Markings or marking-definition Data Markings\n", "\n", "If you need a specific kind of marking, you can also filter them using the API. By default the library will get both types of markings by default. You can choose between `lang=True/False` or `marking_ref=True/False` depending on your use-case." ] @@ -1385,9 +1385,136 @@ " ]\n", ")\n", "print(v21_indicator)\n", - "print(v21_indicator.get_markings('description')) # Gets both lang and marking_ref markings for 'description'\n", - "print(v21_indicator.get_markings('description', lang=False)) # Exclude lang markings from results\n", - "print(v21_indicator.get_markings('description', marking_ref=False)) # Exclude marking-definition markings from results" + "\n", + "# Gets both lang and marking_ref markings for 'description'\n", + "print(v21_indicator.get_markings('description'))\n", + "\n", + "# Exclude lang markings from results\n", + "print(v21_indicator.get_markings('description', lang=False))\n", + "\n", + "# Exclude marking-definition markings from results\n", + "print(v21_indicator.get_markings('description', marking_ref=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this same manner, calls to `clear_markings` and `set_markings` also have the ability to operate in for one or both types of markings." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"spec_version\": \"2.1\",\n", + " \"id\": \"indicator--a612665a-2df4-4fd2-851c-7fbb8c92339a\",\n", + " \"created\": \"2019-05-03T19:13:59.010Z\",\n", + " \"modified\": \"2019-05-03T19:15:41.173Z\",\n", + " \"description\": \"Una descripcion sobre este indicador\",\n", + " \"indicator_types\": [\n", + " \"malware\"\n", + " ],\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2019-05-03T19:13:59.010624Z\",\n", + " \"object_marking_refs\": [\n", + " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "print(v21_indicator.clear_markings(\"description\")) # By default, both types of markings will be removed" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"spec_version\": \"2.1\",\n", + " \"id\": \"indicator--982aeb4d-4dd3-4b04-aa50-a1d00c31986c\",\n", + " \"created\": \"2019-05-03T19:19:26.542Z\",\n", + " \"modified\": \"2019-05-03T19:20:51.818Z\",\n", + " \"description\": \"Una descripcion sobre este indicador\",\n", + " \"indicator_types\": [\n", + " \"malware\"\n", + " ],\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2019-05-03T19:19:26.542267Z\",\n", + " \"object_marking_refs\": [\n", + " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", + " ],\n", + " \"granular_markings\": [\n", + " {\n", + " \"lang\": \"es\",\n", + " \"selectors\": [\n", + " \"description\"\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "# If lang is False, no lang markings will be removed\n", + "print(v21_indicator.clear_markings(\"description\", lang=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"indicator\",\n", + " \"spec_version\": \"2.1\",\n", + " \"id\": \"indicator--de0316d6-38e1-43c2-af4f-649305251864\",\n", + " \"created\": \"2019-05-03T19:40:21.459Z\",\n", + " \"modified\": \"2019-05-03T19:40:26.431Z\",\n", + " \"description\": \"Una descripcion sobre este indicador\",\n", + " \"indicator_types\": [\n", + " \"malware\"\n", + " ],\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2019-05-03T19:40:21.459582Z\",\n", + " \"object_marking_refs\": [\n", + " \"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82\"\n", + " ],\n", + " \"granular_markings\": [\n", + " {\n", + " \"marking_ref\": \"marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da\",\n", + " \"selectors\": [\n", + " \"description\"\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "# If marking_ref is False, no marking-definition markings will be removed\n", + "print(v21_indicator.clear_markings(\"description\", marking_ref=False))" ] } ],