From 8f58f29358bdb8686ac8388052bf666f771fe2c9 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:16:04 -0400 Subject: [PATCH 01/23] Moved from test_datastore.py -> new file test_datastore_filesystem.py --- stix2/test/test_datastore_filesystem.py | 530 ++++++++++++++++++++++++ 1 file changed, 530 insertions(+) create mode 100644 stix2/test/test_datastore_filesystem.py diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py new file mode 100644 index 0000000..bd1f02c --- /dev/null +++ b/stix2/test/test_datastore_filesystem.py @@ -0,0 +1,530 @@ +import json +import os +import shutil + +import pytest + +from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, + FileSystemSource, FileSystemStore, Filter, Identity, + Indicator, Malware, Relationship, properties) + +from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, + IDENTITY_KWARGS, INDICATOR_ID, + INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS, + RELATIONSHIP_IDS) + +FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") + + +@pytest.fixture +def fs_store(): + # create + yield FileSystemStore(FS_PATH) + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_source(): + # create + fs = FileSystemSource(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_sink(): + # create + fs = FileSystemSink(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def bad_json_files(): + # create erroneous JSON files for tests to make sure handled gracefully + + with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f: + f.write("Im not a JSON file") + + with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f: + f.write("Im not a JSON formatted file") + + yield True # dummy yield so can have teardown + + os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt")) + os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json")) + + +@pytest.fixture +def bad_stix_files(): + # create erroneous STIX JSON files for tests to make sure handled correctly + + # bad STIX object + stix_obj = { + "id": "intrusion-set--test-bad-stix", + "spec_version": "2.0" + # no "type" field + } + + with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f: + f.write(json.dumps(stix_obj)) + + yield True # dummy yield so can have teardown + + os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json")) + + +@pytest.fixture(scope='module') +def rel_fs_store(): + cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) + idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) + ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) + mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) + rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) + rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) + rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) + stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] + fs = FileSystemStore(FS_PATH) + for o in stix_objs: + fs.add(o) + yield fs + + for o in stix_objs: + os.remove(os.path.join(FS_PATH, o.type, o.id + '.json')) + + +def test_filesystem_source_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSource('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + +def test_filesystem_sink_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSink('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + +def test_filesystem_source_bad_json_file(fs_source, bad_json_files): + # this tests the handling of two bad json files + # - one file should just be skipped (silently) as its a ".txt" extension + # - one file should be parsed and raise Exception bc its not JSON + try: + fs_source.get("intrusion-set--test-bad-json") + except TypeError as e: + assert "intrusion-set--test-bad-json" in str(e) + assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e) + + +def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files): + # this tests handling of bad STIX json object + try: + fs_source.get("intrusion-set--test-non-stix") + except TypeError as e: + assert "intrusion-set--test-non-stix" in str(e) + assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e) + + +def test_filesytem_source_get_object(fs_source): + # get object + mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" + assert mal.name == "Rover" + + +def test_filesytem_source_get_nonexistent_object(fs_source): + ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert ind is None + + +def test_filesytem_source_all_versions(fs_source): + # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) + id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") + assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" + assert id_.name == "The MITRE Corporation" + assert id_.type == "identity" + + +def test_filesytem_source_query_single(fs_source): + # query2 + is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) + assert len(is_2) == 1 + + is_2 = is_2[0] + assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" + assert is_2.type == "attack-pattern" + + +def test_filesytem_source_query_multiple(fs_source): + # query + intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) + assert len(intrusion_sets) == 2 + assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] + assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] + + is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] + assert "DragonOK" in is_1.aliases + assert len(is_1.external_references) == 4 + + +def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): + # add python stix object + camp1 = Campaign(name="Hannibal", + objective="Targeting Italian and Spanish Diplomat internet accounts", + aliases=["War Elephant"]) + + fs_sink.add(camp1) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) + + camp1_r = fs_source.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == "Hannibal" + assert "War Elephant" in camp1_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): + # add stix object dict + camp2 = { + "name": "Aurelius", + "type": "campaign", + "objective": "German and French Intelligence Services", + "aliases": ["Purple Robes"], + "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add(camp2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) + + camp2_r = fs_source.get(camp2["id"]) + assert camp2_r.id == camp2["id"] + assert camp2_r.name == camp2["name"] + assert "Purple Robes" in camp2_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) + + +def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): + # add stix bundle dict + bund = { + "type": "bundle", + "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", + "spec_version": "2.0", + "objects": [ + { + "name": "Atilla", + "type": "campaign", + "objective": "Bulgarian, Albanian and Romanian Intelligence Services", + "aliases": ["Huns"], + "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + ] + } + + fs_sink.add(bund) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) + + camp3_r = fs_source.get(bund["objects"][0]["id"]) + assert camp3_r.id == bund["objects"][0]["id"] + assert camp3_r.name == bund["objects"][0]["name"] + assert "Huns" in camp3_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) + + +def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): + # add json-encoded stix obj + camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' + + fs_sink.add(camp4) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") + assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + assert camp4_r.name == "Ghengis Khan" + + os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) + + +def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): + # add json-encoded stix bundle + bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ + ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' + fs_sink.add(bund2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") + assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + assert camp5_r.name == "Spartacus" + + os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) + + +def test_filesystem_sink_add_objects_list(fs_sink, fs_source): + # add list of objects + camp6 = Campaign(name="Comanche", + objective="US Midwest manufacturing firms, oil refineries, and businesses", + aliases=["Horse Warrior"]) + + camp7 = { + "name": "Napolean", + "type": "campaign", + "objective": "Central and Eastern Europe military commands and departments", + "aliases": ["The Frenchmen"], + "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add([camp6, camp7]) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp6_r = fs_source.get(camp6.id) + assert camp6_r.id == camp6.id + assert "Horse Warrior" in camp6_r.aliases + + camp7_r = fs_source.get(camp7["id"]) + assert camp7_r.id == camp7["id"] + assert "The Frenchmen" in camp7_r.aliases + + # remove all added objects + os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) + os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) + + +def test_filesystem_store_get_stored_as_bundle(fs_store): + coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f") + assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f" + assert coa.type == "course-of-action" + + +def test_filesystem_store_get_stored_as_object(fs_store): + coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") + assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" + assert coa.type == "course-of-action" + + +def test_filesystem_store_all_versions(fs_store): + # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) + rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] + assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" + assert rel.type == "relationship" + + +def test_filesystem_store_query(fs_store): + # query() + tools = fs_store.query([Filter("labels", "in", "tool")]) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_query_single_filter(fs_store): + query = Filter("labels", "in", "tool") + tools = fs_store.query(query) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_empty_query(fs_store): + results = fs_store.query() # returns all + assert len(results) == 26 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] + assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] + + +def test_filesystem_store_query_multiple_filters(fs_store): + fs_store.source.filters.add(Filter("labels", "in", "tool")) + tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966")) + assert len(tools) == 1 + assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966" + + +def test_filesystem_store_query_dont_include_type_folder(fs_store): + results = fs_store.query(Filter("type", "!=", "tool")) + assert len(results) == 24 + + +def test_filesystem_store_add(fs_store): + # add() + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + # remove + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_store_add_as_bundle(): + fs_store = FileSystemStore(FS_PATH, bundlify=True) + + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: + assert '"type": "bundle"' in bundle_file.read() + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +def test_filesystem_add_bundle_object(fs_store): + bundle = Bundle() + fs_store.add(bundle) + + +def test_filesystem_store_add_invalid_object(fs_store): + ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a') # tuple isn't valid + with pytest.raises(TypeError) as excinfo: + fs_store.add(ind) + assert 'stix_data must be' in str(excinfo.value) + assert 'a STIX object' in str(excinfo.value) + assert 'JSON formatted STIX' in str(excinfo.value) + assert 'JSON formatted STIX bundle' in str(excinfo.value) + + +def test_filesystem_object_with_custom_property(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + fs_store.add(camp, True) + + camp_r = fs_store.get(camp.id) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_object_with_custom_property_in_bundle(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + bundle = Bundle(camp, allow_custom=True) + fs_store.add(bundle) + + camp_r = fs_store.get(camp.id) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_custom_object(fs_store): + @CustomObject('x-new-obj', [ + ('property1', properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + newobj = NewObj(property1='something') + fs_store.add(newobj) + + newobj_r = fs_store.get(newobj.id) + assert newobj_r.id == newobj.id + assert newobj_r.property1 == 'something' + + # remove dir + shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) + + +def test_relationships(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.relationships(mal) + + assert len(resp) == 3 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_type(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.relationships(mal, relationship_type='indicates') + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[0] + + +def test_relationships_by_source(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[1] + + +def test_relationships_by_target(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_type(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True) + + assert len(resp) == 1 + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_source(rel_fs_store): + with pytest.raises(ValueError) as excinfo: + rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True) + + assert 'not both' in str(excinfo.value) + + +def test_related_to(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.related_to(mal) + + assert len(resp) == 3 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_source(rel_fs_store): + resp = rel_fs_store.related_to(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_target(rel_fs_store): + resp = rel_fs_store.related_to(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) From 5c4472cbbdd945e5c4cd6bd981d4e7b18d9656fd Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:17:19 -0400 Subject: [PATCH 02/23] Moved from test_datastore.py -> new file test_datastore_filters.py --- stix2/test/test_datastore_filters.py | 280 +++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 stix2/test/test_datastore_filters.py diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py new file mode 100644 index 0000000..4f66ff5 --- /dev/null +++ b/stix2/test/test_datastore_filters.py @@ -0,0 +1,280 @@ +import pytest + +from stix2.datastore.filters import Filter, apply_common_filters + + +def test_filter_ops_check(): + # invalid filters - non supported operators + + with pytest.raises(ValueError) as excinfo: + # create Filter that has an operator that is not allowed + Filter('modified', '*', 'not supported operator') + assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'" + + with pytest.raises(ValueError) as excinfo: + Filter("type", "%", "4") + assert "Filter operator '%' not supported for specified property" in str(excinfo.value) + + +def test_filter_value_type_check(): + # invalid filters - non supported value types + + with pytest.raises(TypeError) as excinfo: + Filter('created', '=', object()) + # On Python 2, the type of object() is `` On Python 3, it's ``. + assert any([s in str(excinfo.value) for s in ["", "''"]]) + assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + Filter("type", "=", complex(2, -1)) + assert any([s in str(excinfo.value) for s in ["", "''"]]) + assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + Filter("type", "=", set([16, 23])) + assert any([s in str(excinfo.value) for s in ["", "''"]]) + assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) + + +def test_filter_type_underscore_check(): + # check that Filters where property="type", value (name) doesnt have underscores + with pytest.raises(ValueError) as excinfo: + Filter("type", "=", "oh_underscore") + assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value) + + +def test_apply_common_filters(): + stix_objs = [ + { + "created": "2017-01-27T13:49:53.997Z", + "description": "\n\nTITLE:\n\tPoison Ivy", + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "labels": [ + "remote-access-trojan" + ], + "modified": "2017-01-27T13:49:53.997Z", + "name": "Poison Ivy", + "type": "malware" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "labels": [ + "file-hash-watchlist" + ], + "modified": "2014-05-08T09:00:00.000Z", + "name": "File hash for Poison Ivy variant", + "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "type": "indicator", + "valid_from": "2014-05-08T09:00:00.000000Z" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "granular_markings": [ + { + "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + "selectors": [ + "relationship_type" + ] + } + ], + "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", + "modified": "2014-05-08T09:00:00.000Z", + "object_marking_refs": [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + ], + "relationship_type": "indicates", + "revoked": True, + "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "type": "relationship" + }, + { + "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", + "created": "2016-02-14T00:00:00.000Z", + "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", + "modified": "2016-02-14T00:00:00.000Z", + "type": "vulnerability", + "name": "CVE-2014-0160", + "description": "The (1) TLS...", + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2014-0160" + } + ], + "labels": ["heartbleed", "has-logo"] + } + ] + + filters = [ + Filter("type", "!=", "relationship"), + Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), + Filter("labels", "in", "remote-access-trojan"), + Filter("created", ">", "2015-01-01T01:00:00.000Z"), + Filter("revoked", "=", True), + Filter("revoked", "!=", True), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "relationship_type"), + Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), + Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), + Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "description"), + Filter("external_references.source_name", "=", "CVE"), + ] + + # "Return any object whose type is not relationship" + resp = list(apply_common_filters(stix_objs, [filters[0]])) + ids = [r['id'] for r in resp] + assert stix_objs[0]['id'] in ids + assert stix_objs[1]['id'] in ids + assert stix_objs[3]['id'] in ids + assert len(ids) == 3 + + # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" + resp = list(apply_common_filters(stix_objs, [filters[1]])) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object that contains remote-access-trojan in labels" + resp = list(apply_common_filters(stix_objs, [filters[2]])) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 1 + + # "Return any object created after 2015-01-01T01:00:00.000Z" + resp = list(apply_common_filters(stix_objs, [filters[3]])) + assert resp[0]['id'] == stix_objs[0]['id'] + assert len(resp) == 2 + + # "Return any revoked object" + resp = list(apply_common_filters(stix_objs, [filters[4]])) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object whose not revoked" + # Note that if 'revoked' property is not present in object. + # Currently we can't use such an expression to filter for... :( + resp = list(apply_common_filters(stix_objs, [filters[5]])) + assert len(resp) == 0 + + # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs" + resp = list(apply_common_filters(stix_objs, [filters[6]])) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object that contains relationship_type in their selectors AND + # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref" + resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]])) + assert resp[0]['id'] == stix_objs[2]['id'] + assert len(resp) == 1 + + # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id" + resp = list(apply_common_filters(stix_objs, [filters[9]])) + assert resp[0]['id'] == stix_objs[3]['id'] + assert len(resp) == 1 + + # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9" + resp = list(apply_common_filters(stix_objs, [filters[10]])) + assert len(resp) == 1 + + # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None) + resp = list(apply_common_filters(stix_objs, [filters[11]])) + assert len(resp) == 0 + + # "Return any object that contains description in its selectors" (None) + resp = list(apply_common_filters(stix_objs, [filters[12]])) + assert len(resp) == 0 + + # "Return any object that object that matches CVE in source_name" (None, case sensitive) + resp = list(apply_common_filters(stix_objs, [filters[13]])) + assert len(resp) == 0 + + +def test_filters0(): + # "Return any object modified before 2017-01-28T13:49:53.935Z" + resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + +def test_filters1(): + # "Return any object modified after 2017-01-28T13:49:53.935Z" + resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + +def test_filters2(): + # "Return any object modified after or on 2017-01-28T13:49:53.935Z" + resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + +def test_filters3(): + # "Return any object modified before or on 2017-01-28T13:49:53.935Z" + resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) + assert resp[0]['id'] == STIX_OBJS2[1]['id'] + assert len(resp) == 2 + + +def test_filters4(): + # Assert invalid Filter cannot be created + with pytest.raises(ValueError) as excinfo: + Filter("modified", "?", "2017-01-27T13:49:53.935Z") + assert str(excinfo.value) == ("Filter operator '?' not supported " + "for specified property: 'modified'") + + +def test_filters5(): + # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 1 + + +def test_filters6(): + # Test filtering on non-common property + resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + +def test_filters7(): + # Test filtering on embedded property + stix_objects = list(STIX_OBJS2) + [{ + "type": "observed-data", + "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 50, + "objects": { + "0": { + "type": "file", + "hashes": { + "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + }, + "extensions": { + "pdf-ext": { + "version": "1.7", + "document_info_dict": { + "Title": "Sample document", + "Author": "Adobe Systems Incorporated", + "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh", + "Producer": "Acrobat Distiller 3.01 for Power Macintosh", + "CreationDate": "20070412090123-02" + }, + "pdfid0": "DFCE52BD827ECF765649852119D", + "pdfid1": "57A1E0F9ED2AE523E313C" + } + } + } + } + }] + resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) + assert resp[0]['id'] == stix_objects[3]['id'] + assert len(resp) == 1 From 59ff0c4f26d71aed662bee4ce1bab691f9530af6 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:17:49 -0400 Subject: [PATCH 03/23] Moved from test_datastore.py -> new file test_datastore_memory.py --- stix2/test/test_datastore_memory.py | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 stix2/test/test_datastore_memory.py diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py new file mode 100644 index 0000000..b027640 --- /dev/null +++ b/stix2/test/test_datastore_memory.py @@ -0,0 +1,86 @@ +import pytest + +from stix2.datastore import CompositeDataSource, make_id +from stix2.datastore.memory import MemorySource, MemorySink + + +def test_add_remove_composite_datasource(): + cds = CompositeDataSource() + ds1 = MemorySource() + ds2 = MemorySource() + ds3 = MemorySink() + + with pytest.raises(TypeError) as excinfo: + cds.add_data_sources([ds1, ds2, ds1, ds3]) + assert str(excinfo.value) == ("DataSource (to be added) is not of type " + "stix2.DataSource. DataSource type is ''") + + cds.add_data_sources([ds1, ds2, ds1]) + + assert len(cds.get_all_data_sources()) == 2 + + cds.remove_data_sources([ds1.id, ds2.id]) + + assert len(cds.get_all_data_sources()) == 0 + + +def test_composite_datasource_operations(): + BUNDLE1 = dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS1, + spec_version="2.0", + type="bundle") + cds1 = CompositeDataSource() + ds1_1 = MemorySource(stix_data=BUNDLE1) + ds1_2 = MemorySource(stix_data=STIX_OBJS2) + + cds2 = CompositeDataSource() + ds2_1 = MemorySource(stix_data=BUNDLE1) + ds2_2 = MemorySource(stix_data=STIX_OBJS2) + + cds1.add_data_sources([ds1_1, ds1_2]) + cds2.add_data_sources([ds2_1, ds2_2]) + + indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + # In STIX_OBJS2 changed the 'modified' property to a later time... + assert len(indicators) == 2 + + cds1.add_data_sources([cds2]) + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + query1 = [ + Filter("type", "=", "indicator") + ] + + query2 = [ + Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") + ] + + cds1.filters.update(query2) + + results = cds1.query(query1) + + # STIX_OBJS2 has indicator with later time, one with different id, one with + # original time in STIX_OBJS1 + assert len(results) == 3 + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + # There is only one indicator with different ID. Since we use the same data + # when deduplicated, only two indicators (one with different modified). + results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(results) == 2 + + # Since we have filters already associated with our CompositeSource providing + # nothing returns the same as cds1.query(query1) (the associated query is query2) + results = cds1.query([]) + assert len(results) == 3 From c2e7cbb3e3c3676f14f24212b863a013f0429eb0 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:18:05 -0400 Subject: [PATCH 04/23] Moved from test_datastore.py -> new file test_datastore_taxii.py --- stix2/test/test_datastore_taxii.py | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 stix2/test/test_datastore_taxii.py diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py new file mode 100644 index 0000000..0a293b7 --- /dev/null +++ b/stix2/test/test_datastore_taxii.py @@ -0,0 +1,78 @@ +COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' + + +class MockTAXIIClient(object): + """Mock for taxii2_client.TAXIIClient""" + pass + + +@pytest.fixture +def collection(): + return Collection(COLLECTION_URL, MockTAXIIClient()) + + +def test_ds_taxii(collection): + ds = taxii.TAXIICollectionSource(collection) + assert ds.collection is not None + + +def test_ds_taxii_name(collection): + ds = taxii.TAXIICollectionSource(collection) + assert ds.collection is not None + + +def test_parse_taxii_filters(): + query = [ + Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), + Filter("id", "=", "taxii stix object ID"), + Filter("type", "=", "taxii stix object ID"), + Filter("version", "=", "first"), + Filter("created_by_ref", "=", "Bane"), + ] + + taxii_filters_expected = set([ + Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), + Filter("id", "=", "taxii stix object ID"), + Filter("type", "=", "taxii stix object ID"), + Filter("version", "=", "first") + ]) + + ds = taxii.TAXIICollectionSource(collection) + + taxii_filters = ds._parse_taxii_filters(query) + + assert taxii_filters == taxii_filters_expected + + +def test_add_get_remove_filter(): + ds = taxii.TAXIICollectionSource(collection) + + # First 3 filters are valid, remaining properties are erroneous in some way + valid_filters = [ + Filter('type', '=', 'malware'), + Filter('id', '!=', 'stix object id'), + Filter('labels', 'in', ["heartbleed", "malicious-activity"]), + ] + + assert len(ds.filters) == 0 + + ds.filters.add(valid_filters[0]) + assert len(ds.filters) == 1 + + # Addin the same filter again will have no effect since `filters` uses a set + ds.filters.add(valid_filters[0]) + assert len(ds.filters) == 1 + + ds.filters.add(valid_filters[1]) + assert len(ds.filters) == 2 + ds.filters.add(valid_filters[2]) + assert len(ds.filters) == 3 + + assert set(valid_filters) == ds.filters + + # remove + ds.filters.remove(valid_filters[0]) + + assert len(ds.filters) == 2 + + ds.filters.update(valid_filters) From 80e4706ac4ea45ca9ced0d61b90889f420a966c1 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:19:51 -0400 Subject: [PATCH 05/23] Add some test objs previously in test_datastore.py --- stix2/test/conftest.py | 106 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py index 9f61bc2..acbb8b0 100644 --- a/stix2/test/conftest.py +++ b/stix2/test/conftest.py @@ -46,3 +46,109 @@ def malware(uuid4, clock): @pytest.fixture def relationship(uuid4, clock): return stix2.Relationship(**RELATIONSHIP_KWARGS) + + +@pytest.fixture +def stix1_objs(): + ind1 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind2 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind3 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind4 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind5 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + return [ind1, ind2, ind3, ind4, ind5] + + +@pytest.fixture +def stix2_objs(): + ind6 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind7 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ind8 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + return [ind6, ind7, ind8] From 0d1729bbd7000d7c5f4d9eaf661743cf12d11587 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:20:39 -0400 Subject: [PATCH 06/23] Deleted test_filesystem in favor of test_datastore_filesystem.py --- stix2/test/test_filesystem.py | 529 ---------------------------------- 1 file changed, 529 deletions(-) delete mode 100644 stix2/test/test_filesystem.py diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py deleted file mode 100644 index f59136e..0000000 --- a/stix2/test/test_filesystem.py +++ /dev/null @@ -1,529 +0,0 @@ -import json -import os -import shutil - -import pytest - -from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, - FileSystemSource, FileSystemStore, Filter, Identity, - Indicator, Malware, Relationship, properties) - -from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, - IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, - MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS) - -FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") - - -@pytest.fixture -def fs_store(): - # create - yield FileSystemStore(FS_PATH) - - # remove campaign dir - shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) - - -@pytest.fixture -def fs_source(): - # create - fs = FileSystemSource(FS_PATH) - assert fs.stix_dir == FS_PATH - yield fs - - # remove campaign dir - shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) - - -@pytest.fixture -def fs_sink(): - # create - fs = FileSystemSink(FS_PATH) - assert fs.stix_dir == FS_PATH - yield fs - - # remove campaign dir - shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) - - -@pytest.fixture -def bad_json_files(): - # create erroneous JSON files for tests to make sure handled gracefully - - with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt"), "w+") as f: - f.write("Im not a JSON file") - - with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json"), "w+") as f: - f.write("Im not a JSON formatted file") - - yield True # dummy yield so can have teardown - - os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-json.txt")) - os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-bad-json.json")) - - -@pytest.fixture -def bad_stix_files(): - # create erroneous STIX JSON files for tests to make sure handled correctly - - # bad STIX object - stix_obj = { - "id": "intrusion-set--test-bad-stix", - "spec_version": "2.0" - # no "type" field - } - - with open(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json"), "w+") as f: - f.write(json.dumps(stix_obj)) - - yield True # dummy yield so can have teardown - - os.remove(os.path.join(FS_PATH, "intrusion-set", "intrusion-set--test-non-stix.json")) - - -@pytest.fixture(scope='module') -def rel_fs_store(): - cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) - idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) - ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) - mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) - rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) - rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) - rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) - stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] - fs = FileSystemStore(FS_PATH) - for o in stix_objs: - fs.add(o) - yield fs - - for o in stix_objs: - os.remove(os.path.join(FS_PATH, o.type, o.id + '.json')) - - -def test_filesystem_source_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: - FileSystemSource('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) - - -def test_filesystem_sink_nonexistent_folder(): - with pytest.raises(ValueError) as excinfo: - FileSystemSink('nonexistent-folder') - assert "for STIX data does not exist" in str(excinfo) - - -def test_filesystem_source_bad_json_file(fs_source, bad_json_files): - # this tests the handling of two bad json files - # - one file should just be skipped (silently) as its a ".txt" extension - # - one file should be parsed and raise Exception bc its not JSON - try: - fs_source.get("intrusion-set--test-bad-json") - except TypeError as e: - assert "intrusion-set--test-bad-json" in str(e) - assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e) - - -def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files): - # this tests handling of bad STIX json object - try: - fs_source.get("intrusion-set--test-non-stix") - except TypeError as e: - assert "intrusion-set--test-non-stix" in str(e) - assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e) - - -def test_filesytem_source_get_object(fs_source): - # get object - mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") - assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" - assert mal.name == "Rover" - - -def test_filesytem_source_get_nonexistent_object(fs_source): - ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38") - assert ind is None - - -def test_filesytem_source_all_versions(fs_source): - # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) - id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") - assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" - assert id_.name == "The MITRE Corporation" - assert id_.type == "identity" - - -def test_filesytem_source_query_single(fs_source): - # query2 - is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) - assert len(is_2) == 1 - - is_2 = is_2[0] - assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" - assert is_2.type == "attack-pattern" - - -def test_filesytem_source_query_multiple(fs_source): - # query - intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) - assert len(intrusion_sets) == 2 - assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] - assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] - - is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] - assert "DragonOK" in is_1.aliases - assert len(is_1.external_references) == 4 - - -def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): - # add python stix object - camp1 = Campaign(name="Hannibal", - objective="Targeting Italian and Spanish Diplomat internet accounts", - aliases=["War Elephant"]) - - fs_sink.add(camp1) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) - - camp1_r = fs_source.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == "Hannibal" - assert "War Elephant" in camp1_r.aliases - - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - - -def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): - # add stix object dict - camp2 = { - "name": "Aurelius", - "type": "campaign", - "objective": "German and French Intelligence Services", - "aliases": ["Purple Robes"], - "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add(camp2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) - - camp2_r = fs_source.get(camp2["id"]) - assert camp2_r.id == camp2["id"] - assert camp2_r.name == camp2["name"] - assert "Purple Robes" in camp2_r.aliases - - os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) - - -def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): - # add stix bundle dict - bund = { - "type": "bundle", - "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", - "spec_version": "2.0", - "objects": [ - { - "name": "Atilla", - "type": "campaign", - "objective": "Bulgarian, Albanian and Romanian Intelligence Services", - "aliases": ["Huns"], - "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - ] - } - - fs_sink.add(bund) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) - - camp3_r = fs_source.get(bund["objects"][0]["id"]) - assert camp3_r.id == bund["objects"][0]["id"] - assert camp3_r.name == bund["objects"][0]["name"] - assert "Huns" in camp3_r.aliases - - os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) - - -def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): - # add json-encoded stix obj - camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' - - fs_sink.add(camp4) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") - assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" - assert camp4_r.name == "Ghengis Khan" - - os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) - - -def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): - # add json-encoded stix bundle - bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ - ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' - fs_sink.add(bund2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") - assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" - assert camp5_r.name == "Spartacus" - - os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) - - -def test_filesystem_sink_add_objects_list(fs_sink, fs_source): - # add list of objects - camp6 = Campaign(name="Comanche", - objective="US Midwest manufacturing firms, oil refineries, and businesses", - aliases=["Horse Warrior"]) - - camp7 = { - "name": "Napolean", - "type": "campaign", - "objective": "Central and Eastern Europe military commands and departments", - "aliases": ["The Frenchmen"], - "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add([camp6, camp7]) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp6_r = fs_source.get(camp6.id) - assert camp6_r.id == camp6.id - assert "Horse Warrior" in camp6_r.aliases - - camp7_r = fs_source.get(camp7["id"]) - assert camp7_r.id == camp7["id"] - assert "The Frenchmen" in camp7_r.aliases - - # remove all added objects - os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) - - -def test_filesystem_store_get_stored_as_bundle(fs_store): - coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f") - assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f" - assert coa.type == "course-of-action" - - -def test_filesystem_store_get_stored_as_object(fs_store): - coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") - assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" - assert coa.type == "course-of-action" - - -def test_filesystem_store_all_versions(fs_store): - # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) - rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] - assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" - assert rel.type == "relationship" - - -def test_filesystem_store_query(fs_store): - # query() - tools = fs_store.query([Filter("labels", "in", "tool")]) - assert len(tools) == 2 - assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] - assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] - - -def test_filesystem_store_query_single_filter(fs_store): - query = Filter("labels", "in", "tool") - tools = fs_store.query(query) - assert len(tools) == 2 - assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] - assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] - - -def test_filesystem_store_empty_query(fs_store): - results = fs_store.query() # returns all - assert len(results) == 26 - assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] - assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] - - -def test_filesystem_store_query_multiple_filters(fs_store): - fs_store.source.filters.add(Filter("labels", "in", "tool")) - tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966")) - assert len(tools) == 1 - assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966" - - -def test_filesystem_store_query_dont_include_type_folder(fs_store): - results = fs_store.query(Filter("type", "!=", "tool")) - assert len(results) == 24 - - -def test_filesystem_store_add(fs_store): - # add() - camp1 = Campaign(name="Great Heathen Army", - objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", - aliases=["Ragnar"]) - fs_store.add(camp1) - - camp1_r = fs_store.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == camp1.name - - # remove - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - - -def test_filesystem_store_add_as_bundle(): - fs_store = FileSystemStore(FS_PATH, bundlify=True) - - camp1 = Campaign(name="Great Heathen Army", - objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", - aliases=["Ragnar"]) - fs_store.add(camp1) - - with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: - assert '"type": "bundle"' in bundle_file.read() - - camp1_r = fs_store.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == camp1.name - - shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) - - -def test_filesystem_add_bundle_object(fs_store): - bundle = Bundle() - fs_store.add(bundle) - - -def test_filesystem_store_add_invalid_object(fs_store): - ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a') # tuple isn't valid - with pytest.raises(TypeError) as excinfo: - fs_store.add(ind) - assert 'stix_data must be' in str(excinfo.value) - assert 'a STIX object' in str(excinfo.value) - assert 'JSON formatted STIX' in str(excinfo.value) - assert 'JSON formatted STIX bundle' in str(excinfo.value) - - -def test_filesystem_object_with_custom_property(fs_store): - camp = Campaign(name="Scipio Africanus", - objective="Defeat the Carthaginians", - x_empire="Roman", - allow_custom=True) - - fs_store.add(camp, True) - - camp_r = fs_store.get(camp.id) - assert camp_r.id == camp.id - assert camp_r.x_empire == camp.x_empire - - -def test_filesystem_object_with_custom_property_in_bundle(fs_store): - camp = Campaign(name="Scipio Africanus", - objective="Defeat the Carthaginians", - x_empire="Roman", - allow_custom=True) - - bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle) - - camp_r = fs_store.get(camp.id) - assert camp_r.id == camp.id - assert camp_r.x_empire == camp.x_empire - - -def test_filesystem_custom_object(fs_store): - @CustomObject('x-new-obj', [ - ('property1', properties.StringProperty(required=True)), - ]) - class NewObj(): - pass - - newobj = NewObj(property1='something') - fs_store.add(newobj) - - newobj_r = fs_store.get(newobj.id) - assert newobj_r.id == newobj.id - assert newobj_r.property1 == 'something' - - # remove dir - shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) - - -def test_relationships(rel_fs_store): - mal = rel_fs_store.get(MALWARE_ID) - resp = rel_fs_store.relationships(mal) - - assert len(resp) == 3 - assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) - assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp) - assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) - - -def test_relationships_by_type(rel_fs_store): - mal = rel_fs_store.get(MALWARE_ID) - resp = rel_fs_store.relationships(mal, relationship_type='indicates') - - assert len(resp) == 1 - assert resp[0]['id'] == RELATIONSHIP_IDS[0] - - -def test_relationships_by_source(rel_fs_store): - resp = rel_fs_store.relationships(MALWARE_ID, source_only=True) - - assert len(resp) == 1 - assert resp[0]['id'] == RELATIONSHIP_IDS[1] - - -def test_relationships_by_target(rel_fs_store): - resp = rel_fs_store.relationships(MALWARE_ID, target_only=True) - - assert len(resp) == 2 - assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) - assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) - - -def test_relationships_by_target_and_type(rel_fs_store): - resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True) - - assert len(resp) == 1 - assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) - - -def test_relationships_by_target_and_source(rel_fs_store): - with pytest.raises(ValueError) as excinfo: - rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True) - - assert 'not both' in str(excinfo.value) - - -def test_related_to(rel_fs_store): - mal = rel_fs_store.get(MALWARE_ID) - resp = rel_fs_store.related_to(mal) - - assert len(resp) == 3 - assert any(x['id'] == CAMPAIGN_ID for x in resp) - assert any(x['id'] == INDICATOR_ID for x in resp) - assert any(x['id'] == IDENTITY_ID for x in resp) - - -def test_related_to_by_source(rel_fs_store): - resp = rel_fs_store.related_to(MALWARE_ID, source_only=True) - - assert len(resp) == 1 - assert any(x['id'] == IDENTITY_ID for x in resp) - - -def test_related_to_by_target(rel_fs_store): - resp = rel_fs_store.related_to(MALWARE_ID, target_only=True) - - assert len(resp) == 2 - assert any(x['id'] == CAMPAIGN_ID for x in resp) - assert any(x['id'] == INDICATOR_ID for x in resp) From cd85683f2b8c110981e9c5c21a4743a00a05433e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:23:48 -0400 Subject: [PATCH 07/23] WIP: Cleaned test_datastore to only include base store tests --- stix2/test/test_datastore.py | 661 +++++------------------------------ 1 file changed, 95 insertions(+), 566 deletions(-) diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py index e80e8d8..0121a85 100644 --- a/stix2/test/test_datastore.py +++ b/stix2/test/test_datastore.py @@ -1,581 +1,110 @@ import pytest -from taxii2client import Collection -from stix2 import Filter, MemorySink, MemorySource from stix2.datastore import (CompositeDataSource, DataSink, DataSource, - make_id, taxii) -from stix2.datastore.filters import apply_common_filters -from stix2.utils import deduplicate - -COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' + DataStoreMixin) +from stix2.datastore.filters import Filter +from stix2.test.constants import CAMPAIGN_MORE_KWARGS -class MockTAXIIClient(object): - """Mock for taxii2_client.TAXIIClient""" - pass - - -@pytest.fixture -def collection(): - return Collection(COLLECTION_URL, MockTAXIIClient()) - - -IND1 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND2 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND3 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.936Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND4 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND5 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND6 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-31T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND7 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} -IND8 = { - "created": "2017-01-27T13:49:53.935Z", - "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", - "labels": [ - "url-watchlist" - ], - "modified": "2017-01-27T13:49:53.935Z", - "name": "Malicious site hosting downloader", - "pattern": "[url:value = 'http://x4z9arb.cn/4712']", - "type": "indicator", - "valid_from": "2017-01-27T13:49:53.935382Z" -} - -STIX_OBJS2 = [IND6, IND7, IND8] -STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] - - -def test_ds_abstract_class_smoke(): +def test_datasource_abstract_class_raises_error(): with pytest.raises(TypeError): DataSource() + +def test_datasink_abstract_class_raises_error(): with pytest.raises(TypeError): DataSink() -def test_ds_taxii(collection): - ds = taxii.TAXIICollectionSource(collection) - assert ds.collection is not None +def test_datastore_smoke(): + assert DataStoreMixin() is not None -def test_ds_taxii_name(collection): - ds = taxii.TAXIICollectionSource(collection) - assert ds.collection is not None - - -def test_parse_taxii_filters(): - query = [ - Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), - Filter("id", "=", "taxii stix object ID"), - Filter("type", "=", "taxii stix object ID"), - Filter("version", "=", "first"), - Filter("created_by_ref", "=", "Bane"), - ] - - taxii_filters_expected = set([ - Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), - Filter("id", "=", "taxii stix object ID"), - Filter("type", "=", "taxii stix object ID"), - Filter("version", "=", "first") - ]) - - ds = taxii.TAXIICollectionSource(collection) - - taxii_filters = ds._parse_taxii_filters(query) - - assert taxii_filters == taxii_filters_expected - - -def test_add_get_remove_filter(): - ds = taxii.TAXIICollectionSource(collection) - - # First 3 filters are valid, remaining properties are erroneous in some way - valid_filters = [ - Filter('type', '=', 'malware'), - Filter('id', '!=', 'stix object id'), - Filter('labels', 'in', ["heartbleed", "malicious-activity"]), - ] - - assert len(ds.filters) == 0 - - ds.filters.add(valid_filters[0]) - assert len(ds.filters) == 1 - - # Addin the same filter again will have no effect since `filters` uses a set - ds.filters.add(valid_filters[0]) - assert len(ds.filters) == 1 - - ds.filters.add(valid_filters[1]) - assert len(ds.filters) == 2 - ds.filters.add(valid_filters[2]) - assert len(ds.filters) == 3 - - assert set(valid_filters) == ds.filters - - # remove - ds.filters.remove(valid_filters[0]) - - assert len(ds.filters) == 2 - - ds.filters.update(valid_filters) - - -def test_filter_ops_check(): - # invalid filters - non supported operators - - with pytest.raises(ValueError) as excinfo: - # create Filter that has an operator that is not allowed - Filter('modified', '*', 'not supported operator') - assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'" - - with pytest.raises(ValueError) as excinfo: - Filter("type", "%", "4") - assert "Filter operator '%' not supported for specified property" in str(excinfo.value) - - -def test_filter_value_type_check(): - # invalid filters - non supported value types - - with pytest.raises(TypeError) as excinfo: - Filter('created', '=', object()) - # On Python 2, the type of object() is `` On Python 3, it's ``. - assert any([s in str(excinfo.value) for s in ["", "''"]]) - assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - Filter("type", "=", complex(2, -1)) - assert any([s in str(excinfo.value) for s in ["", "''"]]) - assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - Filter("type", "=", set([16, 23])) - assert any([s in str(excinfo.value) for s in ["", "''"]]) - assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) - - -def test_filter_type_underscore_check(): - # check that Filters where property="type", value (name) doesnt have underscores - with pytest.raises(ValueError) as excinfo: - Filter("type", "=", "oh_underscore") - assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value) - - -def test_apply_common_filters(): - stix_objs = [ - { - "created": "2017-01-27T13:49:53.997Z", - "description": "\n\nTITLE:\n\tPoison Ivy", - "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", - "labels": [ - "remote-access-trojan" - ], - "modified": "2017-01-27T13:49:53.997Z", - "name": "Poison Ivy", - "type": "malware" - }, - { - "created": "2014-05-08T09:00:00.000Z", - "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", - "labels": [ - "file-hash-watchlist" - ], - "modified": "2014-05-08T09:00:00.000Z", - "name": "File hash for Poison Ivy variant", - "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", - "type": "indicator", - "valid_from": "2014-05-08T09:00:00.000000Z" - }, - { - "created": "2014-05-08T09:00:00.000Z", - "granular_markings": [ - { - "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", - "selectors": [ - "relationship_type" - ] - } - ], - "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", - "modified": "2014-05-08T09:00:00.000Z", - "object_marking_refs": [ - "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" - ], - "relationship_type": "indicates", - "revoked": True, - "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", - "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", - "type": "relationship" - }, - { - "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", - "created": "2016-02-14T00:00:00.000Z", - "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", - "modified": "2016-02-14T00:00:00.000Z", - "type": "vulnerability", - "name": "CVE-2014-0160", - "description": "The (1) TLS...", - "external_references": [ - { - "source_name": "cve", - "external_id": "CVE-2014-0160" - } - ], - "labels": ["heartbleed", "has-logo"] - } - ] - - filters = [ - Filter("type", "!=", "relationship"), - Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), - Filter("labels", "in", "remote-access-trojan"), - Filter("created", ">", "2015-01-01T01:00:00.000Z"), - Filter("revoked", "=", True), - Filter("revoked", "!=", True), - Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), - Filter("granular_markings.selectors", "in", "relationship_type"), - Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), - Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), - Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), - Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), - Filter("granular_markings.selectors", "in", "description"), - Filter("external_references.source_name", "=", "CVE"), - ] - - # "Return any object whose type is not relationship" - resp = list(apply_common_filters(stix_objs, [filters[0]])) - ids = [r['id'] for r in resp] - assert stix_objs[0]['id'] in ids - assert stix_objs[1]['id'] in ids - assert stix_objs[3]['id'] in ids - assert len(ids) == 3 - - # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" - resp = list(apply_common_filters(stix_objs, [filters[1]])) - assert resp[0]['id'] == stix_objs[2]['id'] - assert len(resp) == 1 - - # "Return any object that contains remote-access-trojan in labels" - resp = list(apply_common_filters(stix_objs, [filters[2]])) - assert resp[0]['id'] == stix_objs[0]['id'] - assert len(resp) == 1 - - # "Return any object created after 2015-01-01T01:00:00.000Z" - resp = list(apply_common_filters(stix_objs, [filters[3]])) - assert resp[0]['id'] == stix_objs[0]['id'] - assert len(resp) == 2 - - # "Return any revoked object" - resp = list(apply_common_filters(stix_objs, [filters[4]])) - assert resp[0]['id'] == stix_objs[2]['id'] - assert len(resp) == 1 - - # "Return any object whose not revoked" - # Note that if 'revoked' property is not present in object. - # Currently we can't use such an expression to filter for... :( - resp = list(apply_common_filters(stix_objs, [filters[5]])) - assert len(resp) == 0 - - # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs" - resp = list(apply_common_filters(stix_objs, [filters[6]])) - assert resp[0]['id'] == stix_objs[2]['id'] - assert len(resp) == 1 - - # "Return any object that contains relationship_type in their selectors AND - # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref" - resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]])) - assert resp[0]['id'] == stix_objs[2]['id'] - assert len(resp) == 1 - - # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id" - resp = list(apply_common_filters(stix_objs, [filters[9]])) - assert resp[0]['id'] == stix_objs[3]['id'] - assert len(resp) == 1 - - # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9" - resp = list(apply_common_filters(stix_objs, [filters[10]])) - assert len(resp) == 1 - - # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None) - resp = list(apply_common_filters(stix_objs, [filters[11]])) - assert len(resp) == 0 - - # "Return any object that contains description in its selectors" (None) - resp = list(apply_common_filters(stix_objs, [filters[12]])) - assert len(resp) == 0 - - # "Return any object that object that matches CVE in source_name" (None, case sensitive) - resp = list(apply_common_filters(stix_objs, [filters[13]])) - assert len(resp) == 0 - - -def test_filters0(): - # "Return any object modified before 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[1]['id'] - assert len(resp) == 2 - - -def test_filters1(): - # "Return any object modified after 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] - assert len(resp) == 1 - - -def test_filters2(): - # "Return any object modified after or on 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] - assert len(resp) == 3 - - -def test_filters3(): - # "Return any object modified before or on 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[1]['id'] - assert len(resp) == 2 - - -def test_filters4(): - # Assert invalid Filter cannot be created - with pytest.raises(ValueError) as excinfo: - Filter("modified", "?", "2017-01-27T13:49:53.935Z") - assert str(excinfo.value) == ("Filter operator '?' not supported " - "for specified property: 'modified'") - - -def test_filters5(): - # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] - assert len(resp) == 1 - - -def test_filters6(): - # Test filtering on non-common property - resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] - assert len(resp) == 3 - - -def test_filters7(): - # Test filtering on embedded property - stix_objects = list(STIX_OBJS2) + [{ - "type": "observed-data", - "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2016-04-06T19:58:16.000Z", - "modified": "2016-04-06T19:58:16.000Z", - "first_observed": "2015-12-21T19:00:00Z", - "last_observed": "2015-12-21T19:00:00Z", - "number_observed": 50, - "objects": { - "0": { - "type": "file", - "hashes": { - "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" - }, - "extensions": { - "pdf-ext": { - "version": "1.7", - "document_info_dict": { - "Title": "Sample document", - "Author": "Adobe Systems Incorporated", - "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh", - "Producer": "Acrobat Distiller 3.01 for Power Macintosh", - "CreationDate": "20070412090123-02" - }, - "pdfid0": "DFCE52BD827ECF765649852119D", - "pdfid1": "57A1E0F9ED2AE523E313C" - } - } - } - } - }] - resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) - assert resp[0]['id'] == stix_objects[3]['id'] - assert len(resp) == 1 - - -def test_deduplicate(): - unique = deduplicate(STIX_OBJS1) - - # Only 3 objects are unique - # 2 id's vary - # 2 modified times vary for a particular id - - assert len(unique) == 3 - - ids = [obj['id'] for obj in unique] - mods = [obj['modified'] for obj in unique] - - assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids - assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids - assert "2017-01-27T13:49:53.935Z" in mods - assert "2017-01-27T13:49:53.936Z" in mods - - -def test_add_remove_composite_datasource(): - cds = CompositeDataSource() - ds1 = MemorySource() - ds2 = MemorySource() - ds3 = MemorySink() - - with pytest.raises(TypeError) as excinfo: - cds.add_data_sources([ds1, ds2, ds1, ds3]) - assert str(excinfo.value) == ("DataSource (to be added) is not of type " - "stix2.DataSource. DataSource type is ''") - - cds.add_data_sources([ds1, ds2, ds1]) - - assert len(cds.get_all_data_sources()) == 2 - - cds.remove_data_sources([ds1.id, ds2.id]) - - assert len(cds.get_all_data_sources()) == 0 - - -def test_composite_datasource_operations(): - BUNDLE1 = dict(id="bundle--%s" % make_id(), - objects=STIX_OBJS1, - spec_version="2.0", - type="bundle") - cds1 = CompositeDataSource() - ds1_1 = MemorySource(stix_data=BUNDLE1) - ds1_2 = MemorySource(stix_data=STIX_OBJS2) - - cds2 = CompositeDataSource() - ds2_1 = MemorySource(stix_data=BUNDLE1) - ds2_2 = MemorySource(stix_data=STIX_OBJS2) - - cds1.add_data_sources([ds1_1, ds1_2]) - cds2.add_data_sources([ds2_1, ds2_2]) - - indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - - # In STIX_OBJS2 changed the 'modified' property to a later time... - assert len(indicators) == 2 - - cds1.add_data_sources([cds2]) - - indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - - assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" - assert indicator["modified"] == "2017-01-31T13:49:53.935Z" - assert indicator["type"] == "indicator" - - query1 = [ - Filter("type", "=", "indicator") - ] - - query2 = [ - Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") - ] - - cds1.filters.update(query2) - - results = cds1.query(query1) - - # STIX_OBJS2 has indicator with later time, one with different id, one with - # original time in STIX_OBJS1 - assert len(results) == 3 - - indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - - assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" - assert indicator["modified"] == "2017-01-31T13:49:53.935Z" - assert indicator["type"] == "indicator" - - # There is only one indicator with different ID. Since we use the same data - # when deduplicated, only two indicators (one with different modified). - results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - assert len(results) == 2 - - # Since we have filters already associated with our CompositeSource providing - # nothing returns the same as cds1.query(query1) (the associated query is query2) - results = cds1.query([]) - assert len(results) == 3 - - -def test_composite_datastore_no_datasource(): - cds = CompositeDataSource() - +def test_datastore_get_raises(): with pytest.raises(AttributeError) as excinfo: - cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - assert 'CompositeDataSource has no data source' in str(excinfo.value) + DataStoreMixin().get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_all_versions_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_query_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().query([Filter("type", "=", "indicator")]) + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_creator_of_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().creator_of(CAMPAIGN_MORE_KWARGS) + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_relationships_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().relationships(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + target_only=True) + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_related_to_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().related_to(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + target_only=True) + assert "DataStoreMixin has no data source to query" == str(excinfo.value) + + +def test_datastore_add_raises(): + with pytest.raises(AttributeError) as excinfo: + DataStoreMixin().add(CAMPAIGN_MORE_KWARGS) + assert "DataStoreMixin has no data sink to put objects in" == str(excinfo.value) + + +def test_composite_datastore_get_raises_error(): + with pytest.raises(AttributeError) as excinfo: + CompositeDataSource().get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert "CompositeDataSource has no data sources" == str(excinfo.value) + + +def test_composite_datastore_all_versions_raises_error(): + with pytest.raises(AttributeError) as excinfo: + CompositeDataSource().all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert "CompositeDataSource has no data sources" == str(excinfo.value) + + +def test_composite_datastore_query_raises_error(): + with pytest.raises(AttributeError) as excinfo: + CompositeDataSource().query([Filter("type", "=", "indicator")]) + assert "CompositeDataSource has no data sources" == str(excinfo.value) + + +def test_composite_datastore_relationships_raises_error(): + with pytest.raises(AttributeError) as excinfo: + CompositeDataSource().relationships(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + target_only=True) + assert "CompositeDataSource has no data sources" == str(excinfo.value) + + +def test_composite_datastore_related_to_raises_error(): + with pytest.raises(AttributeError) as excinfo: + CompositeDataSource().related_to(obj="indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + target_only=True) + assert "CompositeDataSource has no data sources" == str(excinfo.value) + + +def test_composite_datastore_add_data_source_raises_error(): + with pytest.raises(TypeError) as excinfo: + ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + CompositeDataSource().add_data_source(ind) + assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value) + + +def test_composite_datastore_add_data_sources_raises_error(): + with pytest.raises(TypeError) as excinfo: + ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + CompositeDataSource().add_data_sources(ind) + assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value) From 1ff640de3ce5bce56642c4fa35f22c8958208e20 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:24:15 -0400 Subject: [PATCH 08/23] Moved some test methods here --- stix2/test/test_utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py index cbe5b0f..4705ec8 100644 --- a/stix2/test/test_utils.py +++ b/stix2/test/test_utils.py @@ -82,3 +82,21 @@ def test_get_dict_invalid(data): ]) def test_get_type_from_id(stix_id, typ): assert stix2.utils.get_type_from_id(stix_id) == typ + + +def test_deduplicate(stix1_objs): + unique = stix2.utils.deduplicate(stix1_objs) + + # Only 3 objects are unique + # 2 id's vary + # 2 modified times vary for a particular id + + assert len(unique) == 3 + + ids = [obj['id'] for obj in unique] + mods = [obj['modified'] for obj in unique] + + assert "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" in ids + assert "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" in ids + assert "2017-01-27T13:49:53.935Z" in mods + assert "2017-01-27T13:49:53.936Z" in mods From 4d35cc6f2878f8320e665d7414d6e5205860a528 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 6 Apr 2018 14:25:06 -0400 Subject: [PATCH 09/23] WIP: Update tox.ini --- tox.ini | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tox.ini b/tox.ini index bfc8c1b..884288a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] -envlist = py27,py34,py35,py36,pycodestyle,isort-check +envlist = py27,py34,py35,py36,pycodestyle,isort-check,taxii-datastore [testenv] deps = - -U - tox - pytest - pytest-cov - coverage + -U + tox + pytest + pytest-cov + coverage commands = - py.test --cov=stix2 stix2/test/ --cov-report term-missing + py.test --cov=stix2 stix2/test/ --cov-report term-missing passenv = CI TRAVIS TRAVIS_* @@ -22,20 +22,28 @@ commands = flake8 [pycodestyle] -max-line-length=160 +max-line-length = 160 [flake8] -max-line-length=160 +max-line-length = 160 [testenv:isort-check] deps = isort commands = - isort -rc stix2 examples -df - isort -rc stix2 examples -c + isort -rc stix2 examples -df + isort -rc stix2 examples -c + +[testenv:taxii-datastore] +deps = + git+https://github.com/oasis-open/cti-taxii-client.git#egg=taxii2-client + git+https://github.com/oasis-open/cti-taxii-server.git#egg=medallion + +commands = + py.test --cov=stix2 stix2/test/test_datastore_taxii.py --cov-report term-missing [travis] python = - 2.7: py27, pycodestyle + 2.7: py27, pycodestyle, taxii-datastore 3.4: py34, pycodestyle 3.5: py35, pycodestyle - 3.6: py36, pycodestyle + 3.6: py36, pycodestyle, taxii-datastore From 84c09d7a8fe8fcba7ecc2eff5c69e355c0fedb43 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 09:08:29 -0400 Subject: [PATCH 10/23] Rename stix object fixture --- stix2/test/conftest.py | 4 +-- stix2/test/test_datastore_filters.py | 40 ++++++++++++++-------------- stix2/test/test_datastore_memory.py | 9 ++++--- stix2/test/test_utils.py | 4 +-- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py index acbb8b0..5cf63b1 100644 --- a/stix2/test/conftest.py +++ b/stix2/test/conftest.py @@ -49,7 +49,7 @@ def relationship(uuid4, clock): @pytest.fixture -def stix1_objs(): +def stix_objs1(): ind1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -114,7 +114,7 @@ def stix1_objs(): @pytest.fixture -def stix2_objs(): +def stix_objs2(): ind6 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py index 4f66ff5..32e91ed 100644 --- a/stix2/test/test_datastore_filters.py +++ b/stix2/test/test_datastore_filters.py @@ -191,31 +191,31 @@ def test_apply_common_filters(): assert len(resp) == 0 -def test_filters0(): +def test_filters0(stix_objs2): # "Return any object modified before 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[1]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) + assert resp[0]['id'] == stix_objs2[1]['id'] assert len(resp) == 2 -def test_filters1(): +def test_filters1(stix_objs2): # "Return any object modified after 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) + assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 1 -def test_filters2(): +def test_filters2(stix_objs2): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) + assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 -def test_filters3(): +def test_filters3(stix_objs2): # "Return any object modified before or on 2017-01-28T13:49:53.935Z" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) - assert resp[0]['id'] == STIX_OBJS2[1]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) + assert resp[0]['id'] == stix_objs2[1]['id'] assert len(resp) == 2 @@ -227,23 +227,23 @@ def test_filters4(): "for specified property: 'modified'") -def test_filters5(): +def test_filters5(stix_objs2): # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" - resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) + assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 1 -def test_filters6(): +def test_filters6(stix_objs2): # Test filtering on non-common property - resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) - assert resp[0]['id'] == STIX_OBJS2[0]['id'] + resp = list(apply_common_filters(stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")])) + assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 -def test_filters7(): +def test_filters7(stix_objs2): # Test filtering on embedded property - stix_objects = list(STIX_OBJS2) + [{ + stix_objects = list(stix_objs2) + [{ "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py index b027640..d341fd2 100644 --- a/stix2/test/test_datastore_memory.py +++ b/stix2/test/test_datastore_memory.py @@ -1,6 +1,7 @@ import pytest from stix2.datastore import CompositeDataSource, make_id +from stix2.datastore.filters import Filter from stix2.datastore.memory import MemorySource, MemorySink @@ -24,18 +25,18 @@ def test_add_remove_composite_datasource(): assert len(cds.get_all_data_sources()) == 0 -def test_composite_datasource_operations(): +def test_composite_datasource_operations(stix_objs1, stix_objs2): BUNDLE1 = dict(id="bundle--%s" % make_id(), - objects=STIX_OBJS1, + objects=stix_objs1, spec_version="2.0", type="bundle") cds1 = CompositeDataSource() ds1_1 = MemorySource(stix_data=BUNDLE1) - ds1_2 = MemorySource(stix_data=STIX_OBJS2) + ds1_2 = MemorySource(stix_data=stix_objs2) cds2 = CompositeDataSource() ds2_1 = MemorySource(stix_data=BUNDLE1) - ds2_2 = MemorySource(stix_data=STIX_OBJS2) + ds2_2 = MemorySource(stix_data=stix_objs2) cds1.add_data_sources([ds1_1, ds1_2]) cds2.add_data_sources([ds2_1, ds2_2]) diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py index 4705ec8..9d25a5e 100644 --- a/stix2/test/test_utils.py +++ b/stix2/test/test_utils.py @@ -84,8 +84,8 @@ def test_get_type_from_id(stix_id, typ): assert stix2.utils.get_type_from_id(stix_id) == typ -def test_deduplicate(stix1_objs): - unique = stix2.utils.deduplicate(stix1_objs) +def test_deduplicate(stix_objs1): + unique = stix2.utils.deduplicate(stix_objs1) # Only 3 objects are unique # 2 id's vary From 988dad79b9b337f00ba3b1d66740fa7d340e65b0 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 09:11:59 -0400 Subject: [PATCH 11/23] WIP: Mock TAXIIClient Collection Endpoint --- stix2/test/test_datastore_taxii.py | 97 +++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index 0a293b7..c20fbb6 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -1,24 +1,101 @@ +from stix2 import Bundle, ThreatActor, TAXIICollectionSource, TAXIICollectionSink +from stix2.datastore.filters import Filter + +import json +import pytest + +from taxii2client import Collection, _filter_kwargs_to_query_params +from medallion.filters.basic_filter import BasicFilter + COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' -class MockTAXIIClient(object): +class MockTAXIICollectionEndpoint(Collection): """Mock for taxii2_client.TAXIIClient""" - pass + + def __init__(self, url, **kwargs): + super(MockTAXIICollectionEndpoint, self).__init__(url, **kwargs) + self.objects = [] + + def add_objects(self, bundle): + self._verify_can_write() + if isinstance(bundle, str): + bundle = json.loads(bundle) + for object in bundle.get("objects", []): + self.objects.append(object) + + def get_objects(self, **filter_kwargs): + self._verify_can_read() + query_params = _filter_kwargs_to_query_params(filter_kwargs) + query_params = json.loads(query_params) + full_filter = BasicFilter(query_params or {}) + objs = full_filter.process_filter( + self.objects, + ("id", "type", "version"), + [] + ) + return Bundle(objects=objs) + + def get_object(self, id, version=None): + self._verify_can_read() + query_params = None + if version: + query_params = _filter_kwargs_to_query_params({"version": version}) + if query_params: + query_params = json.loads(query_params) + full_filter = BasicFilter(query_params or {}) + objs = full_filter.process_filter( + self.objects, + ("version",), + [] + ) + return Bundle(objects=objs) @pytest.fixture -def collection(): - return Collection(COLLECTION_URL, MockTAXIIClient()) +def collection(stix_objs1): + mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{ + "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", + "title": "Writable Collection", + "description": "This collection is a dropbox for submitting indicators", + "can_read": True, + "can_write": True, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + }) + + mock.objects.extend(stix_objs1) + return mock def test_ds_taxii(collection): - ds = taxii.TAXIICollectionSource(collection) + ds = TAXIICollectionSource(collection) assert ds.collection is not None -def test_ds_taxii_name(collection): - ds = taxii.TAXIICollectionSource(collection) - assert ds.collection is not None +def test_add_stix2_object(collection): + tc_sink = TAXIICollectionSink(collection) + + # create new STIX threat-actor + ta = ThreatActor(name="Teddy Bear", + labels=["nation-state"], + sophistication="innovator", + resource_level="government", + goals=[ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector", + ]) + + tc_sink.add(ta) + + +def test_get_stix2_object(collection): + tc_sink = TAXIICollectionSource(collection) + + objects = tc_sink.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert objects def test_parse_taxii_filters(): @@ -37,7 +114,7 @@ def test_parse_taxii_filters(): Filter("version", "=", "first") ]) - ds = taxii.TAXIICollectionSource(collection) + ds = TAXIICollectionSource(collection) taxii_filters = ds._parse_taxii_filters(query) @@ -45,7 +122,7 @@ def test_parse_taxii_filters(): def test_add_get_remove_filter(): - ds = taxii.TAXIICollectionSource(collection) + ds = TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ From 507e3995dd80888c3bcbc636cb3bd9fe62e6dbd3 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 09:42:33 -0400 Subject: [PATCH 12/23] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 395676e..9e12d7d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +.pytest_cache/ # Translations *.mo From 8a9acfe487feecaedb92bb8e8a86358a0ad00d53 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:11:08 -0400 Subject: [PATCH 13/23] Create TAXII query correctly. closes #169 --- stix2/datastore/taxii.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index faa2669..872a510 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -176,8 +176,8 @@ class TAXIICollectionSource(DataSource): """ # make query in TAXII query format since 'id' is TAXII field query = [ - Filter("match[id]", "=", stix_id), - Filter("match[version]", "=", "all") + Filter("id", "=", stix_id), + Filter("version", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) @@ -232,7 +232,8 @@ class TAXIICollectionSource(DataSource): all_data = deduplicate(all_data) # apply local (CompositeDataSource, TAXIICollectionSource and query) filters - all_data = list(apply_common_filters(all_data, (query - taxii_filters))) + query.remove(taxii_filters) + all_data = list(apply_common_filters(all_data, query)) except HTTPError: # if resources not found or access is denied from TAXII server, return empty list @@ -249,7 +250,7 @@ class TAXIICollectionSource(DataSource): Does not put in TAXII spec format as the TAXII2Client (that we use) does this for us. - NOTE: + Notes: Currently, the TAXII2Client can handle TAXII filters where the filter value is list, as both a comma-seperated string or python list @@ -265,7 +266,8 @@ class TAXIICollectionSource(DataSource): query (list): list of filters to extract which ones are TAXII specific. - Returns: a list of the TAXII filters + Returns: + A list of TAXII filters that meet the TAXII filtering parameters. """ taxii_filters = [] From fb79e703b8a2b339c45a3e99d3283d314d0258fb Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:12:16 -0400 Subject: [PATCH 14/23] Simplify FilterSet instance in MemorySource --- stix2/datastore/memory.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index c43da8b..c1d202d 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -268,10 +268,7 @@ class MemorySource(DataSource): is returned in the same form as it as added. """ - if query is None: - query = FilterSet() - else: - query = FilterSet(query) + query = FilterSet(query) # combine all query filters if self.filters: From 488012d39d44fb4f27d3e541a08f06f1bbccea58 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:12:54 -0400 Subject: [PATCH 15/23] Add support for 'real_stix_objs' for test cases --- stix2/test/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py index 5cf63b1..ddbcb7e 100644 --- a/stix2/test/conftest.py +++ b/stix2/test/conftest.py @@ -152,3 +152,13 @@ def stix_objs2(): "valid_from": "2017-01-27T13:49:53.935382Z" } return [ind6, ind7, ind8] + + +@pytest.fixture +def real_stix_objs1(stix_objs1): + return [stix2.parse(x) for x in stix_objs1] + + +@pytest.fixture +def real_stix_objs2(stix_objs2): + return [stix2.parse(x) for x in stix_objs2] From e4a226cae6ae3a5e6c1a507942f7bd8e4c52fc16 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:14:16 -0400 Subject: [PATCH 16/23] Finish merging all the changes from @mbastian1135 --- stix2/test/test_datastore.py | 7 + stix2/test/test_datastore_filters.py | 376 ++++++++++++++++++++------- stix2/test/test_datastore_memory.py | 2 +- 3 files changed, 288 insertions(+), 97 deletions(-) diff --git a/stix2/test/test_datastore.py b/stix2/test/test_datastore.py index 0121a85..323365a 100644 --- a/stix2/test/test_datastore.py +++ b/stix2/test/test_datastore.py @@ -108,3 +108,10 @@ def test_composite_datastore_add_data_sources_raises_error(): ind = "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" CompositeDataSource().add_data_sources(ind) assert "DataSource (to be added) is not of type stix2.DataSource. DataSource type is '{}'".format(type(ind)) == str(excinfo.value) + + +def test_composite_datastore_no_datasource(): + cds = CompositeDataSource() + with pytest.raises(AttributeError) as excinfo: + cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert 'CompositeDataSource has no data source' in str(excinfo.value) diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py index 32e91ed..8803e4b 100644 --- a/stix2/test/test_datastore_filters.py +++ b/stix2/test/test_datastore_filters.py @@ -1,6 +1,112 @@ import pytest +from stix2 import parse from stix2.datastore.filters import Filter, apply_common_filters +from stix2.utils import STIXdatetime, parse_into_datetime + + +stix_objs = [ + { + "created": "2017-01-27T13:49:53.997Z", + "description": "\n\nTITLE:\n\tPoison Ivy", + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "labels": [ + "remote-access-trojan" + ], + "modified": "2017-01-27T13:49:53.997Z", + "name": "Poison Ivy", + "type": "malware" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "labels": [ + "file-hash-watchlist" + ], + "modified": "2014-05-08T09:00:00.000Z", + "name": "File hash for Poison Ivy variant", + "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "type": "indicator", + "valid_from": "2014-05-08T09:00:00.000000Z" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "granular_markings": [ + { + "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + "selectors": [ + "relationship_type" + ] + } + ], + "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", + "modified": "2014-05-08T09:00:00.000Z", + "object_marking_refs": [ + "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" + ], + "relationship_type": "indicates", + "revoked": True, + "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "type": "relationship" + }, + { + "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", + "created": "2016-02-14T00:00:00.000Z", + "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", + "modified": "2016-02-14T00:00:00.000Z", + "type": "vulnerability", + "name": "CVE-2014-0160", + "description": "The (1) TLS...", + "external_references": [ + { + "source_name": "cve", + "external_id": "CVE-2014-0160" + } + ], + "labels": ["heartbleed", "has-logo"] + }, + { + "type": "observed-data", + "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "file", + "name": "HAL 9000.exe" + } + } + + } +] + + +filters = [ + Filter("type", "!=", "relationship"), + Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), + Filter("labels", "in", "remote-access-trojan"), + Filter("created", ">", "2015-01-01T01:00:00.000Z"), + Filter("revoked", "=", True), + Filter("revoked", "!=", True), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "relationship_type"), + Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), + Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), + Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), + Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), + Filter("granular_markings.selectors", "in", "description"), + Filter("external_references.source_name", "=", "CVE"), + Filter("objects", "=", {"0": {"type": "file", "name": "HAL 9000.exe"}}) +] + +# same as above objects but converted to real Python STIX2 objects +# to test filters against true Python STIX2 objects +real_stix_objs = [parse(stix_obj) for stix_obj in stix_objs] def test_filter_ops_check(): @@ -43,181 +149,243 @@ def test_filter_type_underscore_check(): assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str(excinfo.value) -def test_apply_common_filters(): - stix_objs = [ - { - "created": "2017-01-27T13:49:53.997Z", - "description": "\n\nTITLE:\n\tPoison Ivy", - "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", - "labels": [ - "remote-access-trojan" - ], - "modified": "2017-01-27T13:49:53.997Z", - "name": "Poison Ivy", - "type": "malware" - }, - { - "created": "2014-05-08T09:00:00.000Z", - "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", - "labels": [ - "file-hash-watchlist" - ], - "modified": "2014-05-08T09:00:00.000Z", - "name": "File hash for Poison Ivy variant", - "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", - "type": "indicator", - "valid_from": "2014-05-08T09:00:00.000000Z" - }, - { - "created": "2014-05-08T09:00:00.000Z", - "granular_markings": [ - { - "marking_ref": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", - "selectors": [ - "relationship_type" - ] - } - ], - "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", - "modified": "2014-05-08T09:00:00.000Z", - "object_marking_refs": [ - "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" - ], - "relationship_type": "indicates", - "revoked": True, - "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", - "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", - "type": "relationship" - }, - { - "id": "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef", - "created": "2016-02-14T00:00:00.000Z", - "created_by_ref": "identity--00000000-0000-0000-0000-b8e91df99dc9", - "modified": "2016-02-14T00:00:00.000Z", - "type": "vulnerability", - "name": "CVE-2014-0160", - "description": "The (1) TLS...", - "external_references": [ - { - "source_name": "cve", - "external_id": "CVE-2014-0160" - } - ], - "labels": ["heartbleed", "has-logo"] - } - ] - - filters = [ - Filter("type", "!=", "relationship"), - Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), - Filter("labels", "in", "remote-access-trojan"), - Filter("created", ">", "2015-01-01T01:00:00.000Z"), - Filter("revoked", "=", True), - Filter("revoked", "!=", True), - Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), - Filter("granular_markings.selectors", "in", "relationship_type"), - Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), - Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), - Filter("created_by_ref", "=", "identity--00000000-0000-0000-0000-b8e91df99dc9"), - Filter("object_marking_refs", "=", "marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9"), - Filter("granular_markings.selectors", "in", "description"), - Filter("external_references.source_name", "=", "CVE"), - ] - +def test_apply_common_filters0(): # "Return any object whose type is not relationship" resp = list(apply_common_filters(stix_objs, [filters[0]])) ids = [r['id'] for r in resp] assert stix_objs[0]['id'] in ids assert stix_objs[1]['id'] in ids assert stix_objs[3]['id'] in ids - assert len(ids) == 3 + assert len(ids) == 4 + resp = list(apply_common_filters(real_stix_objs, [filters[0]])) + ids = [r.id for r in resp] + assert real_stix_objs[0].id in ids + assert real_stix_objs[1].id in ids + assert real_stix_objs[3].id in ids + assert len(ids) == 4 + + +def test_apply_common_filters1(): # "Return any object that matched id relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463" resp = list(apply_common_filters(stix_objs, [filters[1]])) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[1]])) + assert resp[0].id == real_stix_objs[2].id + assert len(resp) == 1 + + +def test_apply_common_filters2(): # "Return any object that contains remote-access-trojan in labels" resp = list(apply_common_filters(stix_objs, [filters[2]])) assert resp[0]['id'] == stix_objs[0]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[2]])) + assert resp[0].id == real_stix_objs[0].id + assert len(resp) == 1 + + +def test_apply_common_filters3(): # "Return any object created after 2015-01-01T01:00:00.000Z" resp = list(apply_common_filters(stix_objs, [filters[3]])) assert resp[0]['id'] == stix_objs[0]['id'] - assert len(resp) == 2 + assert len(resp) == 3 + resp = list(apply_common_filters(real_stix_objs, [filters[3]])) + assert resp[0].id == real_stix_objs[0].id + assert len(resp) == 3 + + +def test_apply_common_filters4(): # "Return any revoked object" resp = list(apply_common_filters(stix_objs, [filters[4]])) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[4]])) + assert resp[0].id == real_stix_objs[2].id + assert len(resp) == 1 + + +def test_apply_common_filters5(): # "Return any object whose not revoked" - # Note that if 'revoked' property is not present in object. - # Currently we can't use such an expression to filter for... :( resp = list(apply_common_filters(stix_objs, [filters[5]])) assert len(resp) == 0 + resp = list(apply_common_filters(real_stix_objs, [filters[5]])) + assert len(resp) == 4 + + +def test_apply_common_filters6(): # "Return any object that matches marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9 in object_marking_refs" resp = list(apply_common_filters(stix_objs, [filters[6]])) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[6]])) + assert resp[0].id == real_stix_objs[2].id + assert len(resp) == 1 + + +def test_apply_common_filters7(): # "Return any object that contains relationship_type in their selectors AND # also has marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed in marking_ref" resp = list(apply_common_filters(stix_objs, [filters[7], filters[8]])) assert resp[0]['id'] == stix_objs[2]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[7], filters[8]])) + assert resp[0].id == real_stix_objs[2].id + assert len(resp) == 1 + + +def test_apply_common_filters8(): # "Return any object that contains CVE-2014-0160,CVE-2017-6608 in their external_id" resp = list(apply_common_filters(stix_objs, [filters[9]])) assert resp[0]['id'] == stix_objs[3]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[9]])) + assert resp[0].id == real_stix_objs[3].id + assert len(resp) == 1 + + +def test_apply_common_filters9(): # "Return any object that matches created_by_ref identity--00000000-0000-0000-0000-b8e91df99dc9" resp = list(apply_common_filters(stix_objs, [filters[10]])) assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs, [filters[10]])) + assert len(resp) == 1 + + +def test_apply_common_filters10(): # "Return any object that matches marking-definition--613f2e26-0000-0000-0000-b8e91df99dc9 in object_marking_refs" (None) resp = list(apply_common_filters(stix_objs, [filters[11]])) assert len(resp) == 0 + resp = list(apply_common_filters(real_stix_objs, [filters[11]])) + assert len(resp) == 0 + + +def test_apply_common_filters11(): # "Return any object that contains description in its selectors" (None) resp = list(apply_common_filters(stix_objs, [filters[12]])) assert len(resp) == 0 - # "Return any object that object that matches CVE in source_name" (None, case sensitive) - resp = list(apply_common_filters(stix_objs, [filters[13]])) + resp = list(apply_common_filters(real_stix_objs, [filters[12]])) assert len(resp) == 0 -def test_filters0(stix_objs2): +def test_apply_common_filters12(): + # "Return any object that matches CVE in source_name" (None, case sensitive) + resp = list(apply_common_filters(stix_objs, [filters[13]])) + assert len(resp) == 0 + + resp = list(apply_common_filters(real_stix_objs, [filters[13]])) + assert len(resp) == 0 + + +def test_apply_common_filters13(): + # Return any object that matches file object in "objects" + resp = list(apply_common_filters(stix_objs, [filters[14]])) + assert resp[0]["id"] == stix_objs[4]["id"] + assert len(resp) == 1 + # important additional check to make sure original File dict was + # not converted to File object. (this was a deep bug found) + assert isinstance(resp[0]["objects"]["0"], dict) + + resp = list(apply_common_filters(real_stix_objs, [filters[14]])) + assert resp[0].id == real_stix_objs[4].id + assert len(resp) == 1 + + +def test_datetime_filter_behavior(): + """if a filter is initialized with its value being a datetime object + OR the STIX object property being filtered on is a datetime object, all + resulting comparisons executed are done on the string representations + of the datetime objects, as the Filter functionality will convert + all datetime objects to there string forms using format_datetim() + + This test makes sure all datetime comparisons are carried out correctly + """ + filter_with_dt_obj = Filter("created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond")) + filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z") + + # check that filter value is converted from datetime to str + assert isinstance(filter_with_dt_obj.value, str) + + # compare datetime string to filter w/ datetime obj + resp = list(apply_common_filters(stix_objs, [filter_with_dt_obj])) + assert len(resp) == 1 + assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" + + # compare datetime obj to filter w/ datetime obj + resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj])) + assert len(resp) == 1 + assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" + assert isinstance(resp[0].created, STIXdatetime) # make sure original object not altered + + # compare datetime string to filter w/ str + resp = list(apply_common_filters(stix_objs, [filter_with_str])) + assert len(resp) == 1 + assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" + + # compare datetime obj to filter w/ str + resp = list(apply_common_filters(real_stix_objs, [filter_with_str])) + assert len(resp) == 1 + assert resp[0]["id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" + assert isinstance(resp[0].created, STIXdatetime) # make sure original object not altered + + +def test_filters0(stix_objs2, real_stix_objs2): # "Return any object modified before 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == stix_objs2[1]['id'] assert len(resp) == 2 + resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", "<", parse_into_datetime("2017-01-28T13:49:53.935Z"))])) + assert resp[0].id == real_stix_objs2[1].id + assert len(resp) == 2 -def test_filters1(stix_objs2): + +def test_filters1(stix_objs2, real_stix_objs2): # "Return any object modified after 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">", parse_into_datetime("2017-01-28T13:49:53.935Z"))])) + assert resp[0].id == real_stix_objs2[0].id + assert len(resp) == 1 -def test_filters2(stix_objs2): + +def test_filters2(stix_objs2, real_stix_objs2): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 + resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))])) + assert resp[0].id == real_stix_objs2[0].id + assert len(resp) == 3 -def test_filters3(stix_objs2): + +def test_filters3(stix_objs2, real_stix_objs2): # "Return any object modified before or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(stix_objs2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == stix_objs2[1]['id'] assert len(resp) == 2 + # "Return any object modified before or on 2017-01-28T13:49:53.935Z" + fv = Filter("modified", "<=", parse_into_datetime("2017-01-27T13:49:53.935Z")) + resp = list(apply_common_filters(real_stix_objs2, [fv])) + assert resp[0].id == real_stix_objs2[1].id + assert len(resp) == 2 + def test_filters4(): # Assert invalid Filter cannot be created @@ -227,23 +395,31 @@ def test_filters4(): "for specified property: 'modified'") -def test_filters5(stix_objs2): +def test_filters5(stix_objs2, real_stix_objs2): # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" resp = list(apply_common_filters(stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 1 + resp = list(apply_common_filters(real_stix_objs2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) + assert resp[0].id == real_stix_objs2[0].id + assert len(resp) == 1 -def test_filters6(stix_objs2): + +def test_filters6(stix_objs2, real_stix_objs2): # Test filtering on non-common property resp = list(apply_common_filters(stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 + resp = list(apply_common_filters(real_stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")])) + assert resp[0].id == real_stix_objs2[0].id + assert len(resp) == 3 -def test_filters7(stix_objs2): + +def test_filters7(stix_objs2, real_stix_objs2): # Test filtering on embedded property - stix_objects = list(stix_objs2) + [{ + obsvd_data_obj = { "type": "observed-data", "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", @@ -274,7 +450,15 @@ def test_filters7(stix_objs2): } } } - }] + } + + stix_objects = list(stix_objs2) + [obsvd_data_obj] + real_stix_objects = list(real_stix_objs2) + [parse(obsvd_data_obj)] + resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) assert resp[0]['id'] == stix_objects[3]['id'] assert len(resp) == 1 + + resp = list(apply_common_filters(real_stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) + assert resp[0].id == real_stix_objects[3].id + assert len(resp) == 1 diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py index d341fd2..dcec813 100644 --- a/stix2/test/test_datastore_memory.py +++ b/stix2/test/test_datastore_memory.py @@ -62,7 +62,7 @@ def test_composite_datasource_operations(stix_objs1, stix_objs2): Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") ] - cds1.filters.update(query2) + cds1.filters.add(query2) results = cds1.query(query1) From 2fe9a0f2979cf85159a29d80bb791ce645c1e905 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:15:02 -0400 Subject: [PATCH 17/23] Finish adding new tests for TAXII datastore. closes #148 --- stix2/test/test_datastore_taxii.py | 104 ++++++++++++++++++++++++++--- 1 file changed, 95 insertions(+), 9 deletions(-) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index c20fbb6..dd81cd4 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -1,4 +1,5 @@ -from stix2 import Bundle, ThreatActor, TAXIICollectionSource, TAXIICollectionSink +from stix2 import (Bundle, ThreatActor, TAXIICollectionSink, + TAXIICollectionSource, TAXIICollectionStore) from stix2.datastore.filters import Filter import json @@ -27,7 +28,8 @@ class MockTAXIICollectionEndpoint(Collection): def get_objects(self, **filter_kwargs): self._verify_can_read() query_params = _filter_kwargs_to_query_params(filter_kwargs) - query_params = json.loads(query_params) + if not isinstance(query_params, dict): + query_params = json.loads(query_params) full_filter = BasicFilter(query_params or {}) objs = full_filter.process_filter( self.objects, @@ -90,6 +92,80 @@ def test_add_stix2_object(collection): tc_sink.add(ta) +def test_add_stix2_with_custom_object(collection): + tc_sink = TAXIICollectionStore(collection, allow_custom=True) + + # create new STIX threat-actor + ta = ThreatActor(name="Teddy Bear", + labels=["nation-state"], + sophistication="innovator", + resource_level="government", + goals=[ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector", + ], + foo="bar", + allow_custom=True) + + tc_sink.add(ta) + + +def test_add_list_object(collection, indicator): + tc_sink = TAXIICollectionSink(collection) + + # create new STIX threat-actor + ta = ThreatActor(name="Teddy Bear", + labels=["nation-state"], + sophistication="innovator", + resource_level="government", + goals=[ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector", + ]) + + tc_sink.add([ta, indicator]) + + +def test_add_stix2_bundle_object(collection): + tc_sink = TAXIICollectionSink(collection) + + # create new STIX threat-actor + ta = ThreatActor(name="Teddy Bear", + labels=["nation-state"], + sophistication="innovator", + resource_level="government", + goals=[ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector", + ]) + + tc_sink.add(Bundle(objects=[ta])) + + +def test_add_str_object(collection): + tc_sink = TAXIICollectionSink(collection) + + # create new STIX threat-actor + ta = """{ + "type": "threat-actor", + "id": "threat-actor--eddff64f-feb1-4469-b07c-499a73c96415", + "created": "2018-04-23T16:40:50.847Z", + "modified": "2018-04-23T16:40:50.847Z", + "name": "Teddy Bear", + "goals": [ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector" + ], + "sophistication": "innovator", + "resource_level": "government", + "labels": [ + "nation-state" + ] + }""" + + tc_sink.add(ta) + + def test_get_stix2_object(collection): tc_sink = TAXIICollectionSource(collection) @@ -98,7 +174,7 @@ def test_get_stix2_object(collection): assert objects -def test_parse_taxii_filters(): +def test_parse_taxii_filters(collection): query = [ Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), Filter("id", "=", "taxii stix object ID"), @@ -107,12 +183,12 @@ def test_parse_taxii_filters(): Filter("created_by_ref", "=", "Bane"), ] - taxii_filters_expected = set([ + taxii_filters_expected = [ Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), Filter("id", "=", "taxii stix object ID"), Filter("type", "=", "taxii stix object ID"), Filter("version", "=", "first") - ]) + ] ds = TAXIICollectionSource(collection) @@ -121,7 +197,7 @@ def test_parse_taxii_filters(): assert taxii_filters == taxii_filters_expected -def test_add_get_remove_filter(): +def test_add_get_remove_filter(collection): ds = TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way @@ -136,20 +212,30 @@ def test_add_get_remove_filter(): ds.filters.add(valid_filters[0]) assert len(ds.filters) == 1 - # Addin the same filter again will have no effect since `filters` uses a set + # Addin the same filter again will have no effect since `filters` acts + # like a set ds.filters.add(valid_filters[0]) assert len(ds.filters) == 1 ds.filters.add(valid_filters[1]) assert len(ds.filters) == 2 + ds.filters.add(valid_filters[2]) assert len(ds.filters) == 3 - assert set(valid_filters) == ds.filters + assert valid_filters == [f for f in ds.filters] # remove ds.filters.remove(valid_filters[0]) assert len(ds.filters) == 2 - ds.filters.update(valid_filters) + ds.filters.add(valid_filters) + + +def test_get_all_versions(collection): + ds = TAXIICollectionStore(collection) + + indicators = ds.all_versions('indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f') + # There are 3 indicators but 2 share the same 'modified' timestamp + assert len(indicators) == 2 From 7788ad8583514b62a8f96f37e7103a9f119cbd0f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:26:26 -0400 Subject: [PATCH 18/23] Add `medallion` dependency to Travis CI tests --- .isort.cfg | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/.isort.cfg b/.isort.cfg index cca9d19..56b0448 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -3,6 +3,7 @@ not_skip = __init__.py skip = workbench.py known_third_party = dateutil, + medallion, ordereddict, pytest, pytz, diff --git a/tox.ini b/tox.ini index 7c2e68c..683e137 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = pytest-cov coverage taxii2-client + medallion commands = pytest --ignore=stix2/test/test_workbench.py --cov=stix2 stix2/test/ --cov-report term-missing pytest stix2/test/test_workbench.py --cov=stix2 --cov-report term-missing --cov-append From 7e5c8a94096bdf90dc265d314f5fecf7974ab64d Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:27:36 -0400 Subject: [PATCH 19/23] Sort import statements in tests. --- stix2/test/test_datastore_filesystem.py | 1 - stix2/test/test_datastore_filters.py | 1 - stix2/test/test_datastore_memory.py | 2 +- stix2/test/test_datastore_taxii.py | 12 ++++++------ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py index bd1f02c..4139ab1 100644 --- a/stix2/test/test_datastore_filesystem.py +++ b/stix2/test/test_datastore_filesystem.py @@ -7,7 +7,6 @@ import pytest from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, FileSystemSource, FileSystemStore, Filter, Identity, Indicator, Malware, Relationship, properties) - from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS, diff --git a/stix2/test/test_datastore_filters.py b/stix2/test/test_datastore_filters.py index 8803e4b..5ffd051 100644 --- a/stix2/test/test_datastore_filters.py +++ b/stix2/test/test_datastore_filters.py @@ -4,7 +4,6 @@ from stix2 import parse from stix2.datastore.filters import Filter, apply_common_filters from stix2.utils import STIXdatetime, parse_into_datetime - stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", diff --git a/stix2/test/test_datastore_memory.py b/stix2/test/test_datastore_memory.py index dcec813..7a5bf10 100644 --- a/stix2/test/test_datastore_memory.py +++ b/stix2/test/test_datastore_memory.py @@ -2,7 +2,7 @@ import pytest from stix2.datastore import CompositeDataSource, make_id from stix2.datastore.filters import Filter -from stix2.datastore.memory import MemorySource, MemorySink +from stix2.datastore.memory import MemorySink, MemorySource def test_add_remove_composite_datasource(): diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index dd81cd4..f3b4f64 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -1,12 +1,12 @@ -from stix2 import (Bundle, ThreatActor, TAXIICollectionSink, - TAXIICollectionSource, TAXIICollectionStore) -from stix2.datastore.filters import Filter - import json -import pytest -from taxii2client import Collection, _filter_kwargs_to_query_params from medallion.filters.basic_filter import BasicFilter +import pytest +from taxii2client import Collection, _filter_kwargs_to_query_params + +from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource, + TAXIICollectionStore, ThreatActor) +from stix2.datastore.filters import Filter COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' From 2043a514e14fb77435e5638f2b59d17f3747540b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:47:32 -0400 Subject: [PATCH 20/23] Add two more tests for the `dict` case in TAXIICollectionSink --- stix2/test/test_datastore_taxii.py | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index f3b4f64..098f944 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -166,6 +166,59 @@ def test_add_str_object(collection): tc_sink.add(ta) +def test_add_dict_object(collection): + tc_sink = TAXIICollectionSink(collection) + + ta = { + "type": "threat-actor", + "id": "threat-actor--eddff64f-feb1-4469-b07c-499a73c96415", + "created": "2018-04-23T16:40:50.847Z", + "modified": "2018-04-23T16:40:50.847Z", + "name": "Teddy Bear", + "goals": [ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector" + ], + "sophistication": "innovator", + "resource_level": "government", + "labels": [ + "nation-state" + ] + } + + tc_sink.add(ta) + + +def test_add_dict_bundle_object(collection): + tc_sink = TAXIICollectionSink(collection) + + ta = { + "type": "bundle", + "id": "bundle--860ccc8d-56c9-4fda-9384-84276fb52fb1", + "spec_version": "2.0", + "objects": [ + { + "type": "threat-actor", + "id": "threat-actor--dc5a2f41-f76e-425a-81fe-33afc7aabd75", + "created": "2018-04-23T18:45:11.390Z", + "modified": "2018-04-23T18:45:11.390Z", + "name": "Teddy Bear", + "goals": [ + "compromising environment NGOs", + "water-hole attacks geared towards energy sector" + ], + "sophistication": "innovator", + "resource_level": "government", + "labels": [ + "nation-state" + ] + } + ] + } + + tc_sink.add(ta) + + def test_get_stix2_object(collection): tc_sink = TAXIICollectionSource(collection) From c15267971d7508331f956daddd1576df461663b6 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 14:55:06 -0400 Subject: [PATCH 21/23] Remove 'real_stix_objs1' as it is unused in the tests --- stix2/test/conftest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/stix2/test/conftest.py b/stix2/test/conftest.py index ddbcb7e..c73eafb 100644 --- a/stix2/test/conftest.py +++ b/stix2/test/conftest.py @@ -154,11 +154,6 @@ def stix_objs2(): return [ind6, ind7, ind8] -@pytest.fixture -def real_stix_objs1(stix_objs1): - return [stix2.parse(x) for x in stix_objs1] - - @pytest.fixture def real_stix_objs2(stix_objs2): return [stix2.parse(x) for x in stix_objs2] From 410296e6e1ca617fdb340dba26b2a150af747c9f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Apr 2018 15:22:38 -0400 Subject: [PATCH 22/23] Update file taxii_example.py --- examples/taxii_example.py | 57 +++++++++++++++------------------------ 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/examples/taxii_example.py b/examples/taxii_example.py index e102b17..978f0f2 100644 --- a/examples/taxii_example.py +++ b/examples/taxii_example.py @@ -1,54 +1,39 @@ -import json +from taxii2client import Collection -from stix2.datastore.taxii import TAXIIDataSource +import stix2 -# Flask TAXII server - developmental -ROOT = 'http://localhost:5000' -AUTH = {'user': 'mk', 'pass': 'Pass'} +# This example is based on the medallion server with default_data.json +# See https://github.com/oasis-open/cti-taxii-server for more information def main(): + collection = Collection("http://127.0.0.1:5000/trustgroup1/collections/52892447-4d7e-4f70-b94d-d7f22742ff63/", + user="admin", password="Password0") # instantiate TAXII data source - taxii = TAXIIDataSource(api_root=ROOT, auth=AUTH) + taxii = stix2.TAXIICollectionSource(collection) - # get (file watch indicator) - indicator_fw = taxii.get(id_="indicator--a932fcc6-e032-176c-126f-cb970a5a1ade") + # get (url watch indicator) + indicator_fw = taxii.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") print("\n\n-------Queried for Indicator - got:") - print(json.dumps(indicator_fw, indent=4)) + print(indicator_fw.serialize(indent=4)) - # all versions (file watch indicator - currently only 1. maybe Emmanuelle can add a version) - indicator_fw_versions = taxii.get(id_="indicator--a932fcc6-e032-176c-126f-cb970a5a1ade") + # all versions (url watch indicator - currently two) + indicator_fw_versions = taxii.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") print("\n\n------Queried for indicator (all_versions()) - got:") - print(json.dumps(indicator_fw_versions, indent=4)) + for indicator in indicator_fw_versions: + print(indicator.serialize(indent=4)) # add TAXII filter (ie filter should be passed to TAXII) - taxii_filter_ids, status = taxii.add_filter( - [ - { - "field": "type", - "op": "in", - "value": "malware" - } - ]) + query_filter = stix2.Filter("type", "in", "malware") - print("\n\n-------Added filter:") - print("Filter ID: {0}".format(taxii_filter_ids[0])) - print("Filter status: \n") - print(json.dumps(status, indent=4)) - print("filters: \n") - print(json.dumps(taxii.get_filters(), indent=4)) - - # get() - but with filter attached - malware = taxii.query() + # query() - but with filter attached. There are no malware objects in this collection + malwares = taxii.query(query=query_filter) print("\n\n\n--------Queried for Malware string (with above filter attached) - got:") - print(json.dumps(malware, indent=4)) - - # remove TAXII filter - taxii.remove_filter(taxii_filter_ids) - print("\n\n-------Removed filter(TAXII filter):") - print("filters: \n") - print(json.dumps(taxii.get_filters(), indent=4)) + for malware in malwares: + print(malware.serialize(indent=4)) + if not malwares: + print(malwares) if __name__ == "__main__": From 5d96cf11f131a41b0e4398d238767e753290f9ec Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 24 Apr 2018 15:55:46 -0400 Subject: [PATCH 23/23] Update TAXII2 filter fields in documentation --- docs/guide/datastore.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index c3980a0..2fabf88 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -356,16 +356,16 @@ "TAXII2 filter fields:\n", "\n", "* added_after\n", - "* match[id]\n", - "* match[type]\n", - "* match[version]\n", + "* id\n", + "* type\n", + "* version\n", "\n", "Supported operators:\n", "\n", "* =\n", "* !=\n", "* in\n", - "* >\n", + "* ```>```\n", "* < \n", "* ```>=```\n", "* <=\n",