From cccd42d9846b4e8631d3e029ccf5ef47fa8be5e1 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 11 Oct 2017 15:10:06 -0400 Subject: [PATCH 01/90] Make AS 'number' required, add missing props to SocketExt. closes #77 --- stix2/observables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stix2/observables.py b/stix2/observables.py index 57add29..aaec2d7 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -98,7 +98,7 @@ class AutonomousSystem(_Observable): _properties = OrderedDict() _properties.update([ ('type', TypeProperty(_type)), - ('number', IntegerProperty()), + ('number', IntegerProperty(required=True)), ('name', StringProperty()), ('rir', StringProperty()), ('extensions', ExtensionsProperty(enclosing_type=_type)), @@ -459,6 +459,8 @@ class SocketExt(_Extension): "SOCK_RDM", "SOCK_SEQPACKET", ])), + ('socket_descriptor', IntegerProperty()), + ('socket_handle', IntegerProperty()), ]) From 374539a6cc7fe0f380ab598ed39d791967119afb Mon Sep 17 00:00:00 2001 From: Greg Back Date: Thu, 12 Oct 2017 14:38:25 +0000 Subject: [PATCH 02/90] Don't try to re-parse STIX Objects when adding to Bundle. --- stix2/core.py | 11 ++++++++--- stix2/test/test_bundle.py | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index 8ee11f5..20bd187 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -7,9 +7,9 @@ from .base import _STIXBase from .common import MarkingDefinition from .properties import IDProperty, ListProperty, Property, TypeProperty from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, - IntrusionSet, Malware, ObservedData, Report, ThreatActor, - Tool, Vulnerability) -from .sro import Relationship, Sighting + IntrusionSet, Malware, ObservedData, Report, + STIXDomainObject, ThreatActor, Tool, Vulnerability) +from .sro import Relationship, Sighting, STIXRelationshipObject from .utils import get_dict @@ -20,6 +20,11 @@ class STIXObjectProperty(Property): super(STIXObjectProperty, self).__init__() def clean(self, value): + # Any STIX Object (SDO, SRO, or Marking Definition) can be added to + # a bundle with no further checks. + if isinstance(value, (STIXDomainObject, STIXRelationshipObject, + MarkingDefinition)): + return value try: dictified = get_dict(value) except ValueError: diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index c7c95a8..12b2149 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -158,3 +158,11 @@ def test_parse_unknown_type(): with pytest.raises(stix2.exceptions.ParseError) as excinfo: stix2.parse(unknown) assert str(excinfo.value) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator." + + +def test_stix_object_property(): + prop = stix2.core.STIXObjectProperty() + + identity = stix2.Identity(name="test", identity_class="individual") + assert prop.clean(identity) == identity + assert prop.clean(identity) is identity From 134267bfd6159caed17882c3a6a54ef49eb7346f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 12 Oct 2017 11:20:20 -0400 Subject: [PATCH 03/90] Add FileSystem test fixture, test adding bundle --- stix2/sources/filesystem.py | 10 ++++++---- stix2/test/test_data_sources.py | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 103b882..f11df65 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -8,10 +8,12 @@ TODO: import json import os -from stix2.base import _STIXBase +from stix2.common import MarkingDefinition from stix2.core import Bundle, parse +from stix2.sdo import STIXDomainObject from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters +from stix2.sro import STIXRelationshipObject from stix2.utils import deduplicate @@ -81,7 +83,7 @@ class FileSystemSink(DataSink): # Bundle() can take dict or STIX obj as argument f.write(str(Bundle(stix_obj))) - if isinstance(stix_data, _STIXBase): + if isinstance(stix_data, (STIXDomainObject, STIXRelationshipObject, MarkingDefinition)): # adding python STIX object _check_path_and_write(self._stix_dir, stix_data) @@ -94,11 +96,11 @@ class FileSystemSink(DataSink): # adding json-formatted STIX _check_path_and_write(self._stix_dir, stix_data) - elif isinstance(stix_data, str): + elif isinstance(stix_data, (str, Bundle)): # adding json encoded string of STIX content stix_data = parse(stix_data) if stix_data["type"] == "bundle": - for stix_obj in stix_data["objects"]: + for stix_obj in stix_data.get("objects", []): self.add(stix_obj) else: self.add(stix_data) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 689fe8c..3ee81a9 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,10 +1,11 @@ import os +import shutil import pytest from taxii2client import Collection -from stix2 import (Campaign, FileSystemSink, FileSystemSource, FileSystemStore, - Filter, MemorySource, MemoryStore) +from stix2 import (Bundle, Campaign, FileSystemSink, FileSystemSource, + FileSystemStore, Filter, MemorySource, MemoryStore) from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters @@ -29,6 +30,15 @@ def ds(): return DataSource() +@pytest.fixture +def fs_store(): + # create + yield FileSystemStore(FS_PATH) + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign")) + + IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -681,10 +691,7 @@ def test_filesystem_sink(): os.rmdir(os.path.join(FS_PATH, "campaign")) -def test_filesystem_store(): - # creation - fs_store = FileSystemStore(FS_PATH) - +def test_filesystem_store(fs_store): # get() coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" @@ -716,3 +723,8 @@ def test_filesystem_store(): # remove campaign dir os.rmdir(os.path.join(FS_PATH, "campaign")) + + +def test_filesystem_add_object_with_custom_property_in_bundle(fs_store): + bundle = Bundle() + fs_store.add(bundle) From 6deaddc04eaee700042ae96da91473a63b4e4fa1 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 12 Oct 2017 14:08:32 -0400 Subject: [PATCH 04/90] Make sure custom properties are returned by object. closes #85 --- stix2/base.py | 9 ++++++++- stix2/test/test_custom.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/stix2/base.py b/stix2/base.py index 5307393..b0cf6ff 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,7 +40,14 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - return list(self._properties.keys()) + props = set(self._properties.keys()) + custom_props = list(set(self._inner.keys()) - props) + custom_props.sort() + + all_properties = list(self._properties.keys()) + all_properties.extend(custom_props) # Any custom properties to the bottom + + return all_properties def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 48529b9..c5726b8 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -91,6 +91,7 @@ def test_custom_property_in_bundled_object(): bundle = stix2.Bundle(identity, allow_custom=True) assert bundle.objects[0].x_foo == "bar" + assert '"x_foo": "bar"' in str(bundle) @stix2.sdo.CustomObject('x-new-type', [ From 84094e9f79ee0fef741f0335d99b9c6268f23b9b Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 18 Oct 2017 08:27:12 -0400 Subject: [PATCH 05/90] Separate out filesystem tests --- stix2/test/test_data_sources.py | 222 +-------------------------- stix2/test/test_filesystem.py | 255 ++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 221 deletions(-) create mode 100644 stix2/test/test_filesystem.py diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 3ee81a9..d7fd576 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,18 +1,13 @@ -import os -import shutil - import pytest from taxii2client import Collection -from stix2 import (Bundle, Campaign, FileSystemSink, FileSystemSource, - FileSystemStore, Filter, MemorySource, MemoryStore) +from stix2 import Filter, MemorySource, MemoryStore from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters from stix2.utils import deduplicate COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' -FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") class MockTAXIIClient(object): @@ -30,15 +25,6 @@ def ds(): return DataSource() -@pytest.fixture -def fs_store(): - # create - yield FileSystemStore(FS_PATH) - - # remove campaign dir - shutil.rmtree(os.path.join(FS_PATH, "campaign")) - - IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -522,209 +508,3 @@ def test_composite_datasource_operations(): # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 - - -def test_filesytem_source(): - # creation - fs_source = FileSystemSource(FS_PATH) - assert fs_source.stix_dir == FS_PATH - - # get object - mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") - assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" - assert mal.name == "Rover" - - # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) - id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") - assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" - assert id_.name == "The MITRE Corporation" - assert id_.type == "identity" - - # query - intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) - assert len(intrusion_sets) == 2 - assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] - assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] - - is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] - assert "DragonOK" in is_1.aliases - assert len(is_1.external_references) == 4 - - # query2 - is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) - assert len(is_2) == 1 - - is_2 = is_2[0] - assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" - assert is_2.type == "attack-pattern" - - -def test_filesystem_sink(): - # creation - fs_sink = FileSystemSink(FS_PATH) - assert fs_sink.stix_dir == FS_PATH - - fs_source = FileSystemSource(FS_PATH) - - # Test all the ways stix objects can be added (via different supplied forms) - - # add python stix object - camp1 = Campaign(name="Hannibal", - objective="Targeting Italian and Spanish Diplomat internet accounts", - aliases=["War Elephant"]) - - fs_sink.add(camp1) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) - - camp1_r = fs_source.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == "Hannibal" - assert "War Elephant" in camp1_r.aliases - - # add stix object dict - camp2 = { - "name": "Aurelius", - "type": "campaign", - "objective": "German and French Intelligence Services", - "aliases": ["Purple Robes"], - "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add(camp2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) - - camp2_r = fs_source.get(camp2["id"]) - assert camp2_r.id == camp2["id"] - assert camp2_r.name == camp2["name"] - assert "Purple Robes" in camp2_r.aliases - - # add stix bundle dict - bund = { - "type": "bundle", - "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", - "spec_version": "2.0", - "objects": [ - { - "name": "Atilla", - "type": "campaign", - "objective": "Bulgarian, Albanian and Romanian Intelligence Services", - "aliases": ["Huns"], - "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - ] - } - - fs_sink.add(bund) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) - - camp3_r = fs_source.get(bund["objects"][0]["id"]) - assert camp3_r.id == bund["objects"][0]["id"] - assert camp3_r.name == bund["objects"][0]["name"] - assert "Huns" in camp3_r.aliases - - # add json-encoded stix obj - camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' - - fs_sink.add(camp4) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") - assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" - assert camp4_r.name == "Ghengis Khan" - - # add json-encoded stix bundle - bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ - ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' - fs_sink.add(bund2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") - assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" - assert camp5_r.name == "Spartacus" - - # add list of objects - camp6 = Campaign(name="Comanche", - objective="US Midwest manufacturing firms, oil refineries, and businesses", - aliases=["Horse Warrior"]) - - camp7 = { - "name": "Napolean", - "type": "campaign", - "objective": "Central and Eastern Europe military commands and departments", - "aliases": ["The Frenchmen"], - "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add([camp6, camp7]) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp6_r = fs_source.get(camp6.id) - assert camp6_r.id == camp6.id - assert "Horse Warrior" in camp6_r.aliases - - camp7_r = fs_source.get(camp7["id"]) - assert camp7_r.id == camp7["id"] - assert "The Frenchmen" in camp7_r.aliases - - # remove all added objects - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) - - # remove campaign dir (that was added in course of testing) - os.rmdir(os.path.join(FS_PATH, "campaign")) - - -def test_filesystem_store(fs_store): - # get() - coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") - assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" - assert coa.type == "course-of-action" - - # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) - rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] - assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" - assert rel.type == "relationship" - - # query() - tools = fs_store.query([Filter("labels", "in", "tool")]) - assert len(tools) == 2 - assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] - assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] - - # add() - camp1 = Campaign(name="Great Heathen Army", - objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", - aliases=["Ragnar"]) - fs_store.add(camp1) - - camp1_r = fs_store.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == camp1.name - - # remove - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - - # remove campaign dir - os.rmdir(os.path.join(FS_PATH, "campaign")) - - -def test_filesystem_add_object_with_custom_property_in_bundle(fs_store): - bundle = Bundle() - fs_store.add(bundle) diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py new file mode 100644 index 0000000..9ef40a6 --- /dev/null +++ b/stix2/test/test_filesystem.py @@ -0,0 +1,255 @@ +import os +import shutil + +import pytest + +from stix2 import (Bundle, Campaign, FileSystemSink, FileSystemSource, + FileSystemStore, Filter) + +FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") + + +@pytest.fixture +def fs_store(): + # create + yield FileSystemStore(FS_PATH) + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_source(): + # create + fs = FileSystemSource(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_sink(): + # create + fs = FileSystemSink(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +def test_filesytem_source_get_object(fs_source): + # get object + mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" + assert mal.name == "Rover" + + +def test_filesytem_source_all_versions(fs_source): + # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) + id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") + assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" + assert id_.name == "The MITRE Corporation" + assert id_.type == "identity" + + +def test_filesytem_source_query_single(fs_source): + # query2 + is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) + assert len(is_2) == 1 + + is_2 = is_2[0] + assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" + assert is_2.type == "attack-pattern" + + +def test_filesytem_source_query_multiple(fs_source): + # query + intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) + assert len(intrusion_sets) == 2 + assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] + assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] + + is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] + assert "DragonOK" in is_1.aliases + assert len(is_1.external_references) == 4 + + +def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): + # add python stix object + camp1 = Campaign(name="Hannibal", + objective="Targeting Italian and Spanish Diplomat internet accounts", + aliases=["War Elephant"]) + + fs_sink.add(camp1) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) + + camp1_r = fs_source.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == "Hannibal" + assert "War Elephant" in camp1_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): + # add stix object dict + camp2 = { + "name": "Aurelius", + "type": "campaign", + "objective": "German and French Intelligence Services", + "aliases": ["Purple Robes"], + "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add(camp2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) + + camp2_r = fs_source.get(camp2["id"]) + assert camp2_r.id == camp2["id"] + assert camp2_r.name == camp2["name"] + assert "Purple Robes" in camp2_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) + + +def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): + # add stix bundle dict + bund = { + "type": "bundle", + "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", + "spec_version": "2.0", + "objects": [ + { + "name": "Atilla", + "type": "campaign", + "objective": "Bulgarian, Albanian and Romanian Intelligence Services", + "aliases": ["Huns"], + "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + ] + } + + fs_sink.add(bund) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) + + camp3_r = fs_source.get(bund["objects"][0]["id"]) + assert camp3_r.id == bund["objects"][0]["id"] + assert camp3_r.name == bund["objects"][0]["name"] + assert "Huns" in camp3_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) + + +def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): + # add json-encoded stix obj + camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' + + fs_sink.add(camp4) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") + assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + assert camp4_r.name == "Ghengis Khan" + + os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) + + +def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): + # add json-encoded stix bundle + bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ + ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' + fs_sink.add(bund2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") + assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + assert camp5_r.name == "Spartacus" + + os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) + + +def test_filesystem_sink_add_objects_list(fs_sink, fs_source): + # add list of objects + camp6 = Campaign(name="Comanche", + objective="US Midwest manufacturing firms, oil refineries, and businesses", + aliases=["Horse Warrior"]) + + camp7 = { + "name": "Napolean", + "type": "campaign", + "objective": "Central and Eastern Europe military commands and departments", + "aliases": ["The Frenchmen"], + "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add([camp6, camp7]) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp6_r = fs_source.get(camp6.id) + assert camp6_r.id == camp6.id + assert "Horse Warrior" in camp6_r.aliases + + camp7_r = fs_source.get(camp7["id"]) + assert camp7_r.id == camp7["id"] + assert "The Frenchmen" in camp7_r.aliases + + # remove all added objects + os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) + os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) + + +def test_filesystem_store_get(fs_store): + # get() + coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") + assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" + assert coa.type == "course-of-action" + + +def test_filesystem_store_all_versions(fs_store): + # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) + rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] + assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" + assert rel.type == "relationship" + + +def test_filesystem_store_query(fs_store): + # query() + tools = fs_store.query([Filter("labels", "in", "tool")]) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_add(fs_store): + # add() + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + # remove + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_add_object_with_custom_property_in_bundle(fs_store): + bundle = Bundle() + fs_store.add(bundle) From be3e841ecb3ba43018bec27520025426ea613473 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 18 Oct 2017 13:58:24 -0400 Subject: [PATCH 06/90] New object test files --- stix2/test/test_location.py | 0 stix2/test/test_note.py | 0 stix2/test/test_opinion.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 stix2/test/test_location.py create mode 100644 stix2/test/test_note.py create mode 100644 stix2/test/test_opinion.py diff --git a/stix2/test/test_location.py b/stix2/test/test_location.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/test/test_note.py b/stix2/test/test_note.py new file mode 100644 index 0000000..e69de29 diff --git a/stix2/test/test_opinion.py b/stix2/test/test_opinion.py new file mode 100644 index 0000000..e69de29 From e1d8c2872ef5905f6b08945c1c1ca6093b1ac457 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 18 Oct 2017 15:42:21 -0400 Subject: [PATCH 07/90] Don't Bundlify data in FileSystemStore Don't wrap objects in a Bundle when adding them to a FileSystemStore, but still support getting objects from FileSystemStore that were saved in a bundle. Also: - Add option to allow custom content in FileSystemStore - Simplify an if statement - Improve FileSystem docstrings - Remove an unnecessary parse() call in FileSystemSource.get() --- stix2/sources/__init__.py | 34 +++--- stix2/sources/filesystem.py | 101 +++++++++--------- ...-d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json | 21 ++-- stix2/test/test_filesystem.py | 60 ++++++++++- 4 files changed, 127 insertions(+), 89 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 9d46ba9..9c5510e 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -44,7 +44,7 @@ class DataStore(object): self.source = source self.sink = sink - def get(self, stix_id): + def get(self, stix_id, allow_custom=False): """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. @@ -57,9 +57,9 @@ class DataStore(object): object specified by the "id". """ - return self.source.get(stix_id) + return self.source.get(stix_id, allow_custom=allow_custom) - def all_versions(self, stix_id): + def all_versions(self, stix_id, allow_custom=False): """Retrieve all versions of a single STIX object by ID. Implement: Translate all_versions() call to the appropriate DataSource call @@ -71,9 +71,9 @@ class DataStore(object): stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id) + return self.source.all_versions(stix_id, allow_custom=allow_custom) - def query(self, query): + def query(self, query, allow_custom=False): """Retrieve STIX objects matching a set of filters. Implement: Specific data source API calls, processing, @@ -89,7 +89,7 @@ class DataStore(object): """ return self.source.query(query=query) - def add(self, stix_objs): + def add(self, stix_objs, allow_custom=False): """Store STIX objects. Translates add() to the appropriate DataSink call. @@ -97,7 +97,7 @@ class DataStore(object): Args: stix_objs (list): a list of STIX objects """ - return self.sink.add(stix_objs) + return self.sink.add(stix_objs, allow_custom=allow_custom) class DataSink(object): @@ -111,7 +111,7 @@ class DataSink(object): def __init__(self): self.id = make_id() - def add(self, stix_objs): + def add(self, stix_objs, allow_custom=False): """Store STIX objects. Implement: Specific data sink API calls, processing, @@ -139,7 +139,7 @@ class DataSource(object): self.id = make_id() self.filters = set() - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False): """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -158,7 +158,7 @@ class DataSource(object): """ raise NotImplementedError() - def all_versions(self, stix_id, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """ Implement: Similar to get() except returns list of all object versions of the specified "id". In addition, implement the specific data @@ -179,7 +179,7 @@ class DataSource(object): """ raise NotImplementedError() - def query(self, query, _composite_filters=None): + def query(self, query, _composite_filters=None, allow_custom=False): """ Implement:Implement the specific data source API calls, processing, functionality required for retrieving query from the data source @@ -224,7 +224,7 @@ class CompositeDataSource(DataSource): super(CompositeDataSource, self).__init__() self.data_sources = [] - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False): """Retrieve STIX object by STIX ID Federated retrieve method, iterates through all DataSources @@ -259,7 +259,7 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds in self.data_sources: - data = ds.get(stix_id=stix_id, _composite_filters=all_filters) + data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) if data: all_data.append(data) @@ -272,7 +272,7 @@ class CompositeDataSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """Retrieve STIX objects by STIX ID Federated all_versions retrieve method - iterates through all DataSources @@ -305,7 +305,7 @@ class CompositeDataSource(DataSource): # retrieve STIX objects from all configured data sources for ds in self.data_sources: - data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters) + data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -315,7 +315,7 @@ class CompositeDataSource(DataSource): return all_data - def query(self, query=None, _composite_filters=None): + def query(self, query=None, _composite_filters=None, allow_custom=False): """Retrieve STIX objects that match query Federate the query to all DataSources attached to the @@ -351,7 +351,7 @@ class CompositeDataSource(DataSource): # federate query to all attached data sources, # pass composite filters to id for ds in self.data_sources: - data = ds.query(query=query, _composite_filters=all_filters) + data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index f11df65..1dcf85b 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -18,9 +18,8 @@ from stix2.utils import deduplicate class FileSystemStore(DataStore): - """FileSystemStore + """Interface to a file directory of STIX objects. - Provides an interface to an file directory of STIX objects. FileSystemStore is a wrapper around a paired FileSystemSink and FileSystemSource. @@ -40,10 +39,8 @@ class FileSystemStore(DataStore): class FileSystemSink(DataSink): - """FileSystemSink - - Provides an interface for adding/pushing STIX objects - to file directory of STIX objects. + """Interface for adding/pushing STIX objects to file directory of STIX + objects. Can be paired with a FileSystemSource, together as the two components of a FileSystemStore. @@ -63,15 +60,19 @@ class FileSystemSink(DataSink): def stix_dir(self): return self._stix_dir - def add(self, stix_data=None): - """add STIX objects to file directory + def add(self, stix_data=None, allow_custom=False): + """Add STIX objects to file directory. Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content - in a STIX object(or list of), dict (or list of), or a STIX 2.0 - json encoded string + in a STIX object (or list of), dict (or list of), or a STIX 2.0 + json encoded string. + + Note: + ``stix_data`` can be a Bundle object, but each object in it will be + saved separately; you will be able to retrieve any of the objects + the Bundle contained, but not the Bundle itself. - TODO: Bundlify STIX content or no? When dumping to disk. """ def _check_path_and_write(stix_dir, stix_obj): path = os.path.join(stix_dir, stix_obj["type"], stix_obj["id"] + ".json") @@ -80,45 +81,41 @@ class FileSystemSink(DataSink): os.makedirs(os.path.dirname(path)) with open(path, "w") as f: - # Bundle() can take dict or STIX obj as argument - f.write(str(Bundle(stix_obj))) + f.write(str(stix_obj)) if isinstance(stix_data, (STIXDomainObject, STIXRelationshipObject, MarkingDefinition)): # adding python STIX object _check_path_and_write(self._stix_dir, stix_data) - elif isinstance(stix_data, dict): + elif isinstance(stix_data, (str, dict)): + stix_data = parse(stix_data, allow_custom) if stix_data["type"] == "bundle": - # adding json-formatted Bundle - extracting STIX objects - for stix_obj in stix_data["objects"]: + # extract STIX objects + for stix_obj in stix_data.get("objects", []): self.add(stix_obj) else: # adding json-formatted STIX _check_path_and_write(self._stix_dir, stix_data) - elif isinstance(stix_data, (str, Bundle)): - # adding json encoded string of STIX content - stix_data = parse(stix_data) - if stix_data["type"] == "bundle": - for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) - else: - self.add(stix_data) + elif isinstance(stix_data, Bundle): + # recursively add individual STIX objects + for stix_obj in stix_data.get("objects", []): + self.add(stix_obj) elif isinstance(stix_data, list): - # if list, recurse call on individual STIX objects + # recursively add individual STIX objects for stix_obj in stix_data: self.add(stix_obj) else: - raise ValueError("stix_data must be a STIX object(or list of), json formatted STIX(or list of) or a json formatted STIX bundle") + raise ValueError("stix_data must be a STIX object (or list of), " + "json formatted STIX (or list of), " + "or a json formatted STIX bundle") class FileSystemSource(DataSource): - """FileSystemSource - - Provides an interface for searching/retrieving - STIX objects from a STIX object file directory. + """Interface for searching/retrieving STIX objects from a STIX object file + directory. Can be paired with a FileSystemSink, together as the two components of a FileSystemStore. @@ -138,8 +135,8 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None): - """retrieve STIX object from file directory via STIX ID + def get(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. @@ -155,18 +152,17 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - stix_obj = parse(stix_obj) else: stix_obj = None return stix_obj - def all_versions(self, stix_id, _composite_filters=None): - """retrieve STIX object from file directory via STIX ID, all versions + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Pass call to get(). @@ -181,11 +177,12 @@ class FileSystemSource(DataSource): (list): of STIX objects that has the supplied STIX ID. The STIX objects are loaded from their json files, parsed into a python STIX objects and then returned - """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None): - """search and retrieve STIX objects based on the complete query + """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] + + def query(self, query=None, _composite_filters=None, allow_custom=False): + """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters attached to MemorySource, and any filters passed from a @@ -275,34 +272,32 @@ class FileSystemSource(DataSource): for path in include_paths: for root, dirs, files in os.walk(path): for file_ in files: - if id_: - if id_ == file_.split(".")[0]: - # since ID is specified in one of filters, can evaluate against filename first without loading - stix_obj = json.load(open(os.path.join(root, file_)))["objects"][0] - # check against other filters, add if match - all_data.extend(apply_common_filters([stix_obj], query)) - else: + if not id_ or id_ == file_.split(".")[0]: # have to load into memory regardless to evaluate other filters - stix_obj = json.load(open(os.path.join(root, file_)))["objects"][0] + stix_obj = json.load(open(os.path.join(root, file_))) + if stix_obj.get('type', '') == 'bundle': + stix_obj = stix_obj['objects'][0] + # check against other filters, add if match all_data.extend(apply_common_filters([stix_obj], query)) all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom) for stix_obj_dict in all_data] return stix_objs def _parse_file_filters(self, query): - """utility method to extract STIX common filters - that can used to possibly speed up querying STIX objects - from the file system + """Extract STIX common filters. + + Possibly speeds up querying STIX objects from the file system. Extracts filters that are for the "id" and "type" field of a STIX object. As the file directory is organized by STIX object type with filenames that are equivalent to the STIX object ID, these filters can be used first to reduce the - search space of a FileSystemStore(or FileSystemSink) + search space of a FileSystemStore (or FileSystemSink). + """ file_filters = set() for filter_ in query: diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json index 5bfb8bb..cb9cfe2 100755 --- a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json @@ -1,16 +1,9 @@ { - "id": "bundle--2ed6ab6a-ca68-414f-8493-e4db8b75dd51", - "objects": [ - { - "created": "2017-05-31T21:30:41.022744Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", - "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", - "modified": "2017-05-31T21:30:41.022744Z", - "name": "Data from Network Shared Drive Mitigation", - "type": "course-of-action" - } - ], - "spec_version": "2.0", - "type": "bundle" + "created": "2017-05-31T21:30:41.022744Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", + "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", + "modified": "2017-05-31T21:30:41.022744Z", + "name": "Data from Network Shared Drive Mitigation", + "type": "course-of-action" } \ No newline at end of file diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 9ef40a6..8b9aa22 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -3,8 +3,8 @@ import shutil import pytest -from stix2 import (Bundle, Campaign, FileSystemSink, FileSystemSource, - FileSystemStore, Filter) +from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, + FileSystemSource, FileSystemStore, Filter, properties) FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") @@ -213,8 +213,13 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source): os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) -def test_filesystem_store_get(fs_store): - # get() +def test_filesystem_store_get_stored_as_bundle(fs_store): + coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f") + assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f" + assert coa.type == "course-of-action" + + +def test_filesystem_store_get_stored_as_object(fs_store): coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" assert coa.type == "course-of-action" @@ -250,6 +255,51 @@ def test_filesystem_store_add(fs_store): os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) -def test_filesystem_add_object_with_custom_property_in_bundle(fs_store): +def test_filesystem_add_bundle_object(fs_store): bundle = Bundle() fs_store.add(bundle) + + +def test_filesystem_object_with_custom_property(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + fs_store.add(camp, True) + + camp_r = fs_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_object_with_custom_property_in_bundle(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + bundle = Bundle(camp, allow_custom=True) + fs_store.add(bundle, True) + + camp_r = fs_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_custom_object(fs_store): + @CustomObject('x-new-obj', [ + ('property1', properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + newobj = NewObj(property1='something') + fs_store.add(newobj, True) + + newobj_r = fs_store.get(newobj.id, True) + assert newobj_r.id == newobj.id + assert newobj_r.property1 == 'something' + + # remove dir + shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) From c6d5eee0839a51d21c3c171c88f4be59a88be6aa Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 18 Oct 2017 18:31:46 -0400 Subject: [PATCH 08/90] Separate out Memory datatstore tests Makes sure custom content can be added to a MemoryStore. --- stix2/sources/memory.py | 86 ++++++------ stix2/test/test_data_sources.py | 24 +--- stix2/test/test_memory.py | 237 ++++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+), 66 deletions(-) create mode 100644 stix2/test/test_memory.py diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0d5901e..1b738ab 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,8 +24,8 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None): - """Adds STIX objects to MemoryStore/Sink. +def _add(store, stix_data=None, allow_custom=False): + """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. Recursive function, breaks down STIX Bundles and lists. @@ -41,35 +41,35 @@ def _add(store, stix_data=None): elif isinstance(stix_data, dict): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects - for stix_obj in stix_data["objects"]: - _add(store, stix_obj) + for stix_obj in stix_data.get("objects", []): + _add(store, stix_obj, allow_custom=allow_custom) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data) + stix_data = parse(stix_data, allow_custom=allow_custom) if stix_data["type"] == "bundle": # recurse on each STIX object in bundle - for stix_obj in stix_data: - _add(store, stix_obj) + for stix_obj in stix_data.get("objects", []): + _add(store, stix_obj, allow_custom=allow_custom) else: _add(store, stix_data) elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj) + _add(store, stix_obj, allow_custom=allow_custom) else: raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") class MemoryStore(DataStore): - """Provides an interface to an in-memory dictionary - of STIX objects. MemoryStore is a wrapper around a paired - MemorySink and MemorySource + """Interface to an in-memory dictionary of STIX objects. + + MemoryStore is a wrapper around a paired MemorySink and MemorySource. Note: It doesn't make sense to create a MemoryStore by passing in existing MemorySource and MemorySink because there could @@ -87,26 +87,25 @@ class MemoryStore(DataStore): """ - def __init__(self, stix_data=None): + def __init__(self, stix_data=None, allow_custom=False): super(MemoryStore, self).__init__() self._data = {} if stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - self.source = MemorySource(stix_data=self._data, _store=True) - self.sink = MemorySink(stix_data=self._data, _store=True) + self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) + self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) - def save_to_file(self, file_path): - return self.sink.save_to_file(file_path=file_path) + def save_to_file(self, file_path, allow_custom=False): + return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) - def load_from_file(self, file_path): - return self.source.load_from_file(file_path=file_path) + def load_from_file(self, file_path, allow_custom=False): + return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) class MemorySink(DataSink): - """Provides an interface for adding/pushing STIX objects - to an in-memory dictionary. + """Interface for adding/pushing STIX objects to an in-memory dictionary. Designed to be paired with a MemorySource, together as the two components of a MemoryStore. @@ -125,24 +124,24 @@ class MemorySink(DataSink): a MemorySource """ - def __init__(self, stix_data=None, _store=False): + def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySink, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - def add(self, stix_data): + def add(self, stix_data, allow_custom=False): """add STIX objects to in-memory dictionary maintained by the MemorySink (MemoryStore) see "_add()" for args documentation """ - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - def save_to_file(self, file_path): + def save_to_file(self, file_path, allow_custom=False): """write SITX objects in in-memory dictionary to json file, as a STIX Bundle Args: @@ -153,12 +152,12 @@ class MemorySink(DataSink): if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(self._data.values()))) + f.write(str(Bundle(self._data.values(), allow_custom=allow_custom))) class MemorySource(DataSource): - """Provides an interface for searching/retrieving - STIX objects from an in-memory dictionary. + """Interface for searching/retrieving STIX objects from an in-memory + dictionary. Designed to be paired with a MemorySink, together as the two components of a MemoryStore. @@ -177,17 +176,17 @@ class MemorySource(DataSource): a MemorySink """ - def __init__(self, stix_data=None, _store=False): + def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySource, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - def get(self, stix_id, _composite_filters=None): - """retrieve STIX object from in-memory dict via STIX ID + def get(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. @@ -200,8 +199,8 @@ class MemorySource(DataSource): ID. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory as they are supplied (either as python dictionary or STIX object), it is returned in the same form as it as added - """ + """ if _composite_filters is None: # if get call is only based on 'id', no need to search, just retrieve from dict try: @@ -213,15 +212,15 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) # reduce to most recent version stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] return stix_obj - def all_versions(self, stix_id, _composite_filters=None): - """retrieve STIX objects from in-memory dict via STIX ID, all versions of it + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it Note: Since Memory sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Translate call to get(). @@ -239,10 +238,10 @@ class MemorySource(DataSource): is returned in the same form as it as added """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] - def query(self, query=None, _composite_filters=None): - """search and retrieve STIX objects based on the complete query + def query(self, query=None, _composite_filters=None, allow_custom=False): + """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters attached to MemorySource, and any filters passed from a @@ -281,15 +280,16 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path): - """load STIX data from json file + def load_from_file(self, file_path, allow_custom=False): + """Load STIX data from json file. File format is expected to be a single json STIX object or json STIX bundle Args: file_path (str): file path to load STIX data from + """ file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index d7fd576..6f47de8 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,7 @@ import pytest from taxii2client import Collection -from stix2 import Filter, MemorySource, MemoryStore +from stix2 import Filter, MemorySource from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters @@ -144,28 +144,6 @@ def test_ds_abstract_class_smoke(): ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) -def test_memory_store_smoke(): - # Initialize MemoryStore with dict - ms = MemoryStore(STIX_OBJS1) - - # Add item to sink - ms.add(dict(id="bundle--%s" % make_id(), - objects=STIX_OBJS2, - spec_version="2.0", - type="bundle")) - - resp = ms.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - assert len(resp) == 1 - - resp = ms.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") - assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" - - query = [Filter('type', '=', 'malware')] - - resp = ms.query(query) - assert len(resp) == 0 - - def test_ds_taxii(collection): ds = taxii.TAXIICollectionSource(collection) assert ds.collection is not None diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py new file mode 100644 index 0000000..e8ba56b --- /dev/null +++ b/stix2/test/test_memory.py @@ -0,0 +1,237 @@ +import pytest + +from stix2 import (Bundle, Campaign, CustomObject, Filter, MemorySource, + MemoryStore, properties) +from stix2.sources import make_id + +IND1 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND2 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND3 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND4 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND5 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND6 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND7 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND8 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} + +STIX_OBJS2 = [IND6, IND7, IND8] +STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] + + +@pytest.fixture +def mem_store(): + yield MemoryStore(STIX_OBJS1) + + +@pytest.fixture +def mem_source(): + yield MemorySource(STIX_OBJS1) + + +def test_memory_source_get(mem_source): + resp = mem_source.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + + +def test_memory_source_get_nonexistant_object(mem_source): + resp = mem_source.get("tool--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp is None + + +def test_memory_store_all_versions(mem_store): + # Add bundle of items to sink + mem_store.add(dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS2, + spec_version="2.0", + type="bundle")) + + resp = mem_store.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(resp) == 1 # MemoryStore can only store 1 version of each object + + +def test_memory_store_query(mem_store): + query = [Filter('type', '=', 'malware')] + + resp = mem_store.query(query) + assert len(resp) == 0 + + +def test_memory_store_add_stix_object_str(mem_store): + # add stix object string + camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" + camp_name = "Aurelius" + camp_alias = "Purple Robes" + camp = """{ + "name": "%s", + "type": "campaign", + "objective": "German and French Intelligence Services", + "aliases": ["%s"], + "id": "%s", + "created": "2017-05-31T21:31:53.197755Z" + }""" % (camp_name, camp_alias, camp_id) + + mem_store.add(camp) + + camp_r = mem_store.get(camp_id) + assert camp_r["id"] == camp_id + assert camp_r["name"] == camp_name + assert camp_alias in camp_r["aliases"] + + +def test_memory_store_add_stix_bundle_str(mem_store): + # add stix bundle string + camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" + camp_name = "Atilla" + camp_alias = "Huns" + bund = """{ + "type": "bundle", + "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", + "spec_version": "2.0", + "objects": [ + { + "name": "%s", + "type": "campaign", + "objective": "Bulgarian, Albanian and Romanian Intelligence Services", + "aliases": ["%s"], + "id": "%s", + "created": "2017-05-31T21:31:53.197755Z" + } + ] + }""" % (camp_name, camp_alias, camp_id) + + mem_store.add(bund) + + camp_r = mem_store.get(camp_id) + assert camp_r["id"] == camp_id + assert camp_r["name"] == camp_name + assert camp_alias in camp_r["aliases"] + + +def test_memory_store_object_with_custom_property(mem_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + mem_store.add(camp, True) + + camp_r = mem_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_memory_store_object_with_custom_property_in_bundle(mem_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + bundle = Bundle(camp, allow_custom=True) + mem_store.add(bundle, True) + + bundle_r = mem_store.get(bundle.id, True) + camp_r = bundle_r['objects'][0] + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_memory_store_custom_object(mem_store): + @CustomObject('x-new-obj', [ + ('property1', properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + newobj = NewObj(property1='something') + mem_store.add(newobj, True) + + newobj_r = mem_store.get(newobj.id, True) + assert newobj_r.id == newobj.id + assert newobj_r.property1 == 'something' From 476cd1ed5bb84f4a603466ada2556df09368f6ee Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 18 Oct 2017 18:34:08 -0400 Subject: [PATCH 09/90] Add option for custom content to TAXII datastore --- stix2/sources/taxii.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 7ecedca..619af0c 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -41,7 +41,7 @@ class TAXIICollectionSink(DataSink): super(TAXIICollectionSink, self).__init__() self.collection = collection - def add(self, stix_data): + def add(self, stix_data, allow_custom=False): """add/push STIX content to TAXII Collection endpoint Args: @@ -53,27 +53,27 @@ class TAXIICollectionSink(DataSink): if isinstance(stix_data, _STIXBase): # adding python STIX object - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) elif isinstance(stix_data, dict): # adding python dict (of either Bundle or STIX obj) if stix_data["type"] == "bundle": bundle = stix_data else: - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj) + self.add(obj, allow_custom=allow_custom) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data) + stix_data = parse(stix_data, allow_custom=allow_custom) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) else: raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") @@ -93,7 +93,7 @@ class TAXIICollectionSource(DataSource): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False): """retrieve STIX object from local/remote STIX Collection endpoint. @@ -125,13 +125,13 @@ class TAXIICollectionSource(DataSource): if len(stix_obj): stix_obj = stix_obj[0] - stix_obj = parse(stix_obj) + stix_obj = parse(stix_obj, allow_custom=allow_custom) else: stix_obj = None return stix_obj - def all_versions(self, stix_id, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it @@ -151,11 +151,11 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) return all_data - def query(self, query=None, _composite_filters=None): + def query(self, query=None, _composite_filters=None, allow_custom=False): """search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -194,7 +194,7 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters)["objects"] + all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) @@ -203,7 +203,7 @@ class TAXIICollectionSource(DataSource): all_data = list(apply_common_filters(all_data, query)) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] return stix_objs From e1e368c0d2d4c3916d05bbe213ac0294ee885e01 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 20 Oct 2017 09:13:04 -0400 Subject: [PATCH 10/90] Add created_by() function to Environment --- stix2/environment.py | 14 ++++++++++++++ stix2/test/test_environment.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/stix2/environment.py b/stix2/environment.py index c4816ee..bd50b7f 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -152,3 +152,17 @@ class Environment(object): def parse(self, *args, **kwargs): return _parse(*args, **kwargs) parse.__doc__ = _parse.__doc__ + + def created_by(self, obj): + """Retrieve the Identity refered to by the object's `created_by_ref`. + + Args: + obj: The STIX object whose `created_by_ref` property will be looked + up. + + Returns: + The STIX object's creator. + """ + + creator_id = obj.get('created_by_ref', '') + return self.get(creator_id) diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index 81f2cda..5758732 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -184,3 +184,25 @@ def test_parse_malware(): assert mal.modified == FAKE_TIME assert mal.labels == ['ransomware'] assert mal.name == "Cryptolocker" + + +def test_created_by(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) + env.add(identity) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + creator = env.created_by(ind) + assert creator.id == identity.id + + +def test_created_by_no_datasource(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(factory=factory) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + with pytest.raises(AttributeError) as excinfo: + env.created_by(ind) + assert 'Environment has no data source' in str(excinfo.value) From ef98c3893753cc1fce6c420d623b66ca6d523d27 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Oct 2017 08:04:18 -0400 Subject: [PATCH 11/90] Minor changes --- stix2/__init__.py | 3 ++- stix2/common.py | 5 +++-- stix2/observables.py | 2 +- stix2/sdo.py | 2 +- stix2/utils.py | 6 +++++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index db14aa7..661c247 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -22,7 +22,8 @@ from . import exceptions from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) + LanguageContent, MarkingDefinition, StatementMarking, + TLPMarking) from .core import Bundle, _register_type, parse from .environment import Environment, ObjectFactory from .markings import (add_markings, clear_markings, get_markings, is_marked, diff --git a/stix2/common.py b/stix2/common.py index 0d624ff..fdeef03 100644 --- a/stix2/common.py +++ b/stix2/common.py @@ -41,7 +41,7 @@ class GranularMarking(_STIXBase): _properties = OrderedDict() _properties.update([ ('lang', StringProperty()), - ('marking_ref', ReferenceProperty(type="marking-definition")), # TODO: In 2.0 is required, not in 2.1 + ('marking_ref', ReferenceProperty(type="marking-definition")), ('selectors', ListProperty(SelectorProperty, required=True)), ]) @@ -51,6 +51,7 @@ class GranularMarking(_STIXBase): class LanguageContent(_STIXBase): + _type = 'language-content' _properties = OrderedDict() _properties.update([ @@ -61,7 +62,7 @@ class LanguageContent(_STIXBase): ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('object_ref', ReferenceProperty(required=True)), # TODO: 'object_modified' it MUST be an exact match for the modified time of the STIX Object (SRO or SDO) being referenced. - ('object_modified', TimestampProperty(required=True)), + ('object_modified', TimestampProperty(required=True, precision='millisecond')), # TODO: 'contents' https://docs.google.com/document/d/1ShNq4c3e1CkfANmD9O--mdZ5H0O_GLnjN28a_yrEaco/edit#heading=h.cfz5hcantmvx ('contents', DictionaryProperty(required=True)), ('revoked', BooleanProperty()), diff --git a/stix2/observables.py b/stix2/observables.py index 45c3066..aaec2d7 100644 --- a/stix2/observables.py +++ b/stix2/observables.py @@ -460,7 +460,7 @@ class SocketExt(_Extension): "SOCK_SEQPACKET", ])), ('socket_descriptor', IntegerProperty()), - ('socket_handle', IntegerProperty()) + ('socket_handle', IntegerProperty()), ]) diff --git a/stix2/sdo.py b/stix2/sdo.py index d5d093e..da8fa65 100644 --- a/stix2/sdo.py +++ b/stix2/sdo.py @@ -281,7 +281,7 @@ class Opinion(STIXDomainObject): ('created_by_ref', ReferenceProperty(type="identity")), ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), - ('description', StringProperty), + ('description', StringProperty()), ('authors', ListProperty(StringProperty)), ('object_refs', ListProperty(ReferenceProperty, required=True)), ('opinion', EnumProperty(allowed=[ diff --git a/stix2/utils.py b/stix2/utils.py index 8df4323..32aba72 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -87,7 +87,7 @@ def format_datetime(dttm): ms = zoned.strftime("%f") precision = getattr(dttm, "precision", None) if precision == 'second': - pass # Alredy precise to the second + pass # Already precise to the second elif precision == "millisecond": ts = ts + '.' + ms[:3] elif zoned.microsecond > 0: @@ -191,6 +191,10 @@ def find_property_index(obj, properties, tuple_to_find): tuple_to_find) if val is not None: return val + elif isinstance(item, dict) and tuple_to_find[0] in item: + for num, t in enumerate(item.keys(), start=1): + if t == tuple_to_find[0]: + return num def new_version(data, **kwargs): From 7b6236674ceeceb10b10338966e89d7ef72b43ce Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 23 Oct 2017 08:06:29 -0400 Subject: [PATCH 12/90] Add new tests. --- stix2/confidence/scales.py | 4 +- stix2/test/constants.py | 3 + stix2/test/test_confidence.py | 288 ++++++++++++++++++++++++++++ stix2/test/test_language_content.py | 68 +++++++ stix2/test/test_location.py | 82 ++++++++ stix2/test/test_note.py | 110 +++++++++++ stix2/test/test_opinion.py | 82 ++++++++ 7 files changed, 635 insertions(+), 2 deletions(-) create mode 100644 stix2/test/test_confidence.py create mode 100644 stix2/test/test_language_content.py diff --git a/stix2/confidence/scales.py b/stix2/confidence/scales.py index 6d8882d..4880f33 100644 --- a/stix2/confidence/scales.py +++ b/stix2/confidence/scales.py @@ -228,7 +228,7 @@ def admiralty_credibility_to_value(scale_value): """ if scale_value == "6 - Truth cannot be judged": - pass # TODO: Ask what happens here! + raise ValueError("STIX Confidence value cannot be determined for %s" % scale_value) # TODO: What happens here? elif scale_value == "5 - Improbable": return 10 elif scale_value == "4 - Doubtful": @@ -270,7 +270,7 @@ def value_to_admiralty_credibility(confidence_value): ValueError: If `confidence_value` is out of bounds. """ - # TODO: Ask what happens with "6 - Truth cannot be judged" ! + # TODO: Case "6 - Truth cannot be judged" if 19 >= confidence_value >= 0: return "5 - Improbable" elif 39 >= confidence_value >= 20: diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 839b547..7ced397 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -10,9 +10,12 @@ COURSE_OF_ACTION_ID = "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" IDENTITY_ID = "identity--311b2d2d-f010-5473-83ec-1edf84858f4c" INDICATOR_ID = "indicator--01234567-89ab-cdef-0123-456789abcdef" INTRUSION_SET_ID = "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29" +LOCATION_ID = "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64" MALWARE_ID = "malware--fedcba98-7654-3210-fedc-ba9876543210" MARKING_DEFINITION_ID = "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" +NOTE_ID = "note--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061" OBSERVED_DATA_ID = "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf" +OPINION_ID = "opinion--b01efc25-77b4-4003-b18b-f6e24b5cd9f7" REPORT_ID = "report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3" RELATIONSHIP_ID = "relationship--00000000-1111-2222-3333-444444444444" THREAT_ACTOR_ID = "threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" diff --git a/stix2/test/test_confidence.py b/stix2/test/test_confidence.py new file mode 100644 index 0000000..b4146a5 --- /dev/null +++ b/stix2/test/test_confidence.py @@ -0,0 +1,288 @@ +import pytest + +from stix2.confidence.scales import (admiralty_credibility_to_value, + dni_to_value, none_low_med_high_to_value, + value_to_admiralty_credibility, + value_to_dni, + value_to_none_low_medium_high, + value_to_wep, value_to_zero_ten, + wep_to_value, zero_ten_to_value) + +CONFIDENCE_ERROR_STR = "STIX Confidence value cannot be determined for %s" +RANGE_ERROR_STR = "Range of values out of bounds: %s" + + +def _between(x, val, y): + return x >= val >= y + + +def test_confidence_range_none_low_med_high(): + confidence_range = range(-1, 101) + + for val in confidence_range: + if val < 0 or val > 100: + with pytest.raises(ValueError) as excinfo: + value_to_none_low_medium_high(val) + + assert str(excinfo.value) == RANGE_ERROR_STR % val + continue + + if val == 0: + assert value_to_none_low_medium_high(val) == "None" + elif _between(29, val, 1): + assert value_to_none_low_medium_high(val) == "Low" + elif _between(69, val, 30): + assert value_to_none_low_medium_high(val) == "Med" + elif _between(100, val, 70): + assert value_to_none_low_medium_high(val) == "High" + else: + pytest.fail("Unexpected behavior %s" % val) + + +@pytest.mark.parametrize("scale_value,result", [ + ("None", 0), + ("Low", 15), + ("Med", 50), + ("High", 85) +]) +def test_confidence_scale_valid_none_low_med_high(scale_value, result): + val = none_low_med_high_to_value(scale_value) + assert val == result + + +@pytest.mark.parametrize("scale_value", [ + "Super", + "none", + "" +]) +def test_confidence_scale_invalid_none_low_med_high(scale_value): + with pytest.raises(ValueError) as excinfo: + none_low_med_high_to_value(scale_value) + + assert str(excinfo.value) == CONFIDENCE_ERROR_STR % scale_value + + +def test_confidence_range_zero_ten(): + confidence_range = range(-1, 101) + + for val in confidence_range: + if val < 0 or val > 100: + with pytest.raises(ValueError) as excinfo: + value_to_zero_ten(val) + + assert str(excinfo.value) == RANGE_ERROR_STR % val + continue + + if _between(4, val, 0): + assert value_to_zero_ten(val) == "0" + elif _between(14, val, 5): + assert value_to_zero_ten(val) == "1" + elif _between(24, val, 15): + assert value_to_zero_ten(val) == "2" + elif _between(34, val, 25): + assert value_to_zero_ten(val) == "3" + elif _between(44, val, 35): + assert value_to_zero_ten(val) == "4" + elif _between(54, val, 45): + assert value_to_zero_ten(val) == "5" + elif _between(64, val, 55): + assert value_to_zero_ten(val) == "6" + elif _between(74, val, 65): + assert value_to_zero_ten(val) == "7" + elif _between(84, val, 75): + assert value_to_zero_ten(val) == "8" + elif _between(94, val, 85): + assert value_to_zero_ten(val) == "9" + elif _between(100, val, 95): + assert value_to_zero_ten(val) == "10" + else: + pytest.fail("Unexpected behavior %s" % val) + + +@pytest.mark.parametrize("scale_value,result", [ + ("0", 0), + ("1", 10), + ("2", 20), + ("3", 30), + ("4", 40), + ("5", 50), + ("6", 60), + ("7", 70), + ("8", 80), + ("9", 90), + ("10", 100) +]) +def test_confidence_scale_valid_zero_ten(scale_value, result): + val = zero_ten_to_value(scale_value) + assert val == result + + +@pytest.mark.parametrize("scale_value", [ + "11", + 8, + "" +]) +def test_confidence_scale_invalid_zero_ten(scale_value): + with pytest.raises(ValueError) as excinfo: + zero_ten_to_value(scale_value) + + assert str(excinfo.value) == CONFIDENCE_ERROR_STR % scale_value + + +def test_confidence_range_admiralty_credibility(): + confidence_range = range(-1, 101) + + for val in confidence_range: + if val < 0 or val > 100: + with pytest.raises(ValueError) as excinfo: + value_to_admiralty_credibility(val) + + assert str(excinfo.value) == RANGE_ERROR_STR % val + continue + + if _between(19, val, 0): + assert value_to_admiralty_credibility(val) == "5 - Improbable" + elif _between(39, val, 20): + assert value_to_admiralty_credibility(val) == "4 - Doubtful" + elif _between(59, val, 40): + assert value_to_admiralty_credibility(val) == "3 - Possibly True" + elif _between(79, val, 60): + assert value_to_admiralty_credibility(val) == "2 - Probably True" + elif _between(100, val, 80): + assert value_to_admiralty_credibility(val) == "1 - Confirmed by other sources" + else: + pytest.fail("Unexpected behavior %s" % val) + + +@pytest.mark.parametrize("scale_value,result", [ + ("5 - Improbable", 10), + ("4 - Doubtful", 30), + ("3 - Possibly True", 50), + ("2 - Probably True", 70), + ("1 - Confirmed by other sources", 90) +]) +def test_confidence_scale_valid_admiralty_credibility(scale_value, result): + val = admiralty_credibility_to_value(scale_value) + assert val == result + + +@pytest.mark.parametrize("scale_value", [ + "5 - improbable", + "6 - Truth cannot be judged", + "" +]) +def test_confidence_scale_invalid_admiralty_credibility(scale_value): + with pytest.raises(ValueError) as excinfo: + admiralty_credibility_to_value(scale_value) + + assert str(excinfo.value) == CONFIDENCE_ERROR_STR % scale_value + + +def test_confidence_range_wep(): + confidence_range = range(-1, 101) + + for val in confidence_range: + if val < 0 or val > 100: + with pytest.raises(ValueError) as excinfo: + value_to_wep(val) + + assert str(excinfo.value) == RANGE_ERROR_STR % val + continue + + if val == 0: + assert value_to_wep(val) == "Impossible" + elif _between(19, val, 1): + assert value_to_wep(val) == "Highly Unlikely/Almost Certainly Not" + elif _between(39, val, 20): + assert value_to_wep(val) == "Unlikely/Probably Not" + elif _between(59, val, 40): + assert value_to_wep(val) == "Even Chance" + elif _between(79, val, 60): + assert value_to_wep(val) == "Likely/Probable" + elif _between(99, val, 80): + assert value_to_wep(val) == "Highly likely/Almost Certain" + elif val == 100: + assert value_to_wep(val) == "Certain" + else: + pytest.fail("Unexpected behavior %s" % val) + + +@pytest.mark.parametrize("scale_value,result", [ + ("Impossible", 0), + ("Highly Unlikely/Almost Certainly Not", 10), + ("Unlikely/Probably Not", 30), + ("Even Chance", 50), + ("Likely/Probable", 70), + ("Highly likely/Almost Certain", 90), + ("Certain", 100) +]) +def test_confidence_scale_valid_wep(scale_value, result): + val = wep_to_value(scale_value) + assert val == result + + +@pytest.mark.parametrize("scale_value", [ + "Unlikely / Probably Not", + "Almost certain", + "" +]) +def test_confidence_scale_invalid_wep(scale_value): + with pytest.raises(ValueError) as excinfo: + wep_to_value(scale_value) + + assert str(excinfo.value) == CONFIDENCE_ERROR_STR % scale_value + + +def test_confidence_range_dni(): + confidence_range = range(-1, 101) + + for val in confidence_range: + if val < 0 or val > 100: + with pytest.raises(ValueError) as excinfo: + value_to_dni(val) + + assert str(excinfo.value) == RANGE_ERROR_STR % val + continue + + if _between(9, val, 0): + assert value_to_dni(val) == "Almost No Chance / Remote" + elif _between(19, val, 10): + assert value_to_dni(val) == "Very Unlikely / Highly Improbable" + elif _between(39, val, 20): + assert value_to_dni(val) == "Unlikely / Improbable" + elif _between(59, val, 40): + assert value_to_dni(val) == "Roughly Even Change / Roughly Even Odds" + elif _between(79, val, 60): + assert value_to_dni(val) == "Likely / Probable" + elif _between(89, val, 80): + assert value_to_dni(val) == "Very Likely / Highly Probable" + elif _between(100, val, 90): + assert value_to_dni(val) == "Almost Certain / Nearly Certain" + else: + pytest.fail("Unexpected behavior %s" % val) + + +@pytest.mark.parametrize("scale_value,result", [ + ("Almost No Chance / Remote", 5), + ("Very Unlikely / Highly Improbable", 15), + ("Unlikely / Improbable", 30), + ("Roughly Even Change / Roughly Even Odds", 50), + ("Likely / Probable", 70), + ("Very Likely / Highly Probable", 85), + ("Almost Certain / Nearly Certain", 95) +]) +def test_confidence_scale_valid_dni(scale_value, result): + val = dni_to_value(scale_value) + assert val == result + + +@pytest.mark.parametrize("scale_value", [ + "Almost Certain/Nearly Certain", + "Almost Certain / nearly Certain", + "" +]) +def test_confidence_scale_invalid_none_dni(scale_value): + with pytest.raises(ValueError) as excinfo: + dni_to_value(scale_value) + + assert str(excinfo.value) == CONFIDENCE_ERROR_STR % scale_value diff --git a/stix2/test/test_language_content.py b/stix2/test/test_language_content.py new file mode 100644 index 0000000..2343d69 --- /dev/null +++ b/stix2/test/test_language_content.py @@ -0,0 +1,68 @@ +import datetime as dt + +import pytest +import pytz + +import stix2 + +CAMPAIGN_ID = "campaign--12a111f0-b824-4baf-a224-83b80237a094" + +LANGUAGE_CONTENT_ID = "language-content--b86bd89f-98bb-4fa9-8cb2-9ad421da981d" + +TEST_CAMPAIGN = """{ + "type": "campaign", + "id": "campaign--12a111f0-b824-4baf-a224-83b80237a094", + "lang": "en", + "created": "2017-02-08T21:31:22.007Z", + "modified": "2017-02-08T21:31:22.007Z", + "name": "Bank Attack", + "description": "More information about bank attack" +}""" + +TEST_LANGUAGE_CONTENT = """{ + "type": "language-content", + "id": "language-content--b86bd89f-98bb-4fa9-8cb2-9ad421da981d", + "created": "2017-02-08T21:31:22.007Z", + "modified": "2017-02-08T21:31:22.007Z", + "object_ref": "campaign--12a111f0-b824-4baf-a224-83b80237a094", + "object_modified": "2017-02-08T21:31:22.007Z", + "contents": { + "de": { + "name": "Bank Angriff 1", + "description": "Weitere Informationen über Banküberfall" + }, + "fr": { + "name": "Attaque Bank 1", + "description": "Plus d'informations sur la crise bancaire" + } + } +}""" + + +@pytest.mark.xfail(reason="Dictionary keys are too short") +def test_language_content_campaign(): + now = dt.datetime(2017, 2, 8, 21, 31, 22, microsecond=7000, tzinfo=pytz.utc) + + lc = stix2.LanguageContent( + type='language-content', + id=LANGUAGE_CONTENT_ID, + created=now, + modified=now, + object_ref=CAMPAIGN_ID, + object_modified=now, + contents={ + "de": { + "name": "Bank Angriff 1", + "description": "Weitere Informationen über Banküberfall" + }, + "fr": { + "name": "Attaque Bank 1", + "description": "Plus d'informations sur la crise bancaire" + } + } + ) + + camp = stix2.parse(TEST_CAMPAIGN) + + assert str(lc) in TEST_LANGUAGE_CONTENT + assert lc.modified == camp.modified diff --git a/stix2/test/test_location.py b/stix2/test/test_location.py index e69de29..8ce44d7 100644 --- a/stix2/test/test_location.py +++ b/stix2/test/test_location.py @@ -0,0 +1,82 @@ +import datetime as dt +import re + +import pytest +import pytz + +import stix2 + +from .constants import LOCATION_ID + + +EXPECTED_LOCATION_1 = """{ + "type": "location", + "id": "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "latitude": 48.8566, + "longitude": 2.3522 +}""" + +EXPECTED_LOCATION_1_REPR = "Location(" + " ".join(""" + type='location', + id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64', + created='2016-04-06T20:03:00.000Z', + modified='2016-04-06T20:03:00.000Z', + latitude=48.8566, + longitude=2.3522""".split()) + ")" + +EXPECTED_LOCATION_2 = """{ + "type": "location", + "id": "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "region": "north-america" +} +""" + +EXPECTED_LOCATION_2_REPR = "Location(" + " ".join(""" + type='location', + id='location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64', + created='2016-04-06T20:03:00.000Z', + modified='2016-04-06T20:03:00.000Z', + region='north-america'""".split()) + ")" + + +def test_location_with_some_required_properties(): + now = dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) + + loc = stix2.Location( + type="location", + id=LOCATION_ID, + created=now, + modified=now, + latitude=48.8566, + longitude=2.3522 + ) + + assert str(loc) == EXPECTED_LOCATION_1 + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(loc)) + assert rep == EXPECTED_LOCATION_1_REPR + + +@pytest.mark.parametrize("data", [ + EXPECTED_LOCATION_2, + { + "type": "location", + "id": "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64", + "created": "2016-04-06T20:03:00.000Z", + "modified": "2016-04-06T20:03:00.000Z", + "region": "north-america" + } +]) +def test_parse_location(data): + location = stix2.parse(data) + + assert location.type == 'location' + assert location.id == LOCATION_ID + assert location.created == dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) + assert location.modified == dt.datetime(2016, 4, 6, 20, 3, 0, tzinfo=pytz.utc) + assert location.region == 'north-america' + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(location)) + assert rep == EXPECTED_LOCATION_2_REPR diff --git a/stix2/test/test_note.py b/stix2/test/test_note.py index e69de29..df117d8 100644 --- a/stix2/test/test_note.py +++ b/stix2/test/test_note.py @@ -0,0 +1,110 @@ +import datetime as dt +import re + +import pytest +import pytz + +import stix2 + +from .constants import CAMPAIGN_ID, NOTE_ID + +DESCRIPTION = ('This note indicates the various steps taken by the threat' + ' analyst team to investigate this specific campaign. Step' + ' 1) Do a scan 2) Review scanned results for identified ' + 'hosts not known by external intel... etc') + +EXPECTED_NOTE = """{ + "type": "note", + "id": "note--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", + "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "summary": "Tracking Team Note#1", + "description": "%s", + "authors": [ + "John Doe" + ], + "object_refs": [ + "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" + ], + "external_references": [ + { + "source_name": "job-tracker", + "external_id": "job-id-1234" + } + ] +}""" % DESCRIPTION + +EXPECTED_OPINION_REPR = "Note(" + " ".join((""" + type='note', + id='note--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061', + created='2016-05-12T08:17:27.000Z', + modified='2016-05-12T08:17:27.000Z', + summary='Tracking Team Note#1', + description='%s', + authors=['John Doe'], + object_refs=['campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f'], + external_references=[ExternalReference(source_name='job-tracker', external_id='job-id-1234')] +""" % DESCRIPTION).split()) + ")" + + +def test_note_with_required_properties(): + now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + + note = stix2.Note( + type='note', + id=NOTE_ID, + created=now, + modified=now, + summary='Tracking Team Note#1', + object_refs=[CAMPAIGN_ID], + authors=['John Doe'], + description=DESCRIPTION, + external_references=[ + { + 'source_name': 'job-tracker', + 'external_id': 'job-id-1234' + } + ] + ) + + assert str(note) == EXPECTED_NOTE + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(note)) + assert rep == EXPECTED_OPINION_REPR + + +@pytest.mark.parametrize("data", [ + EXPECTED_NOTE, + { + "type": "note", + "id": "note--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061", + "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "summary": "Tracking Team Note#1", + "description": DESCRIPTION, + "authors": [ + "John Doe" + ], + "object_refs": [ + "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f" + ], + "external_references": [ + { + "source_name": "job-tracker", + "external_id": "job-id-1234" + } + ] + } +]) +def test_parse_note(data): + note = stix2.parse(data) + + assert note.type == 'note' + assert note.id == NOTE_ID + assert note.created == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + assert note.modified == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + assert note.object_refs[0] == CAMPAIGN_ID + assert note.authors[0] == 'John Doe' + assert note.summary == 'Tracking Team Note#1' + assert note.description == DESCRIPTION + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(note)) + assert rep == EXPECTED_OPINION_REPR diff --git a/stix2/test/test_opinion.py b/stix2/test/test_opinion.py index e69de29..7de415a 100644 --- a/stix2/test/test_opinion.py +++ b/stix2/test/test_opinion.py @@ -0,0 +1,82 @@ +import datetime as dt +import re + +import pytest +import pytz + +import stix2 + +from .constants import OPINION_ID + +DESCRIPTION = ('This doesn\'t seem like it is feasible. We\'ve seen how ' + 'PandaCat has attacked Spanish infrastructure over the ' + 'last 3 years, so this change in targeting seems too great' + ' to be viable. The methods used are more commonly ' + 'associated with the FlameDragonCrew.') + +EXPECTED_OPINION = """{ + "type": "opinion", + "id": "opinion--b01efc25-77b4-4003-b18b-f6e24b5cd9f7", + "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "description": "%s", + "object_refs": [ + "relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471" + ], + "opinion": "strongly-disagree" +}""" % DESCRIPTION + +EXPECTED_OPINION_REPR = "Opinion(" + " ".join((""" + type='opinion', + id='opinion--b01efc25-77b4-4003-b18b-f6e24b5cd9f7', + created='2016-05-12T08:17:27.000Z', + modified='2016-05-12T08:17:27.000Z', + description="%s", + object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'], + opinion='strongly-disagree'""" % DESCRIPTION).split()) + ")" + + +def test_opinion_with_required_properties(): + now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + + opi = stix2.Opinion( + type='opinion', + id=OPINION_ID, + created=now, + modified=now, + object_refs=['relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'], + opinion='strongly-disagree', + description=DESCRIPTION + ) + + assert str(opi) == EXPECTED_OPINION + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(opi)) + assert rep == EXPECTED_OPINION_REPR + + +@pytest.mark.parametrize("data", [ + EXPECTED_OPINION, + { + "type": "opinion", + "id": "opinion--b01efc25-77b4-4003-b18b-f6e24b5cd9f7", + "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "description": DESCRIPTION, + "object_refs": [ + "relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471" + ], + "opinion": "strongly-disagree" + } +]) +def test_parse_opinion(data): + opinion = stix2.parse(data) + + assert opinion.type == 'opinion' + assert opinion.id == OPINION_ID + assert opinion.created == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + assert opinion.modified == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc) + assert opinion.opinion == 'strongly-disagree' + assert opinion.object_refs[0] == 'relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471' + assert opinion.description == DESCRIPTION + rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(opinion)) + assert rep == EXPECTED_OPINION_REPR From ca7bb77d87b1d75e64d6b40f8485e8cfc2f5ce93 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 23 Oct 2017 14:38:58 -0400 Subject: [PATCH 13/90] Use a better link to the documentation --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c78923b..2753440 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ including data markings, versioning, and for resolving STIX IDs across multiple data sources. For more information, see `the -documentation `__ on +documentation `__ on ReadTheDocs. Installation From 5c28074364ee268242e3578d4644310eb436304e Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 24 Oct 2017 09:15:09 -0400 Subject: [PATCH 14/90] Add `allow_custom` to datastore docstrings --- stix2/sources/__init__.py | 28 +++++++++++--- stix2/sources/filesystem.py | 11 ++++-- stix2/sources/memory.py | 73 +++++++++++++++++++++---------------- stix2/sources/taxii.py | 25 +++++++------ 4 files changed, 84 insertions(+), 53 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 9c5510e..47c7573 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -51,6 +51,8 @@ class DataStore(object): Args: stix_id (str): the id of the STIX object to retrieve. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the single most recent version of the STIX @@ -66,6 +68,8 @@ class DataStore(object): Args: stix_id (str): the id of the STIX object to retrieve. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -82,6 +86,8 @@ class DataStore(object): Args: query (list): a list of filters (which collectively are the query) to conduct search on. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -96,6 +102,8 @@ class DataStore(object): Args: stix_objs (list): a list of STIX objects + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ return self.sink.add(stix_objs, allow_custom=allow_custom) @@ -120,6 +128,8 @@ class DataSink(object): Args: stix_objs (list): a list of STIX objects (where each object is a STIX object) + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ raise NotImplementedError() @@ -148,9 +158,10 @@ class DataSource(object): stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the STIX object @@ -169,9 +180,10 @@ class DataSource(object): stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -187,9 +199,10 @@ class DataSource(object): Args: query (list): a list of filters (which collectively are the query) to conduct search on - _composite_filters (set): a set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -238,10 +251,11 @@ class CompositeDataSource(DataSource): Args: stix_id (str): the id of the STIX object to retrieve. - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to another parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the STIX object to be returned. @@ -283,10 +297,11 @@ class CompositeDataSource(DataSource): Args: stix_id (str): id of the STIX objects to retrieve - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: all_data (list): list of STIX objects that have the specified id @@ -323,10 +338,11 @@ class CompositeDataSource(DataSource): Args: query (list): list of filters to search on - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: all_data (list): list of STIX objects to be returned diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 1dcf85b..1a8366b 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -67,6 +67,8 @@ class FileSystemSink(DataSink): stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or list of), dict (or list of), or a STIX 2.0 json encoded string. + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Note: ``stix_data`` can be a Bundle object, but each object in it will be @@ -140,9 +142,10 @@ class FileSystemSource(DataSource): Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -169,9 +172,10 @@ class FileSystemSource(DataSource): Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): of STIX objects that has the supplied STIX ID. @@ -190,9 +194,10 @@ class FileSystemSource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that matches the supplied diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 1b738ab..0f10cf9 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -32,8 +32,10 @@ def _add(store, stix_data=None, allow_custom=False): Args: stix_data (list OR dict OR STIX object): STIX objects to be added - """ + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + """ if isinstance(stix_data, _STIXBase): # adding a python STIX object store._data[stix_data["id"]] = stix_data @@ -77,16 +79,15 @@ class MemoryStore(DataStore): Args: stix_data (list OR dict OR STIX object): STIX content to be added + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects - source (MemorySource): MemorySource - sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False): super(MemoryStore, self).__init__() self._data = {} @@ -98,9 +99,29 @@ class MemoryStore(DataStore): self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) def save_to_file(self, file_path, allow_custom=False): + """Write SITX objects from in-memory dictionary to JSON file, as a STIX + Bundle. + + Args: + file_path (str): file path to write STIX data to + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + + """ return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) def load_from_file(self, file_path, allow_custom=False): + """Load STIX data from JSON file. + + File format is expected to be a single JSON + STIX object or JSON STIX bundle. + + Args: + file_path (str): file path to load STIX data from + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + + """ return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) @@ -113,17 +134,18 @@ class MemorySink(DataSink): Args: stix_data (dict OR list): valid STIX 2.0 content in bundle or a list. - _store (bool): if the MemorySink is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSource. Not user supplied + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects. If apart of a MemoryStore, dict is shared between with a MemorySource - """ + """ def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySink, self).__init__() self._data = {} @@ -134,25 +156,16 @@ class MemorySink(DataSink): _add(self, stix_data, allow_custom=allow_custom) def add(self, stix_data, allow_custom=False): - """add STIX objects to in-memory dictionary maintained by - the MemorySink (MemoryStore) - - see "_add()" for args documentation - """ _add(self, stix_data, allow_custom=allow_custom) + add.__doc__ = _add.__doc__ def save_to_file(self, file_path, allow_custom=False): - """write SITX objects in in-memory dictionary to json file, as a STIX Bundle - - Args: - file_path (str): file path to write STIX data to - - """ file_path = os.path.abspath(file_path) if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: f.write(str(Bundle(self._data.values(), allow_custom=allow_custom))) + save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ class MemorySource(DataSource): @@ -165,17 +178,18 @@ class MemorySource(DataSource): Args: stix_data (dict OR list OR STIX object): valid STIX 2.0 content in bundle or list. - _store (bool): if the MemorySource is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSink. Not user supplied + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects. If apart of a MemoryStore, dict is shared between with a MemorySink - """ + """ def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySource, self).__init__() self._data = {} @@ -190,9 +204,10 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (dict OR STIX object): STIX object that has the supplied @@ -227,9 +242,10 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that has the supplied ID. As the @@ -249,15 +265,16 @@ class MemorySource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that matches the supplied query. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory as they are supplied (either as python dictionary or STIX object), it - is returned in the same form as it as added + is returned in the same form as it as added. """ if query is None: @@ -281,15 +298,7 @@ class MemorySource(DataSource): return all_data def load_from_file(self, file_path, allow_custom=False): - """Load STIX data from json file. - - File format is expected to be a single json - STIX object or json STIX bundle - - Args: - file_path (str): file path to load STIX data from - - """ file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) _add(self, stix_data, allow_custom=allow_custom) + load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 619af0c..90d632e 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -21,7 +21,7 @@ class TAXIICollectionStore(DataStore): around a paired TAXIICollectionSink and TAXIICollectionSource. Args: - collection (taxii2.Collection): TAXII Collection instance + collection (taxii2.Collection): TAXII Collection instance """ def __init__(self, collection): super(TAXIICollectionStore, self).__init__() @@ -42,15 +42,16 @@ class TAXIICollectionSink(DataSink): self.collection = collection def add(self, stix_data, allow_custom=False): - """add/push STIX content to TAXII Collection endpoint + """Add/push STIX content to TAXII Collection endpoint Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0 json encoded string, or list of any of the following + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ - if isinstance(stix_data, _STIXBase): # adding python STIX object bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) @@ -94,21 +95,21 @@ class TAXIICollectionSource(DataSource): self.collection = collection def get(self, stix_id, _composite_filters=None, allow_custom=False): - """retrieve STIX object from local/remote STIX Collection + """Retrieve STIX object from local/remote STIX Collection endpoint. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (STIX object): STIX object that has the supplied STIX ID. The STIX object is received from TAXII has dict, parsed into a python STIX object and then returned - """ # combine all query filters query = set() @@ -132,14 +133,15 @@ class TAXIICollectionSource(DataSource): return stix_obj def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): - """retrieve STIX object from local/remote TAXII Collection + """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (see query() as all_versions() is just a wrapper) @@ -156,7 +158,7 @@ class TAXIICollectionSource(DataSource): return all_data def query(self, query=None, _composite_filters=None, allow_custom=False): - """search and retreive STIX objects based on the complete query + """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters attached to MemorySource, and any filters passed from a @@ -164,9 +166,10 @@ class TAXIICollectionSource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that matches the supplied @@ -174,7 +177,6 @@ class TAXIICollectionSource(DataSource): parsed into python STIX objects and then returned. """ - if query is None: query = set() else: @@ -225,7 +227,6 @@ class TAXIICollectionSource(DataSource): for 'requests.get()'. """ - params = {} for filter_ in query: From 47b11453fa363b74d0e2213065e8c87d16a7ac0d Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 24 Oct 2017 12:43:30 -0400 Subject: [PATCH 15/90] Increase test coverage for memory datastore --- stix2/sources/__init__.py | 4 ++-- stix2/sources/memory.py | 4 ++-- stix2/test/test_memory.py | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 47c7573..a142e88 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -77,7 +77,7 @@ class DataStore(object): """ return self.source.all_versions(stix_id, allow_custom=allow_custom) - def query(self, query, allow_custom=False): + def query(self, query=None, allow_custom=False): """Retrieve STIX objects matching a set of filters. Implement: Specific data source API calls, processing, @@ -191,7 +191,7 @@ class DataSource(object): """ raise NotImplementedError() - def query(self, query, _composite_filters=None, allow_custom=False): + def query(self, query=None, _composite_filters=None, allow_custom=False): """ Implement:Implement the specific data source API calls, processing, functionality required for retrieving query from the data source diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0f10cf9..0846f9b 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -65,7 +65,7 @@ def _add(store, stix_data=None, allow_custom=False): _add(store, stix_obj, allow_custom=allow_custom) else: - raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") + raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") class MemoryStore(DataStore): @@ -283,7 +283,7 @@ class MemorySource(DataSource): if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index e8ba56b..0603bf7 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -138,11 +138,34 @@ def test_memory_store_all_versions(mem_store): def test_memory_store_query(mem_store): query = [Filter('type', '=', 'malware')] - resp = mem_store.query(query) assert len(resp) == 0 +def test_memory_store_query_single_filter(mem_store): + query = Filter('id', '=', 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f') + resp = mem_store.query(query) + assert len(resp) == 1 + + +def test_memory_store_query_empty_query(mem_store): + resp = mem_store.query() + # sort since returned in random order + resp = sorted(resp, key=lambda k: k['id']) + assert len(resp) == 2 + assert resp[0]['id'] == 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f' + assert resp[0]['modified'] == '2017-01-27T13:49:53.935Z' + assert resp[1]['id'] == 'indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f' + assert resp[1]['modified'] == '2017-01-27T13:49:53.936Z' + + +def test_memory_store_query_multiple_filters(mem_store): + mem_store.source.filters.add(Filter('type', '=', 'indicator')) + query = Filter('id', '=', 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f') + resp = mem_store.query(query) + assert len(resp) == 1 + + def test_memory_store_add_stix_object_str(mem_store): # add stix object string camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" @@ -194,6 +217,16 @@ def test_memory_store_add_stix_bundle_str(mem_store): assert camp_alias in camp_r["aliases"] +def test_memory_store_add_invalid_object(mem_store): + ind = ('indicator', IND1) # tuple isn't valid + with pytest.raises(TypeError) as excinfo: + mem_store.add(ind) + assert 'stix_data must be' in str(excinfo.value) + assert 'a STIX object' in str(excinfo.value) + assert 'JSON formatted STIX' in str(excinfo.value) + assert 'JSON formatted STIX bundle' in str(excinfo.value) + + def test_memory_store_object_with_custom_property(mem_store): camp = Campaign(name="Scipio Africanus", objective="Defeat the Carthaginians", From d4db4f0ab8945f64337f8c870f10074c4b8cb215 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Tue, 24 Oct 2017 12:53:53 -0400 Subject: [PATCH 16/90] Define source code encoding --- stix2/confidence/scales.py | 2 ++ stix2/test/test_language_content.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/stix2/confidence/scales.py b/stix2/confidence/scales.py index 4880f33..a630be9 100644 --- a/stix2/confidence/scales.py +++ b/stix2/confidence/scales.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """Functions to perform conversions between the different Confidence scales. As specified in STIX™ Version 2.1. Part 1: STIX Core Concepts - Appendix B""" diff --git a/stix2/test/test_language_content.py b/stix2/test/test_language_content.py index 2343d69..2a75acc 100644 --- a/stix2/test/test_language_content.py +++ b/stix2/test/test_language_content.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import datetime as dt import pytest From 5dffe74867362566e60bfa973010220f704e009d Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 24 Oct 2017 14:20:42 -0400 Subject: [PATCH 17/90] Clean up creator_of (renamed from created_by) --- stix2/environment.py | 13 +++++++++---- stix2/sources/__init__.py | 2 ++ stix2/sources/memory.py | 10 +++++++--- stix2/test/test_environment.py | 16 +++++++++++++--- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/stix2/environment.py b/stix2/environment.py index bd50b7f..4919335 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -153,7 +153,7 @@ class Environment(object): return _parse(*args, **kwargs) parse.__doc__ = _parse.__doc__ - def created_by(self, obj): + def creator_of(self, obj): """Retrieve the Identity refered to by the object's `created_by_ref`. Args: @@ -161,8 +161,13 @@ class Environment(object): up. Returns: - The STIX object's creator. - """ + The STIX object's creator, or + None, if the object contains no `created_by_ref` property or the + object's creator cannot be found. + """ creator_id = obj.get('created_by_ref', '') - return self.get(creator_id) + if creator_id: + return self.get(creator_id) + else: + return None diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 9d46ba9..49cb3f3 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -266,6 +266,8 @@ class CompositeDataSource(DataSource): # remove duplicate versions if len(all_data) > 0: all_data = deduplicate(all_data) + else: + return None # reduce to most recent version stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0] diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0d5901e..967b886 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -215,10 +215,14 @@ class MemorySource(DataSource): all_data = self.query(query=query, _composite_filters=_composite_filters) - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + if all_data: + print(all_data) + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - return stix_obj + return stix_obj + else: + return None def all_versions(self, stix_id, _composite_filters=None): """retrieve STIX objects from in-memory dict via STIX ID, all versions of it diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index 5758732..c669a33 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -193,8 +193,8 @@ def test_created_by(): env.add(identity) ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) - creator = env.created_by(ind) - assert creator.id == identity.id + creator = env.creator_of(ind) + assert creator is identity def test_created_by_no_datasource(): @@ -204,5 +204,15 @@ def test_created_by_no_datasource(): ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) with pytest.raises(AttributeError) as excinfo: - env.created_by(ind) + env.creator_of(ind) assert 'Environment has no data source' in str(excinfo.value) + + +def test_created_by_not_found(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + creator = env.creator_of(ind) + assert creator is None From 70929905652a7b90d669f5724e204cef02fd32f9 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 25 Oct 2017 14:36:20 -0400 Subject: [PATCH 18/90] tested code with jupyter notebook and local TAXII server --- docs/guide/taxii.ipynb | 2357 +--------------------------------------- stix2/sources/taxii.py | 25 +- 2 files changed, 69 insertions(+), 2313 deletions(-) diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 2890659..ef2cb73 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -79,21 +79,27 @@ "output_type": "stream", "text": [ "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fb2c0e55-52a0-423c-b544-8b09622cafc1\",\n", - " \"created\": \"2017-10-02T19:26:30.000Z\",\n", - " \"modified\": \"2017-10-02T19:26:30.000Z\",\n", + " \"type\": \"malware\",\n", + " \"id\": \"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\",\n", + " \"created\": \"2017-01-27T13:49:53.997Z\",\n", + " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", + " \"name\": \"Poison Ivy\",\n", + " \"description\": \"Poison Ivy\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T19:26:30Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"remote-access-trojan\"\n", + " ]\n", + "}\n", + "-------\n", + "{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\",\n", + " \"created\": \"2014-05-08T09:00:00.000Z\",\n", + " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", + " \"name\": \"File hash for Poison Ivy variant\",\n", + " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", + " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", + " \"labels\": [\n", + " \"file-hash-watchlist\"\n", " ]\n", "}\n" ] @@ -104,2293 +110,41 @@ "from taxii2client import Collection\n", "\n", "# establish TAXII2 Collection instance\n", - "collection = Collection(\"https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/\")\n", + "collection = Collection(\"http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/\", user=\"admin\", password=\"password\")\n", "# supply the TAXII2 collection to TAXIICollection\n", "tc_source = TAXIICollectionSource(collection)\n", "\n", - "#retrieve STIX object by id\n", - "stix_obj = tc_source.get(\"indicator--0f63229c-07a2-46dd-939d-312c7bf6d114\")\n", + "#retrieve STIX objects by id\n", + "stix_obj = tc_source.get(\"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\")\n", + "stix_obj_versions = tc_source.all_versions(\"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\")\n", "\n", "#for visual purposes\n", - "print(stix_obj)\n" + "print(stix_obj)\n", + "print(\"-------\")\n", + "for so in stix_obj_versions:\n", + " print(so)\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "indicators: 126\n", + "indicators: 1\n", "{\n", " \"type\": \"indicator\",\n", - " \"id\": \"indicator--569b8969-bfce-4ab4-9a45-06ce78799a35\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", + " \"id\": \"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\",\n", + " \"created\": \"2014-05-08T09:00:00.000Z\",\n", + " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", + " \"name\": \"File hash for Poison Ivy variant\",\n", + " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", + " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '207.158.1.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9c418633-9970-424e-8030-2c3dfa3105da\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9d7cdfc1-94c3-49b5-b124-ebdce709fd99\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.67.22' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--37390a22-5d82-4ebc-9b90-7368a5efc8f7\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--30731d72-64b0-4851-bd97-c3d164d2fd2b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.24.188.100' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a4eb3524-992c-4b50-9729-99be3048625e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.232.93.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c00fb599-7e7b-4033-a6c2-d279212578a0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.45' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e7273b13-847c-4a69-8faf-08fc24af5ef0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '89.16.176.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b8d21867-c812-4ff9-866b-182a801b88ce\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.239.18.172' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4c39b1a0-17f0-4cf1-9e48-250f0dd1f75c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8eeff049-f7da-45d9-89bb-713063baed2c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.92.8.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e3981158-1934-4236-8454-4dcfc27ac248\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.87.120.111' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--206c2a0c-149f-426f-a734-c0c534aa396b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.93.243.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--58d7aa16-8baf-4026-b3d7-328267ed4bab\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.165.191.52' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e6fd4a21-8290-40e5-9b1c-701f6f11e260\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.204.1.132' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cca5ce5f-4c0e-4031-9997-063eb3badead\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '209.177.146.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--43a7784e-f11c-4739-91a8-dc87d05ddbb6\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '145.220.21.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5716d954-e5b1-4bec-ba43-80b1053dee61\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '50.7.55.82' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9135d4ab-a807-495b-8fff-b8433342501f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.165.47.254' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e070c86b-40e5-49ea-8d83-56bcae10b303\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f4125383-930c-42ae-b57f-2c36f898d0b5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.71.169.36' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fa063c6a-1a9f-4a58-9470-ed80a23cc943\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '204.152.221.218' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--41b3ba86-dd1b-4f3d-a156-5dc27f31fb40\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '78.40.125.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a9fcaba5-cd50-447d-8540-2dfe4e3c6c88\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.68' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--30b68eff-3c38-4c74-9783-1114a7759066\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.197.175.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f10fa7c0-7a10-434e-908f-59a7e25e18c0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.14.236.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--183f8cd7-2e6f-4073-bbe8-d5dc6b570fac\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--dd95ff3a-3ef1-409e-827b-087eb9cc3b2c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a97dc9cb-2b9f-4c1d-92cc-2fc15100e3ed\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '91.205.185.104' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5552096e-b2b8-4057-bf5e-ccf300b8276e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.163.220.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--0cc30ea9-eeaf-4f39-ab8d-3d2664c2b75e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.202.189.170' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--7582ed02-c78d-451d-b0a5-065ae511f3ae\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '86.65.39.15' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--37fde688-ca75-4c1e-b5e1-1acb5bbfb23c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.167.98' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e967d3a0-0cfe-482c-b53a-390c0bb564f4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '199.16.156.6' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fda4f25d-8252-4593-bd8b-0a90764a561f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '217.168.95.245' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--109b3de1-2353-42dc-8316-e2f7c0b5c67d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.16.195' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1efa50e4-ed2c-4fb5-ae9b-cb347bd4ad24\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.18.128.86' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c7b60a1a-4c93-451f-b7c1-993c0dc14391\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.109.129.220' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--469381d9-c24e-4cf4-b25b-18a48975ef14\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.99.193.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c5694bbd-3a11-4c16-ae73-eeed55acf9cc\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '70.84.101.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a9b4301e-0327-4edc-b407-b7915bb0e7bc\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.62' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f5ac23ca-8ab4-4597-837b-3d5e48d325cb\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.61' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1a2a539b-d3f3-410b-a32c-4d1a5599364e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '66.186.59.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--585e6f7b-7bad-45b0-a36b-9f3b3bff72c6\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '93.152.160.101' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--0a7dd603-d826-428f-b5f7-c82ff8bb60f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.46' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cb2cebd2-c11f-43b1-a9a1-3c4b9893f38a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.150.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--6a6c81df-7cb9-48b3-a4ea-db6924e47b5d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.107.206.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--45177dce-6cfe-44b5-ac41-cbc1bee80527\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--6f58bdf5-1f26-4a17-8ba3-14c023e73a0f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '72.51.18.254' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d5731bef-623c-4793-994c-a6f3840bc2cf\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.190.67.98' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4e8ac337-2e00-4d71-8526-bbfdb105e77f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b681b1fc-7cce-473e-81e9-f5f3657cf85b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.237.188.200' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--08453fee-f3b8-449a-95a8-abc0d79710c3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.155.130.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--79e2a4f6-ee8d-4466-8e82-ecb928e87c0d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.71.169.36' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--2d3326c5-c112-4670-b6bd-6de667f4280b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.47' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4adc0666-89d1-4c67-a3c8-3b02fc351442\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.161.196.11' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--dc1e9fec-6d1e-46a1-902c-dc170424a23f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.47.220.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--2d7480b1-ded5-4466-a1dd-470110eacdba\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '152.3.102.53' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f06d6873-1538-4951-a069-d6af0dd0f8ed\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '84.208.29.17' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4eaf258d-28d8-48d8-98f8-0d8442ba83fa\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.96.64.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d7e4bba4-485d-4c1f-95c0-55e7d8a015f8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.179.58.83' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5c060dc8-a8cd-4067-985d-52d85ab3f256\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '128.237.157.136' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d397fccb-3dbb-47c3-84ae-aa09f4223eca\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.110.95.1' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d37d0928-c86b-474a-85ef-46e942fff510\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5e6dd813-58bd-454e-9be7-246f3db01999\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--adb3c6bc-9694-471e-bf1f-0d0a02d70876\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '137.226.34.46' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--3ce88e57-edfb-45fa-81be-ed95d4564316\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '67.198.195.194' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fbce496c-e9a6-4246-ad12-73b8f5a12a2a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '149.9.1.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--efce84a3-0d17-4ae8-88be-86c86aa80bbd\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.109.122.77' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--20789570-8c07-42c4-8a45-b3ab170cf6ee\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '209.126.116.149' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5c1b2889-6fec-4276-83e0-173938934ba9\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.250.116.136' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bdad2fdb-71bd-49c3-8bf2-50d396fa55d5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '163.172.17.231' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--07fd3e36-5500-4652-935f-23a2955b19f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.114.116.5' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9e70a102-3440-4ad0-ab1d-653144632668\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '66.186.59.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f566b659-ca36-42a9-8ebf-9476e6b651ab\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.204.1.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--40c0d87c-287a-4692-8227-b4976d14a5f0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '212.27.60.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--da56b536-6ac7-44d5-a036-0db986926016\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.236.208.178' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--95f9c0f4-351b-43c9-81da-c5fdcfe4fa6d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '94.125.182.252' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5007db19-0906-4aec-b18b-e0819b3f13de\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.83.20.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9e0667cd-9a83-4e19-b16f-78c3ed33bfc5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.18.228.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f170e9a9-abb8-4919-9902-7a5214e95cde\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.150.28' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a30f883d-956d-4fdd-b926-db81d1893d81\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '178.79.132.147' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8fc0e9c0-4d4d-4c4f-86a7-2f6c07cd69a4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.109.122.67' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--586dc7e8-a08e-4ec2-8365-e2ee897d9ca3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.47.220.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--16c9900c-ce48-4306-b8fa-a2de726be847\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.83.20.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--13f73e28-acf7-45b8-a5e9-6c37af914ef2\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '174.143.119.91' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a09c4e42-8843-4c84-a75f-684bf90c5207\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '74.208.174.239' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--76646197-18a2-4513-8465-ccf72734a2e1\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.48' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1169c1db-fd5b-4dcf-b4cb-9c0101ef0ea2\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '212.117.163.190' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cdeb6ddb-5151-49ea-a488-23d806063eff\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.155.130.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--05d1ab76-d0a1-4a58-8137-98f5fdbc777c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '90.147.160.69' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--876d7d09-248a-45ad-bcce-d92c73ad5aa3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '89.16.176.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ae1f860d-dc4f-4953-9e74-d4d7c389fdef\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '85.188.1.26' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--91bb4edc-f29f-41ba-87d9-d6a81ac8fdba\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.239.18.172' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f006d048-f24f-46fa-837b-8f7fa41b43ca\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '8.7.233.233' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--436dcbec-48e2-4dc2-90f0-0876a876a38a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.54' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ff18364d-99f6-4d3d-b267-8401518af42c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.68.45.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8b26f167-b0ad-469b-b221-12896e2a0966\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.33' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--171268fb-f6a7-4085-adf5-2055a461cb93\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.161.254.20' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b56c7a58-71cb-47c2-b615-f4e8a89a0732\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '141.213.238.252' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bf09ce9a-3bb9-47c8-a686-ea1d8e1adbe8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.92.8.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--42490e45-7350-4f48-884b-5d1610794a32\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '72.14.191.81' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c28e91bf-a9a1-4bac-b3f3-cda89c7d28b8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ebe624b5-fb73-420a-a110-c1dc82baa6e4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.61.21.115' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ef65505f-4898-4968-82b4-f980e9705d21\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.18.128.86' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b33c35ce-20f6-4fba-912c-dbf7756113f9\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '161.53.178.240' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b3785934-f4f0-4ce7-b20c-e4384886ec45\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '204.11.244.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--10bbe70c-7bd3-443a-8f2c-1e56cd7a8a54\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.93.242.10' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bcb54665-3461-43e2-8dbf-6b92c2413f67\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.67.23' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a407b16b-cf5b-4f3a-a153-ba4dac5ce0e0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '205.188.234.121' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e7e50d3a-802d-41c8-b667-a27d29871098\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.96.64.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--017dfb8c-84b9-402f-8401-428477af7be4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '80.88.108.18' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--84664128-cc14-480b-8d90-735727fd4b9f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '154.35.200.44' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f0aa750f-82cb-47f9-9c74-ace584fdadcb\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.68.221.222' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9461c426-6404-4b7a-8552-c29dc60c9123\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.197.175.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ba59cc70-03e4-47f4-871e-d40b727267f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '78.129.164.123' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1b48b107-92e2-487f-9eae-3496eb64e125\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.167.99' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e9aea5e2-9ef6-40b6-8f12-dff6ccd8eff4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '85.25.43.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cdbd95b1-17fb-4b2f-89b6-8c0f865b9e4d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.219.128.49' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--afe4738d-bd3c-47de-9cc5-97e248291571\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.40.6.37' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5eecb66e-f8fa-4ab9-85e4-599db7790edf\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '173.252.110.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--40b6b332-9a5a-42a7-8b25-6e3eb6d371d4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.229.70.20' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a16905d7-4452-4e9f-88a3-fc9338ea5116\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.99.64.210' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5fcfa412-514f-43b5-b873-ed8c9b70bbb0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.200.113' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8259bca6-7c9c-4967-b048-a6f13f333f90\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '68.168.184.57' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--96763c7c-4f52-436a-919a-8b09c841f6bd\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.237.34.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"file-hash-watchlist\"\n", " ]\n", "}\n" ] @@ -2457,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -2465,21 +219,14 @@ "output_type": "stream", "text": [ "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d8e1cd37-4a6c-4088-aded-ed79c4ea2caa\",\n", - " \"created\": \"2017-10-02T20:24:03.000Z\",\n", - " \"modified\": \"2017-10-02T20:24:03.000Z\",\n", + " \"type\": \"malware\",\n", + " \"id\": \"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\",\n", + " \"created\": \"2017-01-27T13:49:53.997Z\",\n", + " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", + " \"name\": \"Poison Ivy\",\n", + " \"description\": \"Poison Ivy\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T20:24:03Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"remote-access-trojan\"\n", " ]\n", "}\n" ] @@ -2494,7 +241,7 @@ "\n", "# retrieve STIX object by id from TAXII Collection through\n", "# TAXIICollectionStore\n", - "stix_obj2 = tc_source.get(\"indicator--6850d393-36b6-4a67-ad45-f9e4d512c799\")\n", + "stix_obj2 = tc_source.get(\"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\")\n", "\n", "print(stix_obj2)" ] @@ -2520,21 +267,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "cti-python-stix2", "language": "python", - "name": "python3" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 7ecedca..3d8dcaa 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,8 +1,9 @@ """ -Python STIX 2.0 TAXII Source/Sink - -TODO: - Test everything +Python STIX 2.x +Classes: + TAXIICollectionStore + TAXIICollectionSink + TAXIICollectionSource """ @@ -121,11 +122,13 @@ class TAXIICollectionSource(DataSource): # as directly retrieveing a STIX object by ID stix_objs = self.collection.get_object(stix_id)["objects"] - stix_obj = list(apply_common_filters(stix_objs, query)) + stix_obj = [apply_common_filters(stix_objs, query)] if len(stix_obj): - stix_obj = stix_obj[0] - stix_obj = parse(stix_obj) + stix_obj = parse(stix_obj[0]) + if stix_obj.id != stix_id: + # check - was added to handle erroneous TAXII servers + stix_obj = None else: stix_obj = None @@ -153,7 +156,13 @@ class TAXIICollectionSource(DataSource): all_data = self.query(query=query, _composite_filters=_composite_filters) - return all_data + # parse STIX objects from TAXII returned json + all_data = [parse(stix_obj) for stix_obj in all_data] + + # check - was added to handle erroneous TAXII servers + all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] + + return all_data_clean def query(self, query=None, _composite_filters=None): """search and retreive STIX objects based on the complete query From 313c6f56ffb198495a66aeaebbc398dabd0a38c2 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 25 Oct 2017 14:38:25 -0400 Subject: [PATCH 19/90] guide output --- docs/guide/taxii.ipynb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index ef2cb73..d44b12e 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -127,14 +127,13 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "indicators: 1\n", "{\n", " \"type\": \"indicator\",\n", " \"id\": \"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\",\n", @@ -160,7 +159,6 @@ "indicators = tc_source.query([f1])\n", "\n", "#for visual purposes\n", - "print(\"indicators: {0}\").format(str(len(indicators)))\n", "for indicator in indicators:\n", " print(indicator)" ] @@ -211,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 19, "metadata": {}, "outputs": [ { From 082973f780c89c81245a08ee93d9cfdab1a6d570 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 25 Oct 2017 15:18:32 -0400 Subject: [PATCH 20/90] whitespace - somehow pre-commit got turned off?? --- stix2/sources/taxii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 3d8dcaa..5dcd9f2 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,5 +1,5 @@ """ -Python STIX 2.x +Python STIX 2.x Classes: TAXIICollectionStore TAXIICollectionSink From 8c56adda2157a6ab553c274ef196d12b38fcc161 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 26 Oct 2017 11:39:45 -0400 Subject: [PATCH 21/90] Update package structure --- stix2/__init__.py | 27 +++----------- stix2/core.py | 68 +++++++++++++++++++++------------- stix2/test/test_bundle.py | 5 ++- stix2/test/test_properties.py | 3 +- stix2/v20/__init__.py | 43 +++++++++++++++++++++ stix2/{ => v20}/common.py | 12 +++--- stix2/{ => v20}/observables.py | 18 ++++----- stix2/{ => v20}/sdo.py | 14 +++---- stix2/{ => v20}/sro.py | 12 +++--- 9 files changed, 122 insertions(+), 80 deletions(-) create mode 100644 stix2/v20/__init__.py rename stix2/{ => v20}/common.py (94%) rename stix2/{ => v20}/observables.py (98%) rename stix2/{ => v20}/sdo.py (97%) rename stix2/{ => v20}/sro.py (91%) diff --git a/stix2/__init__.py b/stix2/__init__.py index 55911a4..6fdce6a 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -19,27 +19,11 @@ # flake8: noqa -from . import exceptions -from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, - ExternalReference, GranularMarking, KillChainPhase, - MarkingDefinition, StatementMarking, TLPMarking) -from .core import Bundle, _register_type, parse +from . import exceptions, v20 +from .core import Bundle, _collect_stix2_obj_maps, _register_type, parse from .environment import Environment, ObjectFactory from .markings import (add_markings, clear_markings, get_markings, is_marked, remove_markings, set_markings) -from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, - AutonomousSystem, CustomExtension, CustomObservable, - Directory, DomainName, EmailAddress, EmailMessage, - EmailMIMEComponent, File, HTTPRequestExt, ICMPExt, - IPv4Address, IPv6Address, MACAddress, Mutex, - NetworkTraffic, NTFSExt, PDFExt, Process, - RasterImageExt, SocketExt, Software, TCPExt, - UNIXAccountExt, UserAccount, WindowsPEBinaryExt, - WindowsPEOptionalHeaderType, WindowsPESection, - WindowsProcessExt, WindowsRegistryKey, - WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, - parse_observable) from .patterns import (AndBooleanExpression, AndObservationExpression, BasicObjectPathComponent, EqualityComparisonExpression, FloatConstant, FollowedByObservationExpression, @@ -58,9 +42,6 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) -from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, - Identity, Indicator, IntrusionSet, Malware, ObservedData, - Report, ThreatActor, Tool, Vulnerability) from .sources import CompositeDataSource from .sources.filesystem import (FileSystemSink, FileSystemSource, FileSystemStore) @@ -68,6 +49,8 @@ from .sources.filters import Filter from .sources.memory import MemorySink, MemorySource, MemoryStore from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource, TAXIICollectionStore) -from .sro import Relationship, Sighting from .utils import get_dict, new_version, revoke +from .v20 import * # This import should always be the latest STIX 2.X version from .version import __version__ + +_collect_stix2_obj_maps() diff --git a/stix2/core.py b/stix2/core.py index 8ee11f5..22a8891 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -1,15 +1,12 @@ """STIX 2.0 Objects that are neither SDOs nor SROs.""" from collections import OrderedDict +import importlib +import pkgutil from . import exceptions from .base import _STIXBase -from .common import MarkingDefinition from .properties import IDProperty, ListProperty, Property, TypeProperty -from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, - IntrusionSet, Malware, ObservedData, Report, ThreatActor, - Tool, Vulnerability) -from .sro import Relationship, Sighting from .utils import get_dict @@ -62,37 +59,30 @@ class Bundle(_STIXBase): super(Bundle, self).__init__(**kwargs) -OBJ_MAP = { - 'attack-pattern': AttackPattern, - 'bundle': Bundle, - 'campaign': Campaign, - 'course-of-action': CourseOfAction, - 'identity': Identity, - 'indicator': Indicator, - 'intrusion-set': IntrusionSet, - 'malware': Malware, - 'marking-definition': MarkingDefinition, - 'observed-data': ObservedData, - 'report': Report, - 'relationship': Relationship, - 'threat-actor': ThreatActor, - 'tool': Tool, - 'sighting': Sighting, - 'vulnerability': Vulnerability, -} +STIX2_OBJ_MAPS = {} -def parse(data, allow_custom=False): +def parse(data, allow_custom=False, version=None): """Deserialize a string or file-like object into a STIX object. Args: data (str, dict, file-like object): The STIX 2 content to be parsed. - allow_custom (bool): Whether to allow custom properties or not. Default: False. + allow_custom (bool): Whether to allow custom properties or not. + Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: An instantiated Python STIX object. """ + if not version: + # Use latest version + OBJ_MAP = STIX2_OBJ_MAPS[sorted(STIX2_OBJ_MAPS.keys())[-1]] + else: + v = 'v' + version.replace('.', '') + OBJ_MAP = STIX2_OBJ_MAPS[v] + obj = get_dict(data) if 'type' not in obj: @@ -105,8 +95,34 @@ def parse(data, allow_custom=False): return obj_class(allow_custom=allow_custom, **obj) -def _register_type(new_type): +def _register_type(new_type, version=None): """Register a custom STIX Object type. + Args: + new_type (class): A class to register in the Object map. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ + if not version: + # Use latest version + OBJ_MAP = STIX2_OBJ_MAPS[sorted(STIX2_OBJ_MAPS.keys())[-1]] + else: + v = 'v' + version.replace('.', '') + OBJ_MAP = STIX2_OBJ_MAPS[v] + OBJ_MAP[new_type._type] = new_type + + +def _collect_stix2_obj_maps(): + """Navigate the package once and retrieve all OBJ_MAP dicts for each v2X + package.""" + if not STIX2_OBJ_MAPS: + top_level_module = importlib.import_module('stix2') + path = top_level_module.__path__ + prefix = str(top_level_module.__name__) + '.' + + for module_loader, name, is_pkg in pkgutil.walk_packages(path=path, + prefix=prefix): + if name.startswith('stix2.v2') and is_pkg: + mod = importlib.import_module(name, top_level_module) + STIX2_OBJ_MAPS[name.split('.')[-1]] = mod.OBJ_MAP diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index c7c95a8..8597f0f 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -132,8 +132,9 @@ def test_create_bundle_invalid(indicator, malware, relationship): assert excinfo.value.reason == 'This property may not contain a Bundle object' -def test_parse_bundle(): - bundle = stix2.parse(EXPECTED_BUNDLE) +@pytest.mark.parametrize("version", ["2.0"]) +def test_parse_bundle(version): + bundle = stix2.parse(EXPECTED_BUNDLE, version=version) assert bundle.type == "bundle" assert bundle.id.startswith("bundle--") diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 7d03b9e..6bd1888 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -1,8 +1,7 @@ import pytest -from stix2 import TCPExt +from stix2 import EmailMIMEComponent, ExtensionsProperty, TCPExt from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError -from stix2.observables import EmailMIMEComponent, ExtensionsProperty from stix2.properties import (BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, HashesProperty, diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py new file mode 100644 index 0000000..95f0b7e --- /dev/null +++ b/stix2/v20/__init__.py @@ -0,0 +1,43 @@ + +# flake8: noqa + +from ..core import Bundle +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) +from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, + AutonomousSystem, CustomExtension, CustomObservable, + Directory, DomainName, EmailAddress, EmailMessage, + EmailMIMEComponent, ExtensionsProperty, File, + HTTPRequestExt, ICMPExt, IPv4Address, IPv6Address, + MACAddress, Mutex, NetworkTraffic, NTFSExt, PDFExt, + Process, RasterImageExt, SocketExt, Software, TCPExt, + UNIXAccountExt, UserAccount, WindowsPEBinaryExt, + WindowsPEOptionalHeaderType, WindowsPESection, + WindowsProcessExt, WindowsRegistryKey, + WindowsRegistryValueType, WindowsServiceExt, + X509Certificate, X509V3ExtenstionsType, + parse_observable) +from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, + IntrusionSet, Malware, ObservedData, Report, ThreatActor, + Tool, Vulnerability) +from .sro import Relationship, Sighting + +OBJ_MAP = { + 'attack-pattern': AttackPattern, + 'bundle': Bundle, + 'campaign': Campaign, + 'course-of-action': CourseOfAction, + 'identity': Identity, + 'indicator': Indicator, + 'intrusion-set': IntrusionSet, + 'malware': Malware, + 'marking-definition': MarkingDefinition, + 'observed-data': ObservedData, + 'report': Report, + 'relationship': Relationship, + 'threat-actor': ThreatActor, + 'tool': Tool, + 'sighting': Sighting, + 'vulnerability': Vulnerability, +} diff --git a/stix2/common.py b/stix2/v20/common.py similarity index 94% rename from stix2/common.py rename to stix2/v20/common.py index 449cd54..2d15529 100644 --- a/stix2/common.py +++ b/stix2/v20/common.py @@ -2,12 +2,12 @@ from collections import OrderedDict -from .base import _STIXBase -from .markings import _MarkingsMixin -from .properties import (HashesProperty, IDProperty, ListProperty, Property, - ReferenceProperty, SelectorProperty, StringProperty, - TimestampProperty, TypeProperty) -from .utils import NOW, get_dict +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (HashesProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW, get_dict class ExternalReference(_STIXBase): diff --git a/stix2/observables.py b/stix2/v20/observables.py similarity index 98% rename from stix2/observables.py rename to stix2/v20/observables.py index aaec2d7..a874df9 100644 --- a/stix2/observables.py +++ b/stix2/v20/observables.py @@ -7,15 +7,15 @@ Observable and do not have a ``_type`` attribute. from collections import OrderedDict -from .base import _Extension, _Observable, _STIXBase -from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, - ParseError) -from .properties import (BinaryProperty, BooleanProperty, DictionaryProperty, - EmbeddedObjectProperty, EnumProperty, FloatProperty, - HashesProperty, HexProperty, IntegerProperty, - ListProperty, ObjectReferenceProperty, Property, - StringProperty, TimestampProperty, TypeProperty) -from .utils import get_dict +from ..base import _Extension, _Observable, _STIXBase +from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError, + ParseError) +from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty, + EmbeddedObjectProperty, EnumProperty, FloatProperty, + HashesProperty, HexProperty, IntegerProperty, + ListProperty, ObjectReferenceProperty, Property, + StringProperty, TimestampProperty, TypeProperty) +from ..utils import get_dict class ObservableProperty(Property): diff --git a/stix2/sdo.py b/stix2/v20/sdo.py similarity index 97% rename from stix2/sdo.py rename to stix2/v20/sdo.py index 8dad686..1af0777 100644 --- a/stix2/sdo.py +++ b/stix2/v20/sdo.py @@ -4,14 +4,14 @@ from collections import OrderedDict import stix2 -from .base import _STIXBase +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, PatternProperty, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty) +from ..utils import NOW from .common import ExternalReference, GranularMarking, KillChainPhase -from .markings import _MarkingsMixin from .observables import ObservableProperty -from .properties import (BooleanProperty, IDProperty, IntegerProperty, - ListProperty, PatternProperty, ReferenceProperty, - StringProperty, TimestampProperty, TypeProperty) -from .utils import NOW class STIXDomainObject(_STIXBase, _MarkingsMixin): @@ -358,7 +358,7 @@ def CustomObject(type='x-custom-type', properties=None): return raise e - stix2._register_type(_Custom) + stix2._register_type(_Custom, version="2.0") return _Custom return custom_builder diff --git a/stix2/sro.py b/stix2/v20/sro.py similarity index 91% rename from stix2/sro.py rename to stix2/v20/sro.py index 60f99f5..7f05f5e 100644 --- a/stix2/sro.py +++ b/stix2/v20/sro.py @@ -2,13 +2,13 @@ from collections import OrderedDict -from .base import _STIXBase +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW from .common import ExternalReference, GranularMarking -from .markings import _MarkingsMixin -from .properties import (BooleanProperty, IDProperty, IntegerProperty, - ListProperty, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty) -from .utils import NOW class STIXRelationshipObject(_STIXBase, _MarkingsMixin): From 9aefa611532842a178a18934edc8927ab4eed50c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 26 Oct 2017 12:39:27 -0400 Subject: [PATCH 22/90] Small fix for Python 3.3, 3.4 --- stix2/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix2/core.py b/stix2/core.py index 22a8891..5341b1d 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -124,5 +124,5 @@ def _collect_stix2_obj_maps(): for module_loader, name, is_pkg in pkgutil.walk_packages(path=path, prefix=prefix): if name.startswith('stix2.v2') and is_pkg: - mod = importlib.import_module(name, top_level_module) + mod = importlib.import_module(name, str(top_level_module.__name__)) STIX2_OBJ_MAPS[name.split('.')[-1]] = mod.OBJ_MAP From 4e3752912d81881ee9c81d1c4da53812ea925208 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 26 Oct 2017 15:35:19 -0400 Subject: [PATCH 23/90] Remove stray `print` statement --- stix2/sources/memory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 967b886..9dc7062 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -216,7 +216,6 @@ class MemorySource(DataSource): all_data = self.query(query=query, _composite_filters=_composite_filters) if all_data: - print(all_data) # reduce to most recent version stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] From f10308443988e8195b94132a2c8c985f2a25567a Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Oct 2017 12:23:57 -0400 Subject: [PATCH 24/90] Add `bundlify` parameter to FileSystemStore/Sink This brings back the option (disabled by default) to wrap objects in a bundle when saving them to the filesystem. --- stix2/sources/filesystem.py | 41 +++++++++++++++++++++-------------- stix2/test/test_bundle.py | 1 - stix2/test/test_filesystem.py | 18 +++++++++++++++ 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 1a8366b..9fd658d 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -25,17 +25,18 @@ class FileSystemStore(DataStore): Args: stix_dir (str): path to directory of STIX objects + bundlify (bool): Whether to wrap objects in bundles when saving them. + Default: False. Attributes: source (FileSystemSource): FuleSystemSource - sink (FileSystemSink): FileSystemSink """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, bundlify=False): super(FileSystemStore, self).__init__() self.source = FileSystemSource(stix_dir=stix_dir) - self.sink = FileSystemSink(stix_dir=stix_dir) + self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) class FileSystemSink(DataSink): @@ -46,12 +47,15 @@ class FileSystemSink(DataSink): components of a FileSystemStore. Args: - stix_dir (str): path to directory of STIX objects + stix_dir (str): path to directory of STIX objects. + bundlify (bool): Whether to wrap objects in bundles when saving them. + Default: False. """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, bundlify=False): super(FileSystemSink, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.bundlify = bundlify if not os.path.exists(self._stix_dir): raise ValueError("directory path for STIX data does not exist") @@ -60,6 +64,20 @@ class FileSystemSink(DataSink): def stix_dir(self): return self._stix_dir + def _check_path_and_write(self, stix_obj): + """Write the given STIX object to a file in the STIX file directory. + """ + path = os.path.join(self._stix_dir, stix_obj["type"], stix_obj["id"] + ".json") + + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + if self.bundlify: + stix_obj = Bundle(stix_obj) + + with open(path, "w") as f: + f.write(str(stix_obj)) + def add(self, stix_data=None, allow_custom=False): """Add STIX objects to file directory. @@ -76,18 +94,9 @@ class FileSystemSink(DataSink): the Bundle contained, but not the Bundle itself. """ - def _check_path_and_write(stix_dir, stix_obj): - path = os.path.join(stix_dir, stix_obj["type"], stix_obj["id"] + ".json") - - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - - with open(path, "w") as f: - f.write(str(stix_obj)) - if isinstance(stix_data, (STIXDomainObject, STIXRelationshipObject, MarkingDefinition)): # adding python STIX object - _check_path_and_write(self._stix_dir, stix_data) + self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): stix_data = parse(stix_data, allow_custom) @@ -97,7 +106,7 @@ class FileSystemSink(DataSink): self.add(stix_obj) else: # adding json-formatted STIX - _check_path_and_write(self._stix_dir, stix_data) + self._check_path_and_write(stix_data) elif isinstance(stix_data, Bundle): # recursively add individual STIX objects diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 12b2149..24bbd43 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -164,5 +164,4 @@ def test_stix_object_property(): prop = stix2.core.STIXObjectProperty() identity = stix2.Identity(name="test", identity_class="individual") - assert prop.clean(identity) == identity assert prop.clean(identity) is identity diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 8b9aa22..1e79d05 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -255,6 +255,24 @@ def test_filesystem_store_add(fs_store): os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) +def test_filesystem_store_add_as_bundle(): + fs_store = FileSystemStore(FS_PATH, bundlify=True) + + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: + assert '"type": "bundle"' in bundle_file.read() + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + def test_filesystem_add_bundle_object(fs_store): bundle = Bundle() fs_store.add(bundle) From 71ee73c08a629effad6210e0f354dd1e02286c66 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 27 Oct 2017 12:37:08 -0400 Subject: [PATCH 25/90] Add MANIFEST to exclude test from final package --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c9ec75b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE +include CHANGELOG +recursive-exclude stix2\test * From 42317ddf31e21b6dd752b787d0467d90e433370c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 27 Oct 2017 12:38:03 -0400 Subject: [PATCH 26/90] Update filesystem.py to allow_custom and version --- stix2/__init__.py | 1 - stix2/sources/filesystem.py | 35 +++++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index 6fdce6a..99c70fc 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -19,7 +19,6 @@ # flake8: noqa -from . import exceptions, v20 from .core import Bundle, _collect_stix2_obj_maps, _register_type, parse from .environment import Environment, ObjectFactory from .markings import (add_markings, clear_markings, get_markings, is_marked, diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 103b882..012e828 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -61,13 +61,17 @@ class FileSystemSink(DataSink): def stix_dir(self): return self._stix_dir - def add(self, stix_data=None): + def add(self, stix_data=None, allow_custom=False, version=None): """add STIX objects to file directory Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object(or list of), dict (or list of), or a STIX 2.0 json encoded string + allow_custom (bool): Whether to allow custom properties or not. + Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. TODO: Bundlify STIX content or no? When dumping to disk. """ @@ -96,7 +100,7 @@ class FileSystemSink(DataSink): elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data) + stix_data = parse(stix_data, allow_custom, version) if stix_data["type"] == "bundle": for stix_obj in stix_data["objects"]: self.add(stix_obj) @@ -136,14 +140,18 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): """retrieve STIX object from file directory via STIX ID Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied. + allow_custom (bool): Whether to allow custom properties or not. + Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -157,7 +165,7 @@ class FileSystemSource(DataSource): if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - stix_obj = parse(stix_obj) + stix_obj = parse(stix_obj, allow_custom, version) else: stix_obj = None @@ -182,7 +190,7 @@ class FileSystemSource(DataSource): """ return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None): + def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): """search and retrieve STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -190,10 +198,13 @@ class FileSystemSource(DataSource): CompositeDataSource (i.e. _composite_filters) Args: - query (list): list of filters to search on - - composite_filters (set): set of filters passed from the - CompositeDataSource, not user supplied + query (list): list of filters to search on. + _composite_filters (set): set of filters passed from the + CompositeDataSource, not user supplied. + allow_custom (bool): Whether to allow custom properties or not. + Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): list of STIX objects that matches the supplied @@ -287,7 +298,7 @@ class FileSystemSource(DataSource): all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] return stix_objs From 942a95a4e26c9651ce304550e4439d17a20a9ce2 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 27 Oct 2017 12:38:25 -0400 Subject: [PATCH 27/90] Update documentation --- README.rst | 9 +++ docs/index.rst | 1 + docs/ts_support.rst | 131 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 docs/ts_support.rst diff --git a/README.rst b/README.rst index 2753440..c2ec908 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,15 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``: For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__. +STIX 2.X Technical Specification Support +---------------------------------------- + +The `stix2` Python library is built to support multiple versions of the STIX +Technical Specification. With every major release of stix2 the ``import stix2`` +statement will automatically load the SDO/SROs equivalent to the most recent +supported 2.X Technical Specification. Please see the library documentation +for more details. + Governance ---------- diff --git a/docs/index.rst b/docs/index.rst index 62d07ff..f865467 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Welcome to stix2's documentation! datastore_api roadmap contributing + ts_support Indices and tables diff --git a/docs/ts_support.rst b/docs/ts_support.rst new file mode 100644 index 0000000..09ad009 --- /dev/null +++ b/docs/ts_support.rst @@ -0,0 +1,131 @@ +How imports will work +--------------------- + +Imports can be used in different ways depending on the use case and support +levels. + +People who want to (in general) support the latest version of STIX 2.X without +making changes, implicitly using the latest version + +.. code:: python + + import stix2 + ... + stix2.Indicator(...) + +or + +.. code:: python + + from stix2 import Indicator + ... + Indicator(...) + +People who want to use an explicit version + +.. code:: python + + import stix2.v20 + ... + stix2.v20.Indicator(...) + +or + +.. code:: python + + from stix2.v20 import Indicator + ... + Indicator(...) + +or even, + +.. code:: python + + import stix2.v20 as stix2 + ... + stix2.Indicator(...) + +The last option makes it easy to update to a new version in one place per file, +once you've made the deliberate action to do this. + +People who want to use multiple versions in a single file: + +.. code:: python + + import stix2 + ... + stix2.v20.Indicator(...) + ... + stix2.v21.Indicator(...) + +or + +.. code:: python + + from stix2 import v20, v21 + ... + v20.Indicator(...) + ... + v21.Indicator(...) + +or (less preferred): + +.. code:: python + + from stix2.v20 import Indicator as Indicator_v20 + from stix2.v21 import Indicator as Indicator_v21 + ... + Indicator_v20(...) + ... + Indicator_v21(...) + +How parsing will work +--------------------- + +If the ``version`` positional argument is not provided. The data will be parsed +using the latest version of STIX 2.X supported by the `stix2` library. + +You can lock your `parse()` method to a specific STIX version by + +.. code:: python + + from stix2 import parse + + indicator = parse("""{ + "type": "indicator", + "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394", + "created": "2017-09-26T23:33:39.829Z", + "modified": "2017-09-26T23:33:39.829Z", + "labels": [ + "malicious-activity" + ], + "name": "File hash for malware variant", + "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "valid_from": "2017-09-26T23:33:39.829952Z" + }""", version="2.0") + print(indicator) + +Keep in mind that if a 2.1 or higher object is parsed, the operation will fail. + +How will custom work +-------------------- + +CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be +registered explicitly by STIX version. This is a design decision since properties +or requirements may chance as the STIX Technical Specification advances. + +You can perform this by, + +.. code:: python + + # Make my custom observable available in STIX 2.0 + @stix2.v20.observables.CustomObservable('x-new-object-type', + (("prop", stix2.properties.BooleanProperty()))) + class NewObject2(object): + pass + + # Make my custom observable available in STIX 2.1 + @stix2.v21.observables.CustomObservable('x-new-object-type', + (("prop", stix2.properties.BooleanProperty()))) + class NewObject2(object): + pass From 612f2fbab8e607abd44efb5610184aaadf6ef23f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Oct 2017 15:50:43 -0400 Subject: [PATCH 28/90] Increase code coverage for filesystem datastore Found a couple bugs in the process and fixed them, too. --- stix2/sources/filesystem.py | 16 +++++------ stix2/sources/memory.py | 4 +-- stix2/test/test_filesystem.py | 54 +++++++++++++++++++++++++++++++++++ stix2/utils.py | 9 ++++-- 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 9fd658d..34dbcf0 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -119,9 +119,9 @@ class FileSystemSink(DataSink): self.add(stix_obj) else: - raise ValueError("stix_data must be a STIX object (or list of), " - "json formatted STIX (or list of), " - "or a json formatted STIX bundle") + raise TypeError("stix_data must be a STIX object (or list of), " + "JSON formatted STIX (or list of), " + "or a JSON formatted STIX bundle") class FileSystemSource(DataSource): @@ -198,8 +198,8 @@ class FileSystemSource(DataSource): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters - attached to MemorySource, and any filters passed from a - CompositeDataSource (i.e. _composite_filters) + attached to this FileSystemSource, and any filters passed from a + CompositeDataSource (i.e. _composite_filters). Args: query (list): list of filters to search on @@ -222,7 +222,7 @@ class FileSystemSource(DataSource): if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters @@ -267,8 +267,8 @@ class FileSystemSource(DataSource): # so query will look in all STIX directories that are not # the specified type. Compile correct dir paths for dir in os.listdir(self._stix_dir): - if os.path.abspath(dir) not in declude_paths: - include_paths.append(os.path.abspath(dir)) + if os.path.abspath(os.path.join(self._stix_dir, dir)) not in declude_paths: + include_paths.append(os.path.abspath(os.path.join(self._stix_dir, dir))) # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0846f9b..ec44dba 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -260,8 +260,8 @@ class MemorySource(DataSource): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters - attached to MemorySource, and any filters passed from a - CompositeDataSource (i.e. _composite_filters) + attached to this MemorySource, and any filters passed from a + CompositeDataSource (i.e. _composite_filters). Args: query (list): list of filters to search on diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 1e79d05..7aaa3f5 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -40,6 +40,18 @@ def fs_sink(): shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) +def test_filesystem_source_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSource('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + +def test_filesystem_sink_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSink('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + def test_filesytem_source_get_object(fs_source): # get object mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") @@ -47,6 +59,11 @@ def test_filesytem_source_get_object(fs_source): assert mal.name == "Rover" +def test_filesytem_source_get_nonexistent_object(fs_source): + ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert ind is None + + def test_filesytem_source_all_versions(fs_source): # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") @@ -240,6 +257,33 @@ def test_filesystem_store_query(fs_store): assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] +def test_filesystem_store_query_single_filter(fs_store): + query = Filter("labels", "in", "tool") + tools = fs_store.query(query) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_empty_query(fs_store): + results = fs_store.query() # returns all + assert len(results) == 26 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] + assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] + + +def test_filesystem_store_query_multiple_filters(fs_store): + fs_store.source.filters.add(Filter("labels", "in", "tool")) + tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966")) + assert len(tools) == 1 + assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966" + + +def test_filesystem_store_query_dont_include_type_folder(fs_store): + results = fs_store.query(Filter("type", "!=", "tool")) + assert len(results) == 24 + + def test_filesystem_store_add(fs_store): # add() camp1 = Campaign(name="Great Heathen Army", @@ -278,6 +322,16 @@ def test_filesystem_add_bundle_object(fs_store): fs_store.add(bundle) +def test_filesystem_store_add_invalid_object(fs_store): + ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a') # tuple isn't valid + with pytest.raises(TypeError) as excinfo: + fs_store.add(ind) + assert 'stix_data must be' in str(excinfo.value) + assert 'a STIX object' in str(excinfo.value) + assert 'JSON formatted STIX' in str(excinfo.value) + assert 'JSON formatted STIX bundle' in str(excinfo.value) + + def test_filesystem_object_with_custom_property(fs_store): camp = Campaign(name="Scipio Africanus", objective="Defeat the Carthaginians", diff --git a/stix2/utils.py b/stix2/utils.py index 8df4323..4623f28 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -34,7 +34,7 @@ class STIXdatetime(dt.datetime): def deduplicate(stix_obj_list): - """Deduplicate a list of STIX objects to a unique set + """Deduplicate a list of STIX objects to a unique set. Reduces a set of STIX objects to unique set by looking at 'id' and 'modified' fields - as a unique object version @@ -44,7 +44,6 @@ def deduplicate(stix_obj_list): of deduplicate(),that if the "stix_obj_list" argument has multiple STIX objects of the same version, the last object version found in the list will be the one that is returned. - () Args: stix_obj_list (list): list of STIX objects (dicts) @@ -56,7 +55,11 @@ def deduplicate(stix_obj_list): unique_objs = {} for obj in stix_obj_list: - unique_objs[(obj['id'], obj['modified'])] = obj + try: + unique_objs[(obj['id'], obj['modified'])] = obj + except KeyError: + # Handle objects with no `modified` property, e.g. marking-definition + unique_objs[(obj['id'], obj['created'])] = obj return list(unique_objs.values()) From 4ffad6d34f8312be837e3b241f7331495f613000 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 27 Oct 2017 16:01:51 -0400 Subject: [PATCH 29/90] Remove stix2-validator dependency Fixes #94. --- .isort.cfg | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index f55fec7..d535851 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -9,7 +9,6 @@ known_third_party = simplejson six, stix2patterns, - stix2validator, taxii2client, known_first_party = stix2 force_sort_within_sections = 1 diff --git a/setup.py b/setup.py index e359147..75b5a43 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( 'simplejson', 'six', 'stix2-patterns', - 'stix2-validator', 'taxii2-client', ], ) From b9d25a837527abddd5a2083044e949d3a53456b9 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Sat, 28 Oct 2017 00:32:28 -0400 Subject: [PATCH 30/90] Add notebook docs, fix package distribution --- docs/guide/support.ipynb | 235 +++++++++++++++++++++++++++++++++++++++ docs/ts_support.rst | 13 ++- setup.py | 2 +- 3 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 docs/guide/support.ipynb diff --git a/docs/guide/support.ipynb b/docs/guide/support.ipynb new file mode 100644 index 0000000..3c56f1d --- /dev/null +++ b/docs/guide/support.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How imports will work\n", + "\n", + "Imports can be used in different ways depending on the use case and support levels.\n", + "\n", + "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "People who want to use an explicit version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20\n", + "\n", + "stix2.v20.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or even," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20 as stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The last option makes it easy to update to a new version in one place per file, once you've made the deliberate action to do this.\n", + "\n", + "People who want to use multiple versions in a single file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.v20.Indicator()\n", + "\n", + "stix2.v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import v20, v21\n", + "\n", + "v20.Indicator()\n", + "v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or (less preferred):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator as Indicator_v20\n", + "from stix2.v21 import Indicator as Indicator_v21\n", + "\n", + "Indicator_v20()\n", + "Indicator_v21()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How parsing will work\n", + "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", + "\n", + "You can lock your `parse()` method to a specific STIX version by" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import parse\n", + "\n", + "indicator = parse(\"\"\"{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n", + " \"created\": \"2017-09-26T23:33:39.829Z\",\n", + " \"modified\": \"2017-09-26T23:33:39.829Z\",\n", + " \"labels\": [\n", + " \"malicious-activity\"\n", + " ],\n", + " \"name\": \"File hash for malware variant\",\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2017-09-26T23:33:39.829952Z\"\n", + "}\"\"\", version=\"2.0\")\n", + "print(indicator)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Keep in mind that if a 2.1 or higher object is parsed, the operation will fail." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How will custom work\n", + "\n", + "CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "\n", + "You can perform this by," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "# Make my custom observable available in STIX 2.0\n", + "@stix2.v20.observables.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass\n", + "\n", + "\n", + "# Make my custom observable available in STIX 2.1\n", + "@stix2.v21.observables.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/ts_support.rst b/docs/ts_support.rst index 09ad009..a62e59e 100644 --- a/docs/ts_support.rst +++ b/docs/ts_support.rst @@ -112,20 +112,23 @@ How will custom work CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be registered explicitly by STIX version. This is a design decision since properties -or requirements may chance as the STIX Technical Specification advances. +or requirements may change as the STIX Technical Specification advances. You can perform this by, .. code:: python + import stix2 + # Make my custom observable available in STIX 2.0 @stix2.v20.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) - class NewObject2(object): - pass + class NewObject2(object): + pass + # Make my custom observable available in STIX 2.1 @stix2.v21.observables.CustomObservable('x-new-object-type', (("prop", stix2.properties.BooleanProperty()))) - class NewObject2(object): - pass + class NewObject2(object): + pass diff --git a/setup.py b/setup.py index e359147..c9b1bc7 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( 'Programming Language :: Python :: 3.6', ], keywords="stix stix2 json cti cyber threat intelligence", - packages=find_packages(), + packages=find_packages(exclude=['*.test']), install_requires=[ 'python-dateutil', 'pytz', From a514c5a3df51464c60c30d6a92b06d250242d095 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Sun, 29 Oct 2017 00:09:13 -0400 Subject: [PATCH 31/90] Add test for _register_type() --- stix2/test/test_custom.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 48529b9..3bd79bc 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -483,3 +483,13 @@ def test_parse_observable_with_unregistered_custom_extension(): with pytest.raises(ValueError) as excinfo: stix2.parse_observable(input_str) assert "Can't parse Unknown extension type" in str(excinfo.value) + + +def test_register_custom_object(): + # Not the way to register custom object. + class CustomObject2(object): + _type = 'awesome-object' + + stix2._register_type(CustomObject2) + # Note that we will always check against newest OBJ_MAP. + assert (CustomObject2._type, CustomObject2) in stix2.OBJ_MAP.items() From 924c72e98ad5af8c9884b5df6dcb4c87d10fa3d2 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 30 Oct 2017 17:25:32 -0400 Subject: [PATCH 32/90] Change filters to allow filtering all properties (Not just common properties) --- stix2/sources/filters.py | 234 +++++++------------------------- stix2/test/test_data_sources.py | 48 ++++++- 2 files changed, 99 insertions(+), 183 deletions(-) diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py index 060d2c3..684c792 100644 --- a/stix2/sources/filters.py +++ b/stix2/sources/filters.py @@ -4,7 +4,6 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores """ import collections -import types # Currently, only STIX 2.0 common SDO fields (that are not complex objects) # are supported for filtering on @@ -34,12 +33,9 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] """Supported filter value types""" FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] -# filter lookup map - STIX 2 common fields -> filter method -STIX_COMMON_FILTERS_MAP = {} - def _check_filter_components(field, op, value): - """check filter meets minimum validity + """Check that filter meets minimum validity. Note: Currently can create Filters that are not valid STIX2 object common properties, as filter.field value @@ -57,7 +53,7 @@ def _check_filter_components(field, op, value): if type(value) not in FILTER_VALUE_TYPES: # check filter value type is supported - raise TypeError("Filter value type '%s' is not supported. The type must be a python immutable type or dictionary" % type(value)) + raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value)) return True @@ -66,13 +62,11 @@ class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): """STIX 2 filters that support the querying functionality of STIX 2 DataStores and DataSources. - Initialized like a python tuple + Initialized like a Python tuple. Args: field (str): filter field name, corresponds to STIX 2 object property - op (str): operator of the filter - value (str): filter field value Example: @@ -91,26 +85,11 @@ class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): self = super(Filter, cls).__new__(cls, field, op, value) return self - @property - def common(self): - """return whether Filter is valid STIX2 Object common property - - Note: The Filter operator and Filter value type are checked when - the filter is created, thus only leaving the Filter field to be - checked to make sure a valid STIX2 Object common property. - - Note: Filters that are not valid STIX2 Object common property - Filters are still allowed to be created for extended usage of - Filter. (e.g. TAXII specific filters can be created, which are - then extracted and sent to TAXII endpoint.) - """ - return self.field in STIX_COMMON_FIELDS - def apply_common_filters(stix_objs, query): """Evaluate filters against a set of STIX 2.0 objects. - Supports only STIX 2.0 common property fields + Supports only STIX 2.0 common property fields. Args: stix_objs (list): list of STIX objects to apply the query to @@ -124,25 +103,7 @@ def apply_common_filters(stix_objs, query): for stix_obj in stix_objs: clean = True for filter_ in query: - if not filter_.common: - # skip filter as it is not a STIX2 Object common property filter - continue - - if "." in filter_.field: - # For properties like granular_markings and external_references - # need to extract the first property from the string. - field = filter_.field.split(".")[0] - else: - field = filter_.field - - if field not in stix_obj.keys(): - # check filter "field" is in STIX object - if cant be - # applied to STIX object, STIX object is discarded - # (i.e. did not make it through the filter) - clean = False - break - - match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) + match = _check_filter(filter_, stix_obj) if not match: clean = False @@ -155,7 +116,53 @@ def apply_common_filters(stix_objs, query): yield stix_obj -"""Base type filters""" +def _check_filter(filter_, stix_obj): + """Evaluate a single filter against a single STIX 2.0 object. + + Args: + filter_ (Filter): filter to match against + stix_obj: STIX object to apply the filter to + + Returns: + True if the stix_obj matches the filter, + False if not. + + """ + if "." in filter_.field: + # For properties like granular_markings and external_references + # need to extract the first property from the string. + field = filter_.field.split(".")[0] + else: + field = filter_.field + + if field not in stix_obj.keys(): + # check filter "field" is in STIX object - if cant be + # applied to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + return False + + if "." in filter_.field: + # Check embedded properties, from e.g. granular_markings or external_references + sub_field = filter_.field.split(".", 1)[1] + sub_filter = filter_._replace(field=sub_field) + if isinstance(stix_obj[field], list): + for elem in stix_obj[field]: + r = _check_filter(sub_filter, elem) + if r: + return r + return False + else: + return _check_filter(sub_filter, stix_obj[field]) + elif isinstance(stix_obj[field], list): + # Check each item in list property to see if it matches + for elem in stix_obj[field]: + r = _all_filter(filter_, elem) + if r: + return r + return False + else: + # Check if property matches + return _all_filter(filter_, stix_obj[field]) def _all_filter(filter_, stix_obj_field): @@ -176,140 +183,3 @@ def _all_filter(filter_, stix_obj_field): return stix_obj_field <= filter_.value else: return -1 - - -def _id_filter(filter_, stix_obj_id): - """base STIX id filter""" - if filter_.op == "=": - return stix_obj_id == filter_.value - elif filter_.op == "!=": - return stix_obj_id != filter_.value - else: - return -1 - - -def _boolean_filter(filter_, stix_obj_field): - """base boolean filter""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - else: - return -1 - - -def _string_filter(filter_, stix_obj_field): - """base string filter""" - return _all_filter(filter_, stix_obj_field) - - -def _timestamp_filter(filter_, stix_obj_timestamp): - """base STIX 2 timestamp filter""" - return _all_filter(filter_, stix_obj_timestamp) - - -"""STIX 2.0 Common Property Filters - -The naming of these functions is important as -they are used to index a mapping dictionary from -STIX common field names to these filter functions. - -REQUIRED naming scheme: - "check__filter" - -""" - - -def check_created_filter(filter_, stix_obj): - return _timestamp_filter(filter_, stix_obj["created"]) - - -def check_created_by_ref_filter(filter_, stix_obj): - return _id_filter(filter_, stix_obj["created_by_ref"]) - - -def check_external_references_filter(filter_, stix_obj): - """ - STIX object's can have a list of external references - - external_references properties supported: - external_references.source_name (string) - external_references.description (string) - external_references.url (string) - external_references.external_id (string) - - external_references properties not supported: - external_references.hashes - - """ - for er in stix_obj["external_references"]: - # grab er property name from filter field - filter_field = filter_.field.split(".")[1] - if filter_field in er: - r = _string_filter(filter_, er[filter_field]) - if r: - return r - return False - - -def check_granular_markings_filter(filter_, stix_obj): - """ - STIX object's can have a list of granular marking references - - granular_markings properties: - granular_markings.marking_ref (id) - granular_markings.selectors (string) - - """ - for gm in stix_obj["granular_markings"]: - # grab gm property name from filter field - filter_field = filter_.field.split(".")[1] - - if filter_field == "marking_ref": - return _id_filter(filter_, gm[filter_field]) - - elif filter_field == "selectors": - for selector in gm[filter_field]: - r = _string_filter(filter_, selector) - if r: - return r - return False - - -def check_id_filter(filter_, stix_obj): - return _id_filter(filter_, stix_obj["id"]) - - -def check_labels_filter(filter_, stix_obj): - for label in stix_obj["labels"]: - r = _string_filter(filter_, label) - if r: - return r - return False - - -def check_modified_filter(filter_, stix_obj): - return _timestamp_filter(filter_, stix_obj["modified"]) - - -def check_object_marking_refs_filter(filter_, stix_obj): - for marking_id in stix_obj["object_marking_refs"]: - r = _id_filter(filter_, marking_id) - if r: - return r - return False - - -def check_revoked_filter(filter_, stix_obj): - return _boolean_filter(filter_, stix_obj["revoked"]) - - -def check_type_filter(filter_, stix_obj): - return _string_filter(filter_, stix_obj["type"]) - - -# Create mapping of field names to filter functions -for name, obj in dict(globals()).items(): - if "check_" in name and isinstance(obj, types.FunctionType): - field_name = "_".join(name.split("_")[1:-1]) - STIX_COMMON_FILTERS_MAP[field_name] = obj diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 689fe8c..583acea 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -226,7 +226,7 @@ def test_add_get_remove_filter(ds): Filter('created', '=', object()) # On Python 2, the type of object() is `` On Python 3, it's ``. assert str(excinfo.value).startswith("Filter value type") - assert str(excinfo.value).endswith("is not supported. The type must be a python immutable type or dictionary") + assert str(excinfo.value).endswith("is not supported. The type must be a Python immutable type or dictionary") assert len(ds.filters) == 0 @@ -443,6 +443,52 @@ def test_filters5(ds): assert len(resp) == 1 +def test_filters6(ds): + # Test filtering on non-common property + resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + +def test_filters7(ds): + # Test filtering on embedded property + stix_objects = list(STIX_OBJS2) + [{ + "type": "observed-data", + "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 50, + "objects": { + "0": { + "type": "file", + "hashes": { + "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + }, + "extensions": { + "pdf-ext": { + "version": "1.7", + "document_info_dict": { + "Title": "Sample document", + "Author": "Adobe Systems Incorporated", + "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh", + "Producer": "Acrobat Distiller 3.01 for Power Macintosh", + "CreationDate": "20070412090123-02" + }, + "pdfid0": "DFCE52BD827ECF765649852119D", + "pdfid1": "57A1E0F9ED2AE523E313C" + } + } + } + } + }] + resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) + assert resp[0]['id'] == stix_objects[3]['id'] + assert len(resp) == 1 + + def test_deduplicate(ds): unique = deduplicate(STIX_OBJS1) From d8e4f1ab98b32b8492fbb6f36d30a3cddf5ac770 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 31 Oct 2017 13:48:31 -0400 Subject: [PATCH 33/90] addessing requested changes --- docs/guide/taxii.ipynb | 2 +- stix2/sources/taxii.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index d44b12e..016f9d8 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -110,7 +110,7 @@ "from taxii2client import Collection\n", "\n", "# establish TAXII2 Collection instance\n", - "collection = Collection(\"http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/\", user=\"admin\", password=\"password\")\n", + "collection = Collection(\"http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/\", user=\"admin\", password=\"Password0\")\n", "# supply the TAXII2 collection to TAXIICollection\n", "tc_source = TAXIICollectionSource(collection)\n", "\n", diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 5dcd9f2..4c659ed 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,10 +1,5 @@ """ -Python STIX 2.x -Classes: - TAXIICollectionStore - TAXIICollectionSink - TAXIICollectionSource - +Python STIX 2.x TaxiiCollectionStore """ from stix2.base import _STIXBase @@ -122,7 +117,7 @@ class TAXIICollectionSource(DataSource): # as directly retrieveing a STIX object by ID stix_objs = self.collection.get_object(stix_id)["objects"] - stix_obj = [apply_common_filters(stix_objs, query)] + stix_obj = list(apply_common_filters(stix_objs, query)) if len(stix_obj): stix_obj = parse(stix_obj[0]) @@ -190,7 +185,7 @@ class TAXIICollectionSource(DataSource): if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters From e985d3b1d56f5c7e49a585dc421774727d0220ae Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Nov 2017 10:01:41 -0400 Subject: [PATCH 34/90] Move ts_support to Jupiter Notebook --- docs/guide/ts_support.ipynb | 237 ++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 - docs/ts_support.rst | 134 -------------------- 3 files changed, 237 insertions(+), 135 deletions(-) create mode 100644 docs/guide/ts_support.ipynb delete mode 100644 docs/ts_support.rst diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb new file mode 100644 index 0000000..d353dfb --- /dev/null +++ b/docs/guide/ts_support.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Technical Specification Support\n", + "\n", + "### How imports will work\n", + "\n", + "Imports can be used in different ways depending on the use case and support levels.\n", + "\n", + "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "People who want to use an explicit version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20\n", + "\n", + "stix2.v20.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or even," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20 as stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The last option makes it easy to update to a new version in one place per file, once you've made the deliberate action to do this.\n", + "\n", + "People who want to use multiple versions in a single file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.v20.Indicator()\n", + "\n", + "stix2.v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import v20, v21\n", + "\n", + "v20.Indicator()\n", + "v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or (less preferred):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator as Indicator_v20\n", + "from stix2.v21 import Indicator as Indicator_v21\n", + "\n", + "Indicator_v20()\n", + "Indicator_v21()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How parsing will work\n", + "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", + "\n", + "You can lock your `parse()` method to a specific STIX version by" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import parse\n", + "\n", + "indicator = parse(\"\"\"{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n", + " \"created\": \"2017-09-26T23:33:39.829Z\",\n", + " \"modified\": \"2017-09-26T23:33:39.829Z\",\n", + " \"labels\": [\n", + " \"malicious-activity\"\n", + " ],\n", + " \"name\": \"File hash for malware variant\",\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2017-09-26T23:33:39.829952Z\"\n", + "}\"\"\", version=\"2.0\")\n", + "print(indicator)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Keep in mind that if a 2.1 or higher object is parsed, the operation will fail." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How will custom work\n", + "\n", + "CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "\n", + "You can perform this by," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "# Make my custom observable available in STIX 2.0\n", + "@stix2.v20.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass\n", + "\n", + "\n", + "# Make my custom observable available in STIX 2.1\n", + "@stix2.v21.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/index.rst b/docs/index.rst index f865467..62d07ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,6 @@ Welcome to stix2's documentation! datastore_api roadmap contributing - ts_support Indices and tables diff --git a/docs/ts_support.rst b/docs/ts_support.rst deleted file mode 100644 index a62e59e..0000000 --- a/docs/ts_support.rst +++ /dev/null @@ -1,134 +0,0 @@ -How imports will work ---------------------- - -Imports can be used in different ways depending on the use case and support -levels. - -People who want to (in general) support the latest version of STIX 2.X without -making changes, implicitly using the latest version - -.. code:: python - - import stix2 - ... - stix2.Indicator(...) - -or - -.. code:: python - - from stix2 import Indicator - ... - Indicator(...) - -People who want to use an explicit version - -.. code:: python - - import stix2.v20 - ... - stix2.v20.Indicator(...) - -or - -.. code:: python - - from stix2.v20 import Indicator - ... - Indicator(...) - -or even, - -.. code:: python - - import stix2.v20 as stix2 - ... - stix2.Indicator(...) - -The last option makes it easy to update to a new version in one place per file, -once you've made the deliberate action to do this. - -People who want to use multiple versions in a single file: - -.. code:: python - - import stix2 - ... - stix2.v20.Indicator(...) - ... - stix2.v21.Indicator(...) - -or - -.. code:: python - - from stix2 import v20, v21 - ... - v20.Indicator(...) - ... - v21.Indicator(...) - -or (less preferred): - -.. code:: python - - from stix2.v20 import Indicator as Indicator_v20 - from stix2.v21 import Indicator as Indicator_v21 - ... - Indicator_v20(...) - ... - Indicator_v21(...) - -How parsing will work ---------------------- - -If the ``version`` positional argument is not provided. The data will be parsed -using the latest version of STIX 2.X supported by the `stix2` library. - -You can lock your `parse()` method to a specific STIX version by - -.. code:: python - - from stix2 import parse - - indicator = parse("""{ - "type": "indicator", - "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394", - "created": "2017-09-26T23:33:39.829Z", - "modified": "2017-09-26T23:33:39.829Z", - "labels": [ - "malicious-activity" - ], - "name": "File hash for malware variant", - "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", - "valid_from": "2017-09-26T23:33:39.829952Z" - }""", version="2.0") - print(indicator) - -Keep in mind that if a 2.1 or higher object is parsed, the operation will fail. - -How will custom work --------------------- - -CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be -registered explicitly by STIX version. This is a design decision since properties -or requirements may change as the STIX Technical Specification advances. - -You can perform this by, - -.. code:: python - - import stix2 - - # Make my custom observable available in STIX 2.0 - @stix2.v20.observables.CustomObservable('x-new-object-type', - (("prop", stix2.properties.BooleanProperty()))) - class NewObject2(object): - pass - - - # Make my custom observable available in STIX 2.1 - @stix2.v21.observables.CustomObservable('x-new-object-type', - (("prop", stix2.properties.BooleanProperty()))) - class NewObject2(object): - pass From ef20860400019c89c9475bdf24084b928ae08b74 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Nov 2017 10:03:03 -0400 Subject: [PATCH 35/90] Detele support.ipynb --- docs/guide/support.ipynb | 235 --------------------------------------- 1 file changed, 235 deletions(-) delete mode 100644 docs/guide/support.ipynb diff --git a/docs/guide/support.ipynb b/docs/guide/support.ipynb deleted file mode 100644 index 3c56f1d..0000000 --- a/docs/guide/support.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### How imports will work\n", - "\n", - "Imports can be used in different ways depending on the use case and support levels.\n", - "\n", - "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import stix2\n", - "\n", - "stix2.Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or," - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from stix2 import Indicator\n", - "\n", - "Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "People who want to use an explicit version" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import stix2.v20\n", - "\n", - "stix2.v20.Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or," - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from stix2.v20 import Indicator\n", - "\n", - "Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or even," - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import stix2.v20 as stix2\n", - "\n", - "stix2.Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The last option makes it easy to update to a new version in one place per file, once you've made the deliberate action to do this.\n", - "\n", - "People who want to use multiple versions in a single file:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import stix2\n", - "\n", - "stix2.v20.Indicator()\n", - "\n", - "stix2.v21.Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or," - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from stix2 import v20, v21\n", - "\n", - "v20.Indicator()\n", - "v21.Indicator()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "or (less preferred):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from stix2.v20 import Indicator as Indicator_v20\n", - "from stix2.v21 import Indicator as Indicator_v21\n", - "\n", - "Indicator_v20()\n", - "Indicator_v21()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### How parsing will work\n", - "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", - "\n", - "You can lock your `parse()` method to a specific STIX version by" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from stix2 import parse\n", - "\n", - "indicator = parse(\"\"\"{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n", - " \"created\": \"2017-09-26T23:33:39.829Z\",\n", - " \"modified\": \"2017-09-26T23:33:39.829Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"File hash for malware variant\",\n", - " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", - " \"valid_from\": \"2017-09-26T23:33:39.829952Z\"\n", - "}\"\"\", version=\"2.0\")\n", - "print(indicator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Keep in mind that if a 2.1 or higher object is parsed, the operation will fail." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### How will custom work\n", - "\n", - "CustomObjects, CustomObservable, CustomMarking and CustomExtensions must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", - "\n", - "You can perform this by," - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import stix2\n", - "\n", - "# Make my custom observable available in STIX 2.0\n", - "@stix2.v20.observables.CustomObservable('x-new-object-type',\n", - " ((\"prop\", stix2.properties.BooleanProperty())))\n", - "class NewObject2(object):\n", - " pass\n", - "\n", - "\n", - "# Make my custom observable available in STIX 2.1\n", - "@stix2.v21.observables.CustomObservable('x-new-object-type',\n", - " ((\"prop\", stix2.properties.BooleanProperty())))\n", - "class NewObject2(object):\n", - " pass" - ] - } - ], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 0 -} From c2d1e9777ba9b74c6fd8c0ff2043d90882b433cd Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 1 Nov 2017 10:40:10 -0400 Subject: [PATCH 36/90] Clean up filters - Simplify an if statement since split() with no matches returns single item list - Rename _all_filter -> _check_property and make it a method on Filter - Raise an error instead of returning -1 - s/field/property --- stix2/sources/filesystem.py | 14 +-- stix2/sources/filters.py | 166 ++++++++++++++------------------ stix2/sources/taxii.py | 8 +- stix2/test/test_data_sources.py | 6 +- 4 files changed, 85 insertions(+), 109 deletions(-) diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 103b882..fd737fc 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -225,14 +225,14 @@ class FileSystemSource(DataSource): file_filters = self._parse_file_filters(query) # establish which subdirectories can be avoided in query - # by decluding as many as possible. A filter with "type" as the field + # by decluding as many as possible. A filter with "type" as the property # means that certain STIX object types can be ruled out, and thus # the corresponding subdirectories as well include_paths = [] declude_paths = [] - if "type" in [filter.field for filter in file_filters]: + if "type" in [filter.property for filter in file_filters]: for filter in file_filters: - if filter.field == "type": + if filter.property == "type": if filter.op == "=": include_paths.append(os.path.join(self._stix_dir, filter.value)) elif filter.op == "!=": @@ -259,9 +259,9 @@ class FileSystemSource(DataSource): # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory - if "id" in [filter.field for filter in file_filters]: + if "id" in [filter.property for filter in file_filters]: for filter in file_filters: - if filter.field == "id" and filter.op == "=": + if filter.property == "id" and filter.op == "=": id_ = filter.value break else: @@ -296,7 +296,7 @@ class FileSystemSource(DataSource): that can used to possibly speed up querying STIX objects from the file system - Extracts filters that are for the "id" and "type" field of + Extracts filters that are for the "id" and "type" property of a STIX object. As the file directory is organized by STIX object type with filenames that are equivalent to the STIX object ID, these filters can be used first to reduce the @@ -304,6 +304,6 @@ class FileSystemSource(DataSource): """ file_filters = set() for filter_ in query: - if filter_.field == "id" or filter_.field == "type": + if filter_.property == "id" or filter_.property == "type": file_filters.add(filter_) return file_filters diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py index 684c792..5772112 100644 --- a/stix2/sources/filters.py +++ b/stix2/sources/filters.py @@ -5,28 +5,6 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores import collections -# Currently, only STIX 2.0 common SDO fields (that are not complex objects) -# are supported for filtering on - -"""Supported STIX properties""" -STIX_COMMON_FIELDS = [ - "created", - "created_by_ref", - "external_references.source_name", - "external_references.description", - "external_references.url", - "external_references.hashes", - "external_references.external_id", - "granular_markings.marking_ref", - "granular_markings.selectors", - "id", - "labels", - "modified", - "object_marking_refs", - "revoked", - "type" -] - """Supported filter operations""" FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] @@ -34,40 +12,39 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] -def _check_filter_components(field, op, value): - """Check that filter meets minimum validity. +def _check_filter_components(prop, op, value): + """Check that filter meets minimum validity. - Note: Currently can create Filters that are not valid - STIX2 object common properties, as filter.field value - is not checked, only filter.op, filter.value are checked - here. They are just ignored when - applied within the DataSource API. For example, a user - can add a TAXII Filter, that is extracted and sent to - a TAXII endpoint within TAXIICollection and not applied - locally (within this API). - """ + Note: + Currently can create Filters that are not valid STIX2 object common + properties, as filter.prop value is not checked, only filter.op, + filter value are checked here. They are just ignored when applied + within the DataSource API. For example, a user can add a TAXII Filter, + that is extracted and sent to a TAXII endpoint within TAXIICollection + and not applied locally (within this API). - if op not in FILTER_OPS: - # check filter operator is supported - raise ValueError("Filter operator '%s' not supported for specified field: '%s'" % (op, field)) + """ + if op not in FILTER_OPS: + # check filter operator is supported + raise ValueError("Filter operator '%s' not supported for specified property: '%s'" % (op, prop)) - if type(value) not in FILTER_VALUE_TYPES: - # check filter value type is supported - raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value)) + if type(value) not in FILTER_VALUE_TYPES: + # check filter value type is supported + raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value)) - return True + return True -class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): +class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])): """STIX 2 filters that support the querying functionality of STIX 2 DataStores and DataSources. Initialized like a Python tuple. Args: - field (str): filter field name, corresponds to STIX 2 object property + property (str): filter property name, corresponds to STIX 2 object property op (str): operator of the filter - value (str): filter field value + value (str): filter property value Example: Filter("id", "=", "malware--0f862b01-99da-47cc-9bdb-db4a86a95bb1") @@ -75,29 +52,55 @@ class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): """ __slots__ = () - def __new__(cls, field, op, value): + def __new__(cls, prop, op, value): # If value is a list, convert it to a tuple so it is hashable. if isinstance(value, list): value = tuple(value) - _check_filter_components(field, op, value) + _check_filter_components(prop, op, value) - self = super(Filter, cls).__new__(cls, field, op, value) + self = super(Filter, cls).__new__(cls, prop, op, value) return self + def _check_property(self, stix_obj_property): + """Check a property of a STIX Object against this filter. + + Args: + stix_obj_property: value to check this filter against + + Returns: + True if property matches the filter, + False otherwise. + """ + if self.op == "=": + return stix_obj_property == self.value + elif self.op == "!=": + return stix_obj_property != self.value + elif self.op == "in": + return stix_obj_property in self.value + elif self.op == ">": + return stix_obj_property > self.value + elif self.op == "<": + return stix_obj_property < self.value + elif self.op == ">=": + return stix_obj_property >= self.value + elif self.op == "<=": + return stix_obj_property <= self.value + else: + raise ValueError("Filter operator: {0} not supported for specified property: {1}".format(self.op, self.property)) + def apply_common_filters(stix_objs, query): """Evaluate filters against a set of STIX 2.0 objects. - Supports only STIX 2.0 common property fields. + Supports only STIX 2.0 common property properties. Args: stix_objs (list): list of STIX objects to apply the query to query (set): set of filters (combined form complete query) - Returns: - (generator): of STIX objects that successfully evaluate against - the query. + Yields: + STIX objects that successfully evaluate against the query. """ for stix_obj in stix_objs: @@ -108,8 +111,6 @@ def apply_common_filters(stix_objs, query): if not match: clean = False break - elif match == -1: - raise ValueError("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) # if object unmarked after all filters, add it if clean: @@ -128,58 +129,33 @@ def _check_filter(filter_, stix_obj): False if not. """ - if "." in filter_.field: - # For properties like granular_markings and external_references - # need to extract the first property from the string. - field = filter_.field.split(".")[0] - else: - field = filter_.field + # For properties like granular_markings and external_references + # need to extract the first property from the string. + prop = filter_.property.split(".")[0] - if field not in stix_obj.keys(): - # check filter "field" is in STIX object - if cant be + if prop not in stix_obj.keys(): + # check filter "property" is in STIX object - if cant be # applied to STIX object, STIX object is discarded # (i.e. did not make it through the filter) return False - if "." in filter_.field: + if "." in filter_.property: # Check embedded properties, from e.g. granular_markings or external_references - sub_field = filter_.field.split(".", 1)[1] - sub_filter = filter_._replace(field=sub_field) - if isinstance(stix_obj[field], list): - for elem in stix_obj[field]: - r = _check_filter(sub_filter, elem) - if r: - return r + sub_property = filter_.property.split(".", 1)[1] + sub_filter = filter_._replace(property=sub_property) + if isinstance(stix_obj[prop], list): + for elem in stix_obj[prop]: + if _check_filter(sub_filter, elem) is True: + return True return False else: - return _check_filter(sub_filter, stix_obj[field]) - elif isinstance(stix_obj[field], list): + return _check_filter(sub_filter, stix_obj[prop]) + elif isinstance(stix_obj[prop], list): # Check each item in list property to see if it matches - for elem in stix_obj[field]: - r = _all_filter(filter_, elem) - if r: - return r + for elem in stix_obj[prop]: + if filter_._check_property(elem) is True: + return True return False else: # Check if property matches - return _all_filter(filter_, stix_obj[field]) - - -def _all_filter(filter_, stix_obj_field): - """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - elif filter_.op == "in": - return stix_obj_field in filter_.value - elif filter_.op == ">": - return stix_obj_field > filter_.value - elif filter_.op == "<": - return stix_obj_field < filter_.value - elif filter_.op == ">=": - return stix_obj_field >= filter_.value - elif filter_.op == "<=": - return stix_obj_field <= filter_.value - else: - return -1 + return filter_._check_property(stix_obj[prop]) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 7ecedca..8f8aa4b 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -229,10 +229,10 @@ class TAXIICollectionSource(DataSource): params = {} for filter_ in query: - if filter_.field in TAXII_FILTERS: - if filter_.field == "added_after": - params[filter_.field] = filter_.value + if filter_.property in TAXII_FILTERS: + if filter_.property == "added_after": + params[filter_.property] = filter_.value else: - taxii_field = "match[%s]" % filter_.field + taxii_field = "match[%s]" % filter_.property params[taxii_field] = filter_.value return params diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 583acea..a2d8daa 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -205,7 +205,7 @@ def test_parse_taxii_filters(): def test_add_get_remove_filter(ds): - # First 3 filters are valid, remaining fields are erroneous in some way + # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ Filter('type', '=', 'malware'), Filter('id', '!=', 'stix object id'), @@ -219,7 +219,7 @@ def test_add_get_remove_filter(ds): with pytest.raises(ValueError) as excinfo: # create Filter that has an operator that is not allowed Filter('modified', '*', 'not supported operator - just place holder') - assert str(excinfo.value) == "Filter operator '*' not supported for specified field: 'modified'" + assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'" with pytest.raises(TypeError) as excinfo: # create Filter that has a value type that is not allowed @@ -433,7 +433,7 @@ def test_filters4(ds): with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") assert str(excinfo.value) == ("Filter operator '?' not supported " - "for specified field: 'modified'") + "for specified property: 'modified'") def test_filters5(ds): From 06a50b0178eb826b845cac96681d7ee028a0b5be Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Nov 2017 10:48:28 -0400 Subject: [PATCH 37/90] Update README, add DEFAULT_VERSION --- README.rst | 5 +++-- docs/guide/ts_support.ipynb | 2 +- stix2/__init__.py | 4 +++- stix2/core.py | 10 +++++----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index c2ec908..7884ba0 100644 --- a/README.rst +++ b/README.rst @@ -65,8 +65,9 @@ For more in-depth documentation, please see `https://stix2.readthedocs.io/ Date: Wed, 1 Nov 2017 11:03:02 -0400 Subject: [PATCH 38/90] Fix import statement --- stix2/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index e860e95..10754f0 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -4,7 +4,9 @@ from collections import OrderedDict import importlib import pkgutil -from . import DEFAULT_VERSION, exceptions +import stix2 + +from . import exceptions from .base import _STIXBase from .properties import IDProperty, ListProperty, Property, TypeProperty from .utils import get_dict @@ -78,7 +80,7 @@ def parse(data, allow_custom=False, version=None): """ if not version: # Use latest version - v = 'v' + DEFAULT_VERSION.replace('.', '') + v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') else: v = 'v' + version.replace('.', '') @@ -105,7 +107,7 @@ def _register_type(new_type, version=None): """ if not version: # Use latest version - v = 'v' + DEFAULT_VERSION.replace('.', '') + v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') else: v = 'v' + version.replace('.', '') From 7b9f09ee18dfced7f13e1eac53ae40d5aae48c75 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Nov 2017 12:46:01 -0400 Subject: [PATCH 39/90] Minor change --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 7884ba0..faacc53 100644 --- a/README.rst +++ b/README.rst @@ -65,9 +65,9 @@ For more in-depth documentation, please see `https://stix2.readthedocs.io/ Date: Wed, 1 Nov 2017 12:48:49 -0400 Subject: [PATCH 40/90] Test MemoryStore saving/loading to/from file Python 3 dict.values() returns a view, not a list. See https://stackoverflow.com/a/17431716/ Fix #65. --- stix2/sources/memory.py | 2 +- stix2/test/test_memory.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2d1705d..4d3943b 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -164,7 +164,7 @@ class MemorySink(DataSink): if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(self._data.values(), allow_custom=allow_custom))) + f.write(str(Bundle(list(self._data.values()), allow_custom=allow_custom))) save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index 0603bf7..7a00029 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -1,3 +1,6 @@ +import os +import shutil + import pytest from stix2 import (Bundle, Campaign, CustomObject, Filter, MemorySource, @@ -166,6 +169,22 @@ def test_memory_store_query_multiple_filters(mem_store): assert len(resp) == 1 +def test_memory_store_save_load_file(mem_store): + filename = 'memory_test/mem_store.json' + mem_store.save_to_file(filename) + contents = open(os.path.abspath(filename)).read() + + assert '"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",' in contents + assert '"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",' in contents + + mem_store2 = MemoryStore() + mem_store2.load_from_file(filename) + assert mem_store2.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert mem_store2.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + shutil.rmtree(os.path.dirname(filename)) + + def test_memory_store_add_stix_object_str(mem_store): # add stix object string camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" From b2ff16994f46cfe930ed343d170a5816685b4b54 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 1 Nov 2017 14:21:26 -0400 Subject: [PATCH 41/90] Document new util method --- stix2/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stix2/utils.py b/stix2/utils.py index 91cd071..f23dbe2 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -250,6 +250,7 @@ def revoke(data): def get_class_hierarchy_names(obj): + """Given an object, return the names of the class hierarchy.""" names = [] for cls in obj.__class__.__mro__: names.append(cls.__name__) From bdb91c6ac4e5a7758b5df86ee70eba4eb877756b Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 07:21:24 -0400 Subject: [PATCH 42/90] Update STIX 2.1 structure --- stix2/v21/__init__.py | 49 ++++++++++++++++++++++++++++++++++ stix2/{ => v21}/common.py | 16 +++++------ stix2/{ => v21}/observables.py | 20 +++++++------- stix2/{ => v21}/sdo.py | 18 ++++++------- stix2/{ => v21}/sro.py | 14 +++++----- 5 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 stix2/v21/__init__.py rename stix2/{ => v21}/common.py (94%) rename stix2/{ => v21}/observables.py (98%) rename stix2/{ => v21}/sdo.py (97%) rename stix2/{ => v21}/sro.py (91%) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py new file mode 100644 index 0000000..dad0785 --- /dev/null +++ b/stix2/v21/__init__.py @@ -0,0 +1,49 @@ + +# flake8: noqa + +from ..core import Bundle +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + LanguageContent, MarkingDefinition, StatementMarking, + TLPMarking) +from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, + AutonomousSystem, CustomExtension, CustomObservable, + Directory, DomainName, EmailAddress, EmailMessage, + EmailMIMEComponent, File, HTTPRequestExt, ICMPExt, + IPv4Address, IPv6Address, MACAddress, Mutex, + NetworkTraffic, NTFSExt, PDFExt, Process, + RasterImageExt, SocketExt, Software, TCPExt, + UNIXAccountExt, UserAccount, WindowsPEBinaryExt, + WindowsPEOptionalHeaderType, WindowsPESection, + WindowsProcessExt, WindowsRegistryKey, + WindowsRegistryValueType, WindowsServiceExt, + X509Certificate, X509V3ExtenstionsType, + parse_observable) +from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, + Identity, Indicator, IntrusionSet, Location, Malware, Note, + ObservedData, Opinion, Report, ThreatActor, Tool, + Vulnerability) +from .sro import Relationship, Sighting + +OBJ_MAP = { + 'attack-pattern': AttackPattern, + 'bundle': Bundle, + 'campaign': Campaign, + 'course-of-action': CourseOfAction, + 'identity': Identity, + 'indicator': Indicator, + 'intrusion-set': IntrusionSet, + 'language-content': LanguageContent, + 'location': Location, + 'malware': Malware, + 'note': Note, + 'marking-definition': MarkingDefinition, + 'observed-data': ObservedData, + 'opinion': Opinion, + 'report': Report, + 'relationship': Relationship, + 'threat-actor': ThreatActor, + 'tool': Tool, + 'sighting': Sighting, + 'vulnerability': Vulnerability, +} diff --git a/stix2/common.py b/stix2/v21/common.py similarity index 94% rename from stix2/common.py rename to stix2/v21/common.py index fdeef03..7a7feb0 100644 --- a/stix2/common.py +++ b/stix2/v21/common.py @@ -1,14 +1,14 @@ -"""STIX 2 Common Data Types and Properties.""" +"""STIX 2.1 Common Data Types and Properties.""" from collections import OrderedDict -from .base import _STIXBase -from .markings import _MarkingsMixin -from .properties import (BooleanProperty, DictionaryProperty, HashesProperty, - IDProperty, ListProperty, Property, ReferenceProperty, - SelectorProperty, StringProperty, TimestampProperty, - TypeProperty) -from .utils import NOW, get_dict +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, DictionaryProperty, HashesProperty, + IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW, get_dict class ExternalReference(_STIXBase): diff --git a/stix2/observables.py b/stix2/v21/observables.py similarity index 98% rename from stix2/observables.py rename to stix2/v21/observables.py index aaec2d7..008f8e2 100644 --- a/stix2/observables.py +++ b/stix2/v21/observables.py @@ -1,4 +1,4 @@ -"""STIX 2.0 Cyber Observable Objects. +"""STIX 2.1 Cyber Observable Objects. Embedded observable object types, such as Email MIME Component, which is embedded in Email Message objects, inherit from ``_STIXBase`` instead of @@ -7,15 +7,15 @@ Observable and do not have a ``_type`` attribute. from collections import OrderedDict -from .base import _Extension, _Observable, _STIXBase -from .exceptions import (AtLeastOnePropertyError, DependentPropertiesError, - ParseError) -from .properties import (BinaryProperty, BooleanProperty, DictionaryProperty, - EmbeddedObjectProperty, EnumProperty, FloatProperty, - HashesProperty, HexProperty, IntegerProperty, - ListProperty, ObjectReferenceProperty, Property, - StringProperty, TimestampProperty, TypeProperty) -from .utils import get_dict +from ..base import _Extension, _Observable, _STIXBase +from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError, + ParseError) +from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty, + EmbeddedObjectProperty, EnumProperty, FloatProperty, + HashesProperty, HexProperty, IntegerProperty, + ListProperty, ObjectReferenceProperty, Property, + StringProperty, TimestampProperty, TypeProperty) +from ..utils import get_dict class ObservableProperty(Property): diff --git a/stix2/sdo.py b/stix2/v21/sdo.py similarity index 97% rename from stix2/sdo.py rename to stix2/v21/sdo.py index da8fa65..3c46fa1 100644 --- a/stix2/sdo.py +++ b/stix2/v21/sdo.py @@ -1,18 +1,18 @@ -"""STIX 2.0 Domain Objects""" +"""STIX 2.1 Domain Objects""" from collections import OrderedDict import stix2 -from .base import _STIXBase +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, EnumProperty, FloatProperty, + IDProperty, IntegerProperty, ListProperty, + PatternProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW from .common import ExternalReference, GranularMarking, KillChainPhase -from .markings import _MarkingsMixin from .observables import ObservableProperty -from .properties import (BooleanProperty, EnumProperty, FloatProperty, - IDProperty, IntegerProperty, ListProperty, - PatternProperty, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty) -from .utils import NOW class STIXDomainObject(_STIXBase, _MarkingsMixin): @@ -470,7 +470,7 @@ def CustomObject(type='x-custom-type', properties=None): return raise e - stix2._register_type(_Custom) + stix2._register_type(_Custom, version="2.1") return _Custom return custom_builder diff --git a/stix2/sro.py b/stix2/v21/sro.py similarity index 91% rename from stix2/sro.py rename to stix2/v21/sro.py index 6e86d59..03f08b7 100644 --- a/stix2/sro.py +++ b/stix2/v21/sro.py @@ -1,14 +1,14 @@ -"""STIX 2.0 Relationship Objects.""" +"""STIX 2.1 Relationship Objects.""" from collections import OrderedDict -from .base import _STIXBase +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW from .common import ExternalReference, GranularMarking -from .markings import _MarkingsMixin -from .properties import (BooleanProperty, IDProperty, IntegerProperty, - ListProperty, ReferenceProperty, StringProperty, - TimestampProperty, TypeProperty) -from .utils import NOW class STIXRelationshipObject(_STIXBase, _MarkingsMixin): From f6f7d0aed88220fd729e60c148d17221e17a30d5 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 07:48:37 -0400 Subject: [PATCH 43/90] Merge branch 'master' of github.com:oasis-open/cti-python-stix2 --- .isort.cfg | 1 - MANIFEST.in | 3 + README.rst | 12 +- docs/guide/taxii.ipynb | 2357 +---------------- docs/guide/ts_support.ipynb | 237 ++ setup.py | 5 +- stix2/__init__.py | 30 +- stix2/base.py | 9 +- stix2/core.py | 82 +- stix2/environment.py | 19 + stix2/sources/__init__.py | 64 +- stix2/sources/filesystem.py | 197 +- stix2/sources/filters.py | 344 +-- stix2/sources/memory.py | 172 +- stix2/sources/taxii.py | 79 +- ...-d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json | 21 +- stix2/test/test_bundle.py | 12 +- stix2/test/test_custom.py | 11 + stix2/test/test_data_sources.py | 286 +- stix2/test/test_environment.py | 32 + stix2/test/test_filesystem.py | 377 +++ stix2/test/test_memory.py | 270 ++ stix2/test/test_properties.py | 3 +- stix2/utils.py | 17 +- stix2/v20/__init__.py | 43 + stix2/v20/common.py | 189 ++ stix2/v20/observables.py | 948 +++++++ stix2/v20/sdo.py | 364 +++ stix2/v20/sro.py | 82 + stix2/v21/__init__.py | 8 +- 30 files changed, 3169 insertions(+), 3105 deletions(-) create mode 100644 MANIFEST.in create mode 100644 docs/guide/ts_support.ipynb create mode 100644 stix2/test/test_filesystem.py create mode 100644 stix2/test/test_memory.py create mode 100644 stix2/v20/__init__.py create mode 100644 stix2/v20/common.py create mode 100644 stix2/v20/observables.py create mode 100644 stix2/v20/sdo.py create mode 100644 stix2/v20/sro.py diff --git a/.isort.cfg b/.isort.cfg index f55fec7..d535851 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -9,7 +9,6 @@ known_third_party = simplejson six, stix2patterns, - stix2validator, taxii2client, known_first_party = stix2 force_sort_within_sections = 1 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c9ec75b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE +include CHANGELOG +recursive-exclude stix2\test * diff --git a/README.rst b/README.rst index c78923b..faacc53 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ including data markings, versioning, and for resolving STIX IDs across multiple data sources. For more information, see `the -documentation `__ on +documentation `__ on ReadTheDocs. Installation @@ -62,6 +62,16 @@ To parse a STIX JSON string into a Python STIX object, use ``parse()``: For more in-depth documentation, please see `https://stix2.readthedocs.io/ `__. +STIX 2.X Technical Specification Support +---------------------------------------- + +This version of python-stix2 supports STIX 2.0 by default. Although, the +`stix2` Python library is built to support multiple versions of the STIX +Technical Specification. With every major release of stix2 the ``import stix2`` +statement will automatically load the SDO/SROs equivalent to the most recent +supported 2.X Technical Specification. Please see the library documentation +for more details. + Governance ---------- diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 2890659..016f9d8 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -79,21 +79,27 @@ "output_type": "stream", "text": [ "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fb2c0e55-52a0-423c-b544-8b09622cafc1\",\n", - " \"created\": \"2017-10-02T19:26:30.000Z\",\n", - " \"modified\": \"2017-10-02T19:26:30.000Z\",\n", + " \"type\": \"malware\",\n", + " \"id\": \"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\",\n", + " \"created\": \"2017-01-27T13:49:53.997Z\",\n", + " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", + " \"name\": \"Poison Ivy\",\n", + " \"description\": \"Poison Ivy\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T19:26:30Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"remote-access-trojan\"\n", + " ]\n", + "}\n", + "-------\n", + "{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\",\n", + " \"created\": \"2014-05-08T09:00:00.000Z\",\n", + " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", + " \"name\": \"File hash for Poison Ivy variant\",\n", + " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", + " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", + " \"labels\": [\n", + " \"file-hash-watchlist\"\n", " ]\n", "}\n" ] @@ -104,2293 +110,40 @@ "from taxii2client import Collection\n", "\n", "# establish TAXII2 Collection instance\n", - "collection = Collection(\"https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/\")\n", + "collection = Collection(\"http://127.0.0.1:5000/trustgroup1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/\", user=\"admin\", password=\"Password0\")\n", "# supply the TAXII2 collection to TAXIICollection\n", "tc_source = TAXIICollectionSource(collection)\n", "\n", - "#retrieve STIX object by id\n", - "stix_obj = tc_source.get(\"indicator--0f63229c-07a2-46dd-939d-312c7bf6d114\")\n", + "#retrieve STIX objects by id\n", + "stix_obj = tc_source.get(\"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\")\n", + "stix_obj_versions = tc_source.all_versions(\"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\")\n", "\n", "#for visual purposes\n", - "print(stix_obj)\n" + "print(stix_obj)\n", + "print(\"-------\")\n", + "for so in stix_obj_versions:\n", + " print(so)\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "indicators: 126\n", "{\n", " \"type\": \"indicator\",\n", - " \"id\": \"indicator--569b8969-bfce-4ab4-9a45-06ce78799a35\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", + " \"id\": \"indicator--a932fcc6-e032-176c-126f-cb970a5a1ade\",\n", + " \"created\": \"2014-05-08T09:00:00.000Z\",\n", + " \"modified\": \"2014-05-08T09:00:00.000Z\",\n", + " \"name\": \"File hash for Poison Ivy variant\",\n", + " \"pattern\": \"[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']\",\n", + " \"valid_from\": \"2014-05-08T09:00:00Z\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '207.158.1.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9c418633-9970-424e-8030-2c3dfa3105da\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9d7cdfc1-94c3-49b5-b124-ebdce709fd99\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.67.22' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--37390a22-5d82-4ebc-9b90-7368a5efc8f7\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--30731d72-64b0-4851-bd97-c3d164d2fd2b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.24.188.100' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a4eb3524-992c-4b50-9729-99be3048625e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.232.93.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c00fb599-7e7b-4033-a6c2-d279212578a0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.45' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e7273b13-847c-4a69-8faf-08fc24af5ef0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '89.16.176.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b8d21867-c812-4ff9-866b-182a801b88ce\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.239.18.172' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4c39b1a0-17f0-4cf1-9e48-250f0dd1f75c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8eeff049-f7da-45d9-89bb-713063baed2c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.92.8.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e3981158-1934-4236-8454-4dcfc27ac248\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.87.120.111' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--206c2a0c-149f-426f-a734-c0c534aa396b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.93.243.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--58d7aa16-8baf-4026-b3d7-328267ed4bab\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.165.191.52' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e6fd4a21-8290-40e5-9b1c-701f6f11e260\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.204.1.132' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cca5ce5f-4c0e-4031-9997-063eb3badead\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '209.177.146.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--43a7784e-f11c-4739-91a8-dc87d05ddbb6\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '145.220.21.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5716d954-e5b1-4bec-ba43-80b1053dee61\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '50.7.55.82' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9135d4ab-a807-495b-8fff-b8433342501f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.165.47.254' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e070c86b-40e5-49ea-8d83-56bcae10b303\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f4125383-930c-42ae-b57f-2c36f898d0b5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.71.169.36' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fa063c6a-1a9f-4a58-9470-ed80a23cc943\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '204.152.221.218' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--41b3ba86-dd1b-4f3d-a156-5dc27f31fb40\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '78.40.125.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a9fcaba5-cd50-447d-8540-2dfe4e3c6c88\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.68' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--30b68eff-3c38-4c74-9783-1114a7759066\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.197.175.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f10fa7c0-7a10-434e-908f-59a7e25e18c0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.14.236.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--183f8cd7-2e6f-4073-bbe8-d5dc6b570fac\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--dd95ff3a-3ef1-409e-827b-087eb9cc3b2c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a97dc9cb-2b9f-4c1d-92cc-2fc15100e3ed\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '91.205.185.104' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5552096e-b2b8-4057-bf5e-ccf300b8276e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.163.220.3' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--0cc30ea9-eeaf-4f39-ab8d-3d2664c2b75e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.202.189.170' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--7582ed02-c78d-451d-b0a5-065ae511f3ae\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '86.65.39.15' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--37fde688-ca75-4c1e-b5e1-1acb5bbfb23c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.167.98' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e967d3a0-0cfe-482c-b53a-390c0bb564f4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '199.16.156.6' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fda4f25d-8252-4593-bd8b-0a90764a561f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '217.168.95.245' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--109b3de1-2353-42dc-8316-e2f7c0b5c67d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.16.195' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1efa50e4-ed2c-4fb5-ae9b-cb347bd4ad24\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.18.128.86' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c7b60a1a-4c93-451f-b7c1-993c0dc14391\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.109.129.220' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--469381d9-c24e-4cf4-b25b-18a48975ef14\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.99.193.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c5694bbd-3a11-4c16-ae73-eeed55acf9cc\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '70.84.101.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a9b4301e-0327-4edc-b407-b7915bb0e7bc\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.62' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f5ac23ca-8ab4-4597-837b-3d5e48d325cb\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.61' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1a2a539b-d3f3-410b-a32c-4d1a5599364e\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '66.186.59.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--585e6f7b-7bad-45b0-a36b-9f3b3bff72c6\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '93.152.160.101' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--0a7dd603-d826-428f-b5f7-c82ff8bb60f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.46' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cb2cebd2-c11f-43b1-a9a1-3c4b9893f38a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.150.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--6a6c81df-7cb9-48b3-a4ea-db6924e47b5d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.107.206.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--45177dce-6cfe-44b5-ac41-cbc1bee80527\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--6f58bdf5-1f26-4a17-8ba3-14c023e73a0f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '72.51.18.254' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d5731bef-623c-4793-994c-a6f3840bc2cf\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.190.67.98' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4e8ac337-2e00-4d71-8526-bbfdb105e77f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.166.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b681b1fc-7cce-473e-81e9-f5f3657cf85b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.237.188.200' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--08453fee-f3b8-449a-95a8-abc0d79710c3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.155.130.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--79e2a4f6-ee8d-4466-8e82-ecb928e87c0d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.71.169.36' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--2d3326c5-c112-4670-b6bd-6de667f4280b\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.47' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4adc0666-89d1-4c67-a3c8-3b02fc351442\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.161.196.11' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--dc1e9fec-6d1e-46a1-902c-dc170424a23f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.47.220.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--2d7480b1-ded5-4466-a1dd-470110eacdba\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '152.3.102.53' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f06d6873-1538-4951-a069-d6af0dd0f8ed\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '84.208.29.17' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--4eaf258d-28d8-48d8-98f8-0d8442ba83fa\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.96.64.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d7e4bba4-485d-4c1f-95c0-55e7d8a015f8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.179.58.83' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5c060dc8-a8cd-4067-985d-52d85ab3f256\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '128.237.157.136' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d397fccb-3dbb-47c3-84ae-aa09f4223eca\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.110.95.1' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d37d0928-c86b-474a-85ef-46e942fff510\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5e6dd813-58bd-454e-9be7-246f3db01999\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.40' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--adb3c6bc-9694-471e-bf1f-0d0a02d70876\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '137.226.34.46' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--3ce88e57-edfb-45fa-81be-ed95d4564316\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '67.198.195.194' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--fbce496c-e9a6-4246-ad12-73b8f5a12a2a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '149.9.1.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--efce84a3-0d17-4ae8-88be-86c86aa80bbd\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.109.122.77' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--20789570-8c07-42c4-8a45-b3ab170cf6ee\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '209.126.116.149' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5c1b2889-6fec-4276-83e0-173938934ba9\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.250.116.136' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bdad2fdb-71bd-49c3-8bf2-50d396fa55d5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '163.172.17.231' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--07fd3e36-5500-4652-935f-23a2955b19f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.114.116.5' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9e70a102-3440-4ad0-ab1d-653144632668\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '66.186.59.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f566b659-ca36-42a9-8ebf-9476e6b651ab\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.204.1.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--40c0d87c-287a-4692-8227-b4976d14a5f0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '212.27.60.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--da56b536-6ac7-44d5-a036-0db986926016\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.236.208.178' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--95f9c0f4-351b-43c9-81da-c5fdcfe4fa6d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '94.125.182.252' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5007db19-0906-4aec-b18b-e0819b3f13de\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.83.20.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9e0667cd-9a83-4e19-b16f-78c3ed33bfc5\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.18.228.34' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f170e9a9-abb8-4919-9902-7a5214e95cde\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.150.28' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a30f883d-956d-4fdd-b926-db81d1893d81\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '178.79.132.147' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8fc0e9c0-4d4d-4c4f-86a7-2f6c07cd69a4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.109.122.67' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--586dc7e8-a08e-4ec2-8365-e2ee897d9ca3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.47.220.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--16c9900c-ce48-4306-b8fa-a2de726be847\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '208.83.20.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--13f73e28-acf7-45b8-a5e9-6c37af914ef2\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '174.143.119.91' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a09c4e42-8843-4c84-a75f-684bf90c5207\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '74.208.174.239' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--76646197-18a2-4513-8465-ccf72734a2e1\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.48' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1169c1db-fd5b-4dcf-b4cb-9c0101ef0ea2\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '212.117.163.190' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cdeb6ddb-5151-49ea-a488-23d806063eff\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.155.130.130' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--05d1ab76-d0a1-4a58-8137-98f5fdbc777c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '90.147.160.69' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--876d7d09-248a-45ad-bcce-d92c73ad5aa3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '89.16.176.16' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ae1f860d-dc4f-4953-9e74-d4d7c389fdef\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '85.188.1.26' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--91bb4edc-f29f-41ba-87d9-d6a81ac8fdba\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '130.239.18.172' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f006d048-f24f-46fa-837b-8f7fa41b43ca\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '8.7.233.233' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--436dcbec-48e2-4dc2-90f0-0876a876a38a\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.66.54' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ff18364d-99f6-4d3d-b267-8401518af42c\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '194.68.45.50' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8b26f167-b0ad-469b-b221-12896e2a0966\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.4.30.33' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--171268fb-f6a7-4085-adf5-2055a461cb93\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.161.254.20' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b56c7a58-71cb-47c2-b615-f4e8a89a0732\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '141.213.238.252' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bf09ce9a-3bb9-47c8-a686-ea1d8e1adbe8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '213.92.8.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--42490e45-7350-4f48-884b-5d1610794a32\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '72.14.191.81' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--c28e91bf-a9a1-4bac-b3f3-cda89c7d28b8\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.16.172.2' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ebe624b5-fb73-420a-a110-c1dc82baa6e4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '69.61.21.115' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ef65505f-4898-4968-82b4-f980e9705d21\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.18.128.86' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b33c35ce-20f6-4fba-912c-dbf7756113f9\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '161.53.178.240' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--b3785934-f4f0-4ce7-b20c-e4384886ec45\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '204.11.244.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--10bbe70c-7bd3-443a-8f2c-1e56cd7a8a54\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.93.242.10' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--bcb54665-3461-43e2-8dbf-6b92c2413f67\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '216.152.67.23' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a407b16b-cf5b-4f3a-a153-ba4dac5ce0e0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '205.188.234.121' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e7e50d3a-802d-41c8-b667-a27d29871098\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '82.96.64.4' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--017dfb8c-84b9-402f-8401-428477af7be4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '80.88.108.18' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--84664128-cc14-480b-8d90-735727fd4b9f\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '154.35.200.44' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--f0aa750f-82cb-47f9-9c74-ace584fdadcb\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.68.221.222' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--9461c426-6404-4b7a-8552-c29dc60c9123\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.197.175.21' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--ba59cc70-03e4-47f4-871e-d40b727267f3\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '78.129.164.123' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--1b48b107-92e2-487f-9eae-3496eb64e125\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '140.211.167.99' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--e9aea5e2-9ef6-40b6-8f12-dff6ccd8eff4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '85.25.43.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--cdbd95b1-17fb-4b2f-89b6-8c0f865b9e4d\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '193.219.128.49' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--afe4738d-bd3c-47de-9cc5-97e248291571\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '195.40.6.37' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5eecb66e-f8fa-4ab9-85e4-599db7790edf\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '173.252.110.27' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--40b6b332-9a5a-42a7-8b25-6e3eb6d371d4\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.229.70.20' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--a16905d7-4452-4e9f-88a3-fc9338ea5116\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '38.99.64.210' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--5fcfa412-514f-43b5-b873-ed8c9b70bbb0\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '192.99.200.113' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--8259bca6-7c9c-4967-b048-a6f13f333f90\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '68.168.184.57' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--96763c7c-4f52-436a-919a-8b09c841f6bd\",\n", - " \"created\": \"2017-10-02T20:40:44.000Z\",\n", - " \"modified\": \"2017-10-02T20:40:44.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '64.237.34.150' ]\",\n", - " \"valid_from\": \"2017-10-02T20:40:44Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"file-hash-watchlist\"\n", " ]\n", "}\n" ] @@ -2406,7 +159,6 @@ "indicators = tc_source.query([f1])\n", "\n", "#for visual purposes\n", - "print(\"indicators: {0}\").format(str(len(indicators)))\n", "for indicator in indicators:\n", " print(indicator)" ] @@ -2457,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -2465,21 +217,14 @@ "output_type": "stream", "text": [ "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d8e1cd37-4a6c-4088-aded-ed79c4ea2caa\",\n", - " \"created\": \"2017-10-02T20:24:03.000Z\",\n", - " \"modified\": \"2017-10-02T20:24:03.000Z\",\n", + " \"type\": \"malware\",\n", + " \"id\": \"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\",\n", + " \"created\": \"2017-01-27T13:49:53.997Z\",\n", + " \"modified\": \"2017-01-27T13:49:53.997Z\",\n", + " \"name\": \"Poison Ivy\",\n", + " \"description\": \"Poison Ivy\",\n", " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-02T20:24:03Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", + " \"remote-access-trojan\"\n", " ]\n", "}\n" ] @@ -2494,7 +239,7 @@ "\n", "# retrieve STIX object by id from TAXII Collection through\n", "# TAXIICollectionStore\n", - "stix_obj2 = tc_source.get(\"indicator--6850d393-36b6-4a67-ad45-f9e4d512c799\")\n", + "stix_obj2 = tc_source.get(\"malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111\")\n", "\n", "print(stix_obj2)" ] @@ -2520,21 +265,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "cti-python-stix2", "language": "python", - "name": "python3" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb new file mode 100644 index 0000000..f98d7b5 --- /dev/null +++ b/docs/guide/ts_support.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Technical Specification Support\n", + "\n", + "### How imports will work\n", + "\n", + "Imports can be used in different ways depending on the use case and support levels.\n", + "\n", + "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "People who want to use an explicit version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20\n", + "\n", + "stix2.v20.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator\n", + "\n", + "Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or even," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2.v20 as stix2\n", + "\n", + "stix2.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The last option makes it easy to update to a new version in one place per file, once you've made the deliberate action to do this.\n", + "\n", + "People who want to use multiple versions in a single file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "stix2.v20.Indicator()\n", + "\n", + "stix2.v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import v20, v21\n", + "\n", + "v20.Indicator()\n", + "v21.Indicator()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or (less preferred):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2.v20 import Indicator as Indicator_v20\n", + "from stix2.v21 import Indicator as Indicator_v21\n", + "\n", + "Indicator_v20()\n", + "Indicator_v21()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How parsing will work\n", + "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", + "\n", + "You can lock your `parse()` method to a specific STIX version by" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import parse\n", + "\n", + "indicator = parse(\"\"\"{\n", + " \"type\": \"indicator\",\n", + " \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n", + " \"created\": \"2017-09-26T23:33:39.829Z\",\n", + " \"modified\": \"2017-09-26T23:33:39.829Z\",\n", + " \"labels\": [\n", + " \"malicious-activity\"\n", + " ],\n", + " \"name\": \"File hash for malware variant\",\n", + " \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n", + " \"valid_from\": \"2017-09-26T23:33:39.829952Z\"\n", + "}\"\"\", version=\"2.0\")\n", + "print(indicator)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Keep in mind that if a 2.1 or higher object is parsed, the operation will fail." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How will custom work\n", + "\n", + "CustomObject, CustomObservable, CustomMarking and CustomExtension must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "\n", + "You can perform this by," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import stix2\n", + "\n", + "# Make my custom observable available in STIX 2.0\n", + "@stix2.v20.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass\n", + "\n", + "\n", + "# Make my custom observable available in STIX 2.1\n", + "@stix2.v21.CustomObservable('x-new-object-type',\n", + " ((\"prop\", stix2.properties.BooleanProperty())))\n", + "class NewObject2(object):\n", + " pass" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/setup.py b/setup.py index e359147..3ed7ba2 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ], - keywords="stix stix2 json cti cyber threat intelligence", - packages=find_packages(), + keywords='stix stix2 json cti cyber threat intelligence', + packages=find_packages(exclude=['*.test']), install_requires=[ 'python-dateutil', 'pytz', @@ -53,7 +53,6 @@ setup( 'simplejson', 'six', 'stix2-patterns', - 'stix2-validator', 'taxii2-client', ], ) diff --git a/stix2/__init__.py b/stix2/__init__.py index 661c247..6fe2a79 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -19,28 +19,10 @@ # flake8: noqa -from . import exceptions -from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, - ExternalReference, GranularMarking, KillChainPhase, - LanguageContent, MarkingDefinition, StatementMarking, - TLPMarking) -from .core import Bundle, _register_type, parse +from .core import Bundle, _collect_stix2_obj_maps, _register_type, parse from .environment import Environment, ObjectFactory from .markings import (add_markings, clear_markings, get_markings, is_marked, remove_markings, set_markings) -from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, - AutonomousSystem, CustomExtension, CustomObservable, - Directory, DomainName, EmailAddress, EmailMessage, - EmailMIMEComponent, File, HTTPRequestExt, ICMPExt, - IPv4Address, IPv6Address, MACAddress, Mutex, - NetworkTraffic, NTFSExt, PDFExt, Process, - RasterImageExt, SocketExt, Software, TCPExt, - UNIXAccountExt, UserAccount, WindowsPEBinaryExt, - WindowsPEOptionalHeaderType, WindowsPESection, - WindowsProcessExt, WindowsRegistryKey, - WindowsRegistryValueType, WindowsServiceExt, - X509Certificate, X509V3ExtenstionsType, - parse_observable) from .patterns import (AndBooleanExpression, AndObservationExpression, BasicObjectPathComponent, EqualityComparisonExpression, FloatConstant, FollowedByObservationExpression, @@ -59,10 +41,6 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, ReferenceObjectPathComponent, RepeatQualifier, StartStopQualifier, StringConstant, TimestampConstant, WithinQualifier) -from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, - Identity, Indicator, IntrusionSet, Location, Malware, Note, - ObservedData, Opinion, Report, ThreatActor, Tool, - Vulnerability) from .sources import CompositeDataSource from .sources.filesystem import (FileSystemSink, FileSystemSource, FileSystemStore) @@ -70,6 +48,10 @@ from .sources.filters import Filter from .sources.memory import MemorySink, MemorySource, MemoryStore from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource, TAXIICollectionStore) -from .sro import Relationship, Sighting from .utils import get_dict, new_version, revoke +from .v21 import * # This import will always be the latest STIX 2.X version from .version import __version__ + +_collect_stix2_obj_maps() + +DEFAULT_VERSION = "2.1" # Default version will always be the latest STIX 2.X version diff --git a/stix2/base.py b/stix2/base.py index 5307393..b0cf6ff 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,7 +40,14 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - return list(self._properties.keys()) + props = set(self._properties.keys()) + custom_props = list(set(self._inner.keys()) - props) + custom_props.sort() + + all_properties = list(self._properties.keys()) + all_properties.extend(custom_props) # Any custom properties to the bottom + + return all_properties def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: diff --git a/stix2/core.py b/stix2/core.py index cd0523e..3c98197 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -1,16 +1,15 @@ -"""STIX 2.0 Objects that are neither SDOs nor SROs.""" +"""STIX 2.X Objects that are neither SDOs nor SROs.""" from collections import OrderedDict +import importlib +import pkgutil + +import stix2 from . import exceptions from .base import _STIXBase -from .common import MarkingDefinition from .properties import IDProperty, ListProperty, Property, TypeProperty -from .sdo import (AttackPattern, Campaign, CourseOfAction, Identity, Indicator, - IntrusionSet, Location, Malware, Note, ObservedData, Opinion, - Report, ThreatActor, Tool, Vulnerability) -from .sro import Relationship, Sighting -from .utils import get_dict +from .utils import get_class_hierarchy_names, get_dict class STIXObjectProperty(Property): @@ -20,6 +19,11 @@ class STIXObjectProperty(Property): super(STIXObjectProperty, self).__init__() def clean(self, value): + # Any STIX Object (SDO, SRO, or Marking Definition) can be added to + # a bundle with no further checks. + if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition') + for x in get_class_hierarchy_names(value)): + return value try: dictified = get_dict(value) except ValueError: @@ -62,40 +66,30 @@ class Bundle(_STIXBase): super(Bundle, self).__init__(**kwargs) -OBJ_MAP = { - 'attack-pattern': AttackPattern, - 'bundle': Bundle, - 'campaign': Campaign, - 'course-of-action': CourseOfAction, - 'identity': Identity, - 'indicator': Indicator, - 'intrusion-set': IntrusionSet, - 'location': Location, - 'malware': Malware, - 'note': Note, - 'marking-definition': MarkingDefinition, - 'observed-data': ObservedData, - 'opinion': Opinion, - 'report': Report, - 'relationship': Relationship, - 'threat-actor': ThreatActor, - 'tool': Tool, - 'sighting': Sighting, - 'vulnerability': Vulnerability, -} +STIX2_OBJ_MAPS = {} -def parse(data, allow_custom=False): +def parse(data, allow_custom=False, version=None): """Deserialize a string or file-like object into a STIX object. Args: data (str, dict, file-like object): The STIX 2 content to be parsed. - allow_custom (bool): Whether to allow custom properties or not. Default: False. + allow_custom (bool): Whether to allow custom properties or not. + Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: An instantiated Python STIX object. """ + if not version: + # Use latest version + v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') + else: + v = 'v' + version.replace('.', '') + + OBJ_MAP = STIX2_OBJ_MAPS[v] obj = get_dict(data) if 'type' not in obj: @@ -108,8 +102,34 @@ def parse(data, allow_custom=False): return obj_class(allow_custom=allow_custom, **obj) -def _register_type(new_type): +def _register_type(new_type, version=None): """Register a custom STIX Object type. + Args: + new_type (class): A class to register in the Object map. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ + if not version: + # Use latest version + v = 'v' + stix2.DEFAULT_VERSION.replace('.', '') + else: + v = 'v' + version.replace('.', '') + + OBJ_MAP = STIX2_OBJ_MAPS[v] OBJ_MAP[new_type._type] = new_type + + +def _collect_stix2_obj_maps(): + """Navigate the package once and retrieve all OBJ_MAP dicts for each v2X + package.""" + if not STIX2_OBJ_MAPS: + top_level_module = importlib.import_module('stix2') + path = top_level_module.__path__ + prefix = str(top_level_module.__name__) + '.' + + for module_loader, name, is_pkg in pkgutil.walk_packages(path=path, + prefix=prefix): + if name.startswith('stix2.v2') and is_pkg: + mod = importlib.import_module(name, str(top_level_module.__name__)) + STIX2_OBJ_MAPS[name.split('.')[-1]] = mod.OBJ_MAP diff --git a/stix2/environment.py b/stix2/environment.py index c4816ee..4919335 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -152,3 +152,22 @@ class Environment(object): def parse(self, *args, **kwargs): return _parse(*args, **kwargs) parse.__doc__ = _parse.__doc__ + + def creator_of(self, obj): + """Retrieve the Identity refered to by the object's `created_by_ref`. + + Args: + obj: The STIX object whose `created_by_ref` property will be looked + up. + + Returns: + The STIX object's creator, or + None, if the object contains no `created_by_ref` property or the + object's creator cannot be found. + + """ + creator_id = obj.get('created_by_ref', '') + if creator_id: + return self.get(creator_id) + else: + return None diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 9d46ba9..1fe9391 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -44,36 +44,40 @@ class DataStore(object): self.source = source self.sink = sink - def get(self, stix_id): + def get(self, stix_id, allow_custom=False): """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the single most recent version of the STIX object specified by the "id". """ - return self.source.get(stix_id) + return self.source.get(stix_id, allow_custom=allow_custom) - def all_versions(self, stix_id): + def all_versions(self, stix_id, allow_custom=False): """Retrieve all versions of a single STIX object by ID. Implement: Translate all_versions() call to the appropriate DataSource call Args: stix_id (str): the id of the STIX object to retrieve. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id) + return self.source.all_versions(stix_id, allow_custom=allow_custom) - def query(self, query): + def query(self, query=None, allow_custom=False): """Retrieve STIX objects matching a set of filters. Implement: Specific data source API calls, processing, @@ -82,6 +86,8 @@ class DataStore(object): Args: query (list): a list of filters (which collectively are the query) to conduct search on. + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -89,15 +95,17 @@ class DataStore(object): """ return self.source.query(query=query) - def add(self, stix_objs): + def add(self, stix_objs, allow_custom=False): """Store STIX objects. Translates add() to the appropriate DataSink call. Args: stix_objs (list): a list of STIX objects + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ - return self.sink.add(stix_objs) + return self.sink.add(stix_objs, allow_custom=allow_custom) class DataSink(object): @@ -111,7 +119,7 @@ class DataSink(object): def __init__(self): self.id = make_id() - def add(self, stix_objs): + def add(self, stix_objs, allow_custom=False): """Store STIX objects. Implement: Specific data sink API calls, processing, @@ -120,6 +128,8 @@ class DataSink(object): Args: stix_objs (list): a list of STIX objects (where each object is a STIX object) + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ raise NotImplementedError() @@ -139,7 +149,7 @@ class DataSource(object): self.id = make_id() self.filters = set() - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False): """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -148,9 +158,10 @@ class DataSource(object): stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the STIX object @@ -158,7 +169,7 @@ class DataSource(object): """ raise NotImplementedError() - def all_versions(self, stix_id, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """ Implement: Similar to get() except returns list of all object versions of the specified "id". In addition, implement the specific data @@ -169,9 +180,10 @@ class DataSource(object): stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -179,7 +191,7 @@ class DataSource(object): """ raise NotImplementedError() - def query(self, query, _composite_filters=None): + def query(self, query=None, _composite_filters=None, allow_custom=False): """ Implement:Implement the specific data source API calls, processing, functionality required for retrieving query from the data source @@ -187,9 +199,10 @@ class DataSource(object): Args: query (list): a list of filters (which collectively are the query) to conduct search on - _composite_filters (set): a set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -224,7 +237,7 @@ class CompositeDataSource(DataSource): super(CompositeDataSource, self).__init__() self.data_sources = [] - def get(self, stix_id, _composite_filters=None): + def get(self, stix_id, _composite_filters=None, allow_custom=False): """Retrieve STIX object by STIX ID Federated retrieve method, iterates through all DataSources @@ -238,10 +251,11 @@ class CompositeDataSource(DataSource): Args: stix_id (str): the id of the STIX object to retrieve. - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to another parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: stix_obj: the STIX object to be returned. @@ -259,20 +273,22 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds in self.data_sources: - data = ds.get(stix_id=stix_id, _composite_filters=all_filters) + data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) if data: all_data.append(data) # remove duplicate versions if len(all_data) > 0: all_data = deduplicate(all_data) + else: + return None # reduce to most recent version stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0] return stix_obj - def all_versions(self, stix_id, _composite_filters=None): + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """Retrieve STIX objects by STIX ID Federated all_versions retrieve method - iterates through all DataSources @@ -283,10 +299,11 @@ class CompositeDataSource(DataSource): Args: stix_id (str): id of the STIX objects to retrieve - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: all_data (list): list of STIX objects that have the specified id @@ -305,7 +322,7 @@ class CompositeDataSource(DataSource): # retrieve STIX objects from all configured data sources for ds in self.data_sources: - data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters) + data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -315,7 +332,7 @@ class CompositeDataSource(DataSource): return all_data - def query(self, query=None, _composite_filters=None): + def query(self, query=None, _composite_filters=None, allow_custom=False): """Retrieve STIX objects that match query Federate the query to all DataSources attached to the @@ -323,10 +340,11 @@ class CompositeDataSource(DataSource): Args: query (list): list of filters to search on - _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: all_data (list): list of STIX objects to be returned @@ -351,7 +369,7 @@ class CompositeDataSource(DataSource): # federate query to all attached data sources, # pass composite filters to id for ds in self.data_sources: - data = ds.query(query=query, _composite_filters=all_filters) + data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 103b882..eb83d8c 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -8,51 +8,51 @@ TODO: import json import os -from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -from stix2.utils import deduplicate +from stix2.utils import deduplicate, get_class_hierarchy_names class FileSystemStore(DataStore): - """FileSystemStore + """Interface to a file directory of STIX objects. - Provides an interface to an file directory of STIX objects. FileSystemStore is a wrapper around a paired FileSystemSink and FileSystemSource. Args: stix_dir (str): path to directory of STIX objects + bundlify (bool): Whether to wrap objects in bundles when saving them. + Default: False. Attributes: source (FileSystemSource): FuleSystemSource - sink (FileSystemSink): FileSystemSink """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, bundlify=False): super(FileSystemStore, self).__init__() self.source = FileSystemSource(stix_dir=stix_dir) - self.sink = FileSystemSink(stix_dir=stix_dir) + self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) class FileSystemSink(DataSink): - """FileSystemSink - - Provides an interface for adding/pushing STIX objects - to file directory of STIX objects. + """Interface for adding/pushing STIX objects to file directory of STIX + objects. Can be paired with a FileSystemSource, together as the two components of a FileSystemStore. Args: - stix_dir (str): path to directory of STIX objects + stix_dir (str): path to directory of STIX objects. + bundlify (bool): Whether to wrap objects in bundles when saving them. + Default: False. """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, bundlify=False): super(FileSystemSink, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.bundlify = bundlify if not os.path.exists(self._stix_dir): raise ValueError("directory path for STIX data does not exist") @@ -61,62 +61,72 @@ class FileSystemSink(DataSink): def stix_dir(self): return self._stix_dir - def add(self, stix_data=None): - """add STIX objects to file directory + def _check_path_and_write(self, stix_obj): + """Write the given STIX object to a file in the STIX file directory. + """ + path = os.path.join(self._stix_dir, stix_obj["type"], stix_obj["id"] + ".json") + + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + if self.bundlify: + stix_obj = Bundle(stix_obj) + + with open(path, "w") as f: + f.write(str(stix_obj)) + + def add(self, stix_data=None, allow_custom=False, version=None): + """Add STIX objects to file directory. Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content - in a STIX object(or list of), dict (or list of), or a STIX 2.0 - json encoded string + in a STIX object (or list of), dict (or list of), or a STIX 2.0 + json encoded string. + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Note: + ``stix_data`` can be a Bundle object, but each object in it will be + saved separately; you will be able to retrieve any of the objects + the Bundle contained, but not the Bundle itself. - TODO: Bundlify STIX content or no? When dumping to disk. """ - def _check_path_and_write(stix_dir, stix_obj): - path = os.path.join(stix_dir, stix_obj["type"], stix_obj["id"] + ".json") - - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - - with open(path, "w") as f: - # Bundle() can take dict or STIX obj as argument - f.write(str(Bundle(stix_obj))) - - if isinstance(stix_data, _STIXBase): + if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition') + for x in get_class_hierarchy_names(stix_data)): # adding python STIX object - _check_path_and_write(self._stix_dir, stix_data) + self._check_path_and_write(stix_data) - elif isinstance(stix_data, dict): + elif isinstance(stix_data, (str, dict)): + stix_data = parse(stix_data, allow_custom, version) if stix_data["type"] == "bundle": - # adding json-formatted Bundle - extracting STIX objects - for stix_obj in stix_data["objects"]: + # extract STIX objects + for stix_obj in stix_data.get("objects", []): self.add(stix_obj) else: # adding json-formatted STIX - _check_path_and_write(self._stix_dir, stix_data) + self._check_path_and_write(stix_data) - elif isinstance(stix_data, str): - # adding json encoded string of STIX content - stix_data = parse(stix_data) - if stix_data["type"] == "bundle": - for stix_obj in stix_data["objects"]: - self.add(stix_obj) - else: - self.add(stix_data) + elif isinstance(stix_data, Bundle): + # recursively add individual STIX objects + for stix_obj in stix_data.get("objects", []): + self.add(stix_obj) elif isinstance(stix_data, list): - # if list, recurse call on individual STIX objects + # recursively add individual STIX objects for stix_obj in stix_data: self.add(stix_obj) else: - raise ValueError("stix_data must be a STIX object(or list of), json formatted STIX(or list of) or a json formatted STIX bundle") + raise TypeError("stix_data must be a STIX object (or list of), " + "JSON formatted STIX (or list of), " + "or a JSON formatted STIX bundle") class FileSystemSource(DataSource): - """FileSystemSource - - Provides an interface for searching/retrieving - STIX objects from a STIX object file directory. + """Interface for searching/retrieving STIX objects from a STIX object file + directory. Can be paired with a FileSystemSink, together as the two components of a FileSystemStore. @@ -136,14 +146,17 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None): - """retrieve STIX object from file directory via STIX ID + def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -153,47 +166,55 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, + allow_custom=allow_custom, version=version) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - stix_obj = parse(stix_obj) else: stix_obj = None return stix_obj - def all_versions(self, stix_id, _composite_filters=None): - """retrieve STIX object from file directory via STIX ID, all versions + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Pass call to get(). Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): of STIX objects that has the supplied STIX ID. The STIX objects are loaded from their json files, parsed into a python STIX objects and then returned - """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None): - """search and retrieve STIX objects based on the complete query + """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, + allow_custom=allow_custom, version=version)] + + def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): + """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters - attached to MemorySource, and any filters passed from a - CompositeDataSource (i.e. _composite_filters) + attached to this FileSystemSource, and any filters passed from a + CompositeDataSource (i.e. _composite_filters). Args: query (list): list of filters to search on - - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): list of STIX objects that matches the supplied @@ -209,7 +230,7 @@ class FileSystemSource(DataSource): if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters @@ -225,14 +246,14 @@ class FileSystemSource(DataSource): file_filters = self._parse_file_filters(query) # establish which subdirectories can be avoided in query - # by decluding as many as possible. A filter with "type" as the field + # by decluding as many as possible. A filter with "type" as the property # means that certain STIX object types can be ruled out, and thus # the corresponding subdirectories as well include_paths = [] declude_paths = [] - if "type" in [filter.field for filter in file_filters]: + if "type" in [filter.property for filter in file_filters]: for filter in file_filters: - if filter.field == "type": + if filter.property == "type": if filter.op == "=": include_paths.append(os.path.join(self._stix_dir, filter.value)) elif filter.op == "!=": @@ -254,14 +275,14 @@ class FileSystemSource(DataSource): # so query will look in all STIX directories that are not # the specified type. Compile correct dir paths for dir in os.listdir(self._stix_dir): - if os.path.abspath(dir) not in declude_paths: - include_paths.append(os.path.abspath(dir)) + if os.path.abspath(os.path.join(self._stix_dir, dir)) not in declude_paths: + include_paths.append(os.path.abspath(os.path.join(self._stix_dir, dir))) # grab stix object ID as well - if present in filters, as # may forgo the loading of STIX content into memory - if "id" in [filter.field for filter in file_filters]: + if "id" in [filter.property for filter in file_filters]: for filter in file_filters: - if filter.field == "id" and filter.op == "=": + if filter.property == "id" and filter.op == "=": id_ = filter.value break else: @@ -273,37 +294,35 @@ class FileSystemSource(DataSource): for path in include_paths: for root, dirs, files in os.walk(path): for file_ in files: - if id_: - if id_ == file_.split(".")[0]: - # since ID is specified in one of filters, can evaluate against filename first without loading - stix_obj = json.load(open(os.path.join(root, file_)))["objects"][0] - # check against other filters, add if match - all_data.extend(apply_common_filters([stix_obj], query)) - else: + if not id_ or id_ == file_.split(".")[0]: # have to load into memory regardless to evaluate other filters - stix_obj = json.load(open(os.path.join(root, file_)))["objects"][0] + stix_obj = json.load(open(os.path.join(root, file_))) + if stix_obj.get('type', '') == 'bundle': + stix_obj = stix_obj['objects'][0] + # check against other filters, add if match all_data.extend(apply_common_filters([stix_obj], query)) all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] return stix_objs def _parse_file_filters(self, query): - """utility method to extract STIX common filters - that can used to possibly speed up querying STIX objects - from the file system + """Extract STIX common filters. - Extracts filters that are for the "id" and "type" field of + Possibly speeds up querying STIX objects from the file system. + + Extracts filters that are for the "id" and "type" property of a STIX object. As the file directory is organized by STIX object type with filenames that are equivalent to the STIX object ID, these filters can be used first to reduce the - search space of a FileSystemStore(or FileSystemSink) + search space of a FileSystemStore (or FileSystemSink). + """ file_filters = set() for filter_ in query: - if filter_.field == "id" or filter_.field == "type": + if filter_.property == "id" or filter_.property == "type": file_filters.add(filter_) return file_filters diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py index 060d2c3..5772112 100644 --- a/stix2/sources/filters.py +++ b/stix2/sources/filters.py @@ -4,29 +4,6 @@ Filters for Python STIX 2.0 DataSources, DataSinks, DataStores """ import collections -import types - -# Currently, only STIX 2.0 common SDO fields (that are not complex objects) -# are supported for filtering on - -"""Supported STIX properties""" -STIX_COMMON_FIELDS = [ - "created", - "created_by_ref", - "external_references.source_name", - "external_references.description", - "external_references.url", - "external_references.hashes", - "external_references.external_id", - "granular_markings.marking_ref", - "granular_markings.selectors", - "id", - "labels", - "modified", - "object_marking_refs", - "revoked", - "type" -] """Supported filter operations""" FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] @@ -34,46 +11,40 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] """Supported filter value types""" FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] -# filter lookup map - STIX 2 common fields -> filter method -STIX_COMMON_FILTERS_MAP = {} + +def _check_filter_components(prop, op, value): + """Check that filter meets minimum validity. + + Note: + Currently can create Filters that are not valid STIX2 object common + properties, as filter.prop value is not checked, only filter.op, + filter value are checked here. They are just ignored when applied + within the DataSource API. For example, a user can add a TAXII Filter, + that is extracted and sent to a TAXII endpoint within TAXIICollection + and not applied locally (within this API). + + """ + if op not in FILTER_OPS: + # check filter operator is supported + raise ValueError("Filter operator '%s' not supported for specified property: '%s'" % (op, prop)) + + if type(value) not in FILTER_VALUE_TYPES: + # check filter value type is supported + raise TypeError("Filter value type '%s' is not supported. The type must be a Python immutable type or dictionary" % type(value)) + + return True -def _check_filter_components(field, op, value): - """check filter meets minimum validity - - Note: Currently can create Filters that are not valid - STIX2 object common properties, as filter.field value - is not checked, only filter.op, filter.value are checked - here. They are just ignored when - applied within the DataSource API. For example, a user - can add a TAXII Filter, that is extracted and sent to - a TAXII endpoint within TAXIICollection and not applied - locally (within this API). - """ - - if op not in FILTER_OPS: - # check filter operator is supported - raise ValueError("Filter operator '%s' not supported for specified field: '%s'" % (op, field)) - - if type(value) not in FILTER_VALUE_TYPES: - # check filter value type is supported - raise TypeError("Filter value type '%s' is not supported. The type must be a python immutable type or dictionary" % type(value)) - - return True - - -class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): +class Filter(collections.namedtuple("Filter", ['property', 'op', 'value'])): """STIX 2 filters that support the querying functionality of STIX 2 DataStores and DataSources. - Initialized like a python tuple + Initialized like a Python tuple. Args: - field (str): filter field name, corresponds to STIX 2 object property - + property (str): filter property name, corresponds to STIX 2 object property op (str): operator of the filter - - value (str): filter field value + value (str): filter property value Example: Filter("id", "=", "malware--0f862b01-99da-47cc-9bdb-db4a86a95bb1") @@ -81,235 +52,110 @@ class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])): """ __slots__ = () - def __new__(cls, field, op, value): + def __new__(cls, prop, op, value): # If value is a list, convert it to a tuple so it is hashable. if isinstance(value, list): value = tuple(value) - _check_filter_components(field, op, value) + _check_filter_components(prop, op, value) - self = super(Filter, cls).__new__(cls, field, op, value) + self = super(Filter, cls).__new__(cls, prop, op, value) return self - @property - def common(self): - """return whether Filter is valid STIX2 Object common property + def _check_property(self, stix_obj_property): + """Check a property of a STIX Object against this filter. - Note: The Filter operator and Filter value type are checked when - the filter is created, thus only leaving the Filter field to be - checked to make sure a valid STIX2 Object common property. + Args: + stix_obj_property: value to check this filter against - Note: Filters that are not valid STIX2 Object common property - Filters are still allowed to be created for extended usage of - Filter. (e.g. TAXII specific filters can be created, which are - then extracted and sent to TAXII endpoint.) + Returns: + True if property matches the filter, + False otherwise. """ - return self.field in STIX_COMMON_FIELDS + if self.op == "=": + return stix_obj_property == self.value + elif self.op == "!=": + return stix_obj_property != self.value + elif self.op == "in": + return stix_obj_property in self.value + elif self.op == ">": + return stix_obj_property > self.value + elif self.op == "<": + return stix_obj_property < self.value + elif self.op == ">=": + return stix_obj_property >= self.value + elif self.op == "<=": + return stix_obj_property <= self.value + else: + raise ValueError("Filter operator: {0} not supported for specified property: {1}".format(self.op, self.property)) def apply_common_filters(stix_objs, query): """Evaluate filters against a set of STIX 2.0 objects. - Supports only STIX 2.0 common property fields + Supports only STIX 2.0 common property properties. Args: stix_objs (list): list of STIX objects to apply the query to query (set): set of filters (combined form complete query) - Returns: - (generator): of STIX objects that successfully evaluate against - the query. + Yields: + STIX objects that successfully evaluate against the query. """ for stix_obj in stix_objs: clean = True for filter_ in query: - if not filter_.common: - # skip filter as it is not a STIX2 Object common property filter - continue - - if "." in filter_.field: - # For properties like granular_markings and external_references - # need to extract the first property from the string. - field = filter_.field.split(".")[0] - else: - field = filter_.field - - if field not in stix_obj.keys(): - # check filter "field" is in STIX object - if cant be - # applied to STIX object, STIX object is discarded - # (i.e. did not make it through the filter) - clean = False - break - - match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj) + match = _check_filter(filter_, stix_obj) if not match: clean = False break - elif match == -1: - raise ValueError("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field)) # if object unmarked after all filters, add it if clean: yield stix_obj -"""Base type filters""" +def _check_filter(filter_, stix_obj): + """Evaluate a single filter against a single STIX 2.0 object. + Args: + filter_ (Filter): filter to match against + stix_obj: STIX object to apply the filter to -def _all_filter(filter_, stix_obj_field): - """all filter operations (for filters whose value type can be applied to any operation type)""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - elif filter_.op == "in": - return stix_obj_field in filter_.value - elif filter_.op == ">": - return stix_obj_field > filter_.value - elif filter_.op == "<": - return stix_obj_field < filter_.value - elif filter_.op == ">=": - return stix_obj_field >= filter_.value - elif filter_.op == "<=": - return stix_obj_field <= filter_.value + Returns: + True if the stix_obj matches the filter, + False if not. + + """ + # For properties like granular_markings and external_references + # need to extract the first property from the string. + prop = filter_.property.split(".")[0] + + if prop not in stix_obj.keys(): + # check filter "property" is in STIX object - if cant be + # applied to STIX object, STIX object is discarded + # (i.e. did not make it through the filter) + return False + + if "." in filter_.property: + # Check embedded properties, from e.g. granular_markings or external_references + sub_property = filter_.property.split(".", 1)[1] + sub_filter = filter_._replace(property=sub_property) + if isinstance(stix_obj[prop], list): + for elem in stix_obj[prop]: + if _check_filter(sub_filter, elem) is True: + return True + return False + else: + return _check_filter(sub_filter, stix_obj[prop]) + elif isinstance(stix_obj[prop], list): + # Check each item in list property to see if it matches + for elem in stix_obj[prop]: + if filter_._check_property(elem) is True: + return True + return False else: - return -1 - - -def _id_filter(filter_, stix_obj_id): - """base STIX id filter""" - if filter_.op == "=": - return stix_obj_id == filter_.value - elif filter_.op == "!=": - return stix_obj_id != filter_.value - else: - return -1 - - -def _boolean_filter(filter_, stix_obj_field): - """base boolean filter""" - if filter_.op == "=": - return stix_obj_field == filter_.value - elif filter_.op == "!=": - return stix_obj_field != filter_.value - else: - return -1 - - -def _string_filter(filter_, stix_obj_field): - """base string filter""" - return _all_filter(filter_, stix_obj_field) - - -def _timestamp_filter(filter_, stix_obj_timestamp): - """base STIX 2 timestamp filter""" - return _all_filter(filter_, stix_obj_timestamp) - - -"""STIX 2.0 Common Property Filters - -The naming of these functions is important as -they are used to index a mapping dictionary from -STIX common field names to these filter functions. - -REQUIRED naming scheme: - "check__filter" - -""" - - -def check_created_filter(filter_, stix_obj): - return _timestamp_filter(filter_, stix_obj["created"]) - - -def check_created_by_ref_filter(filter_, stix_obj): - return _id_filter(filter_, stix_obj["created_by_ref"]) - - -def check_external_references_filter(filter_, stix_obj): - """ - STIX object's can have a list of external references - - external_references properties supported: - external_references.source_name (string) - external_references.description (string) - external_references.url (string) - external_references.external_id (string) - - external_references properties not supported: - external_references.hashes - - """ - for er in stix_obj["external_references"]: - # grab er property name from filter field - filter_field = filter_.field.split(".")[1] - if filter_field in er: - r = _string_filter(filter_, er[filter_field]) - if r: - return r - return False - - -def check_granular_markings_filter(filter_, stix_obj): - """ - STIX object's can have a list of granular marking references - - granular_markings properties: - granular_markings.marking_ref (id) - granular_markings.selectors (string) - - """ - for gm in stix_obj["granular_markings"]: - # grab gm property name from filter field - filter_field = filter_.field.split(".")[1] - - if filter_field == "marking_ref": - return _id_filter(filter_, gm[filter_field]) - - elif filter_field == "selectors": - for selector in gm[filter_field]: - r = _string_filter(filter_, selector) - if r: - return r - return False - - -def check_id_filter(filter_, stix_obj): - return _id_filter(filter_, stix_obj["id"]) - - -def check_labels_filter(filter_, stix_obj): - for label in stix_obj["labels"]: - r = _string_filter(filter_, label) - if r: - return r - return False - - -def check_modified_filter(filter_, stix_obj): - return _timestamp_filter(filter_, stix_obj["modified"]) - - -def check_object_marking_refs_filter(filter_, stix_obj): - for marking_id in stix_obj["object_marking_refs"]: - r = _id_filter(filter_, marking_id) - if r: - return r - return False - - -def check_revoked_filter(filter_, stix_obj): - return _boolean_filter(filter_, stix_obj["revoked"]) - - -def check_type_filter(filter_, stix_obj): - return _string_filter(filter_, stix_obj["type"]) - - -# Create mapping of field names to filter functions -for name, obj in dict(globals()).items(): - if "check_" in name and isinstance(obj, types.FunctionType): - field_name = "_".join(name.split("_")[1:-1]) - STIX_COMMON_FILTERS_MAP[field_name] = obj + # Check if property matches + return filter_._check_property(stix_obj[prop]) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 0d5901e..2d1705d 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,16 +24,18 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None): - """Adds STIX objects to MemoryStore/Sink. +def _add(store, stix_data=None, allow_custom=False): + """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. Recursive function, breaks down STIX Bundles and lists. Args: stix_data (list OR dict OR STIX object): STIX objects to be added - """ + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + """ if isinstance(stix_data, _STIXBase): # adding a python STIX object store._data[stix_data["id"]] = stix_data @@ -41,35 +43,35 @@ def _add(store, stix_data=None): elif isinstance(stix_data, dict): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects - for stix_obj in stix_data["objects"]: - _add(store, stix_obj) + for stix_obj in stix_data.get("objects", []): + _add(store, stix_obj, allow_custom=allow_custom) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data) + stix_data = parse(stix_data, allow_custom=allow_custom) if stix_data["type"] == "bundle": # recurse on each STIX object in bundle - for stix_obj in stix_data: - _add(store, stix_obj) + for stix_obj in stix_data.get("objects", []): + _add(store, stix_obj, allow_custom=allow_custom) else: _add(store, stix_data) elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj) + _add(store, stix_obj, allow_custom=allow_custom) else: - raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") + raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") class MemoryStore(DataStore): - """Provides an interface to an in-memory dictionary - of STIX objects. MemoryStore is a wrapper around a paired - MemorySink and MemorySource + """Interface to an in-memory dictionary of STIX objects. + + MemoryStore is a wrapper around a paired MemorySink and MemorySource. Note: It doesn't make sense to create a MemoryStore by passing in existing MemorySource and MemorySink because there could @@ -77,36 +79,54 @@ class MemoryStore(DataStore): Args: stix_data (list OR dict OR STIX object): STIX content to be added + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects - source (MemorySource): MemorySource - sink (MemorySink): MemorySink """ - - def __init__(self, stix_data=None): + def __init__(self, stix_data=None, allow_custom=False): super(MemoryStore, self).__init__() self._data = {} if stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - self.source = MemorySource(stix_data=self._data, _store=True) - self.sink = MemorySink(stix_data=self._data, _store=True) + self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) + self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) - def save_to_file(self, file_path): - return self.sink.save_to_file(file_path=file_path) + def save_to_file(self, file_path, allow_custom=False): + """Write SITX objects from in-memory dictionary to JSON file, as a STIX + Bundle. - def load_from_file(self, file_path): - return self.source.load_from_file(file_path=file_path) + Args: + file_path (str): file path to write STIX data to + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + + """ + return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) + + def load_from_file(self, file_path, allow_custom=False): + """Load STIX data from JSON file. + + File format is expected to be a single JSON + STIX object or JSON STIX bundle. + + Args: + file_path (str): file path to load STIX data from + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + + """ + return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) class MemorySink(DataSink): - """Provides an interface for adding/pushing STIX objects - to an in-memory dictionary. + """Interface for adding/pushing STIX objects to an in-memory dictionary. Designed to be paired with a MemorySource, together as the two components of a MemoryStore. @@ -114,51 +134,43 @@ class MemorySink(DataSink): Args: stix_data (dict OR list): valid STIX 2.0 content in bundle or a list. - _store (bool): if the MemorySink is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSource. Not user supplied + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects. If apart of a MemoryStore, dict is shared between with a MemorySource - """ - def __init__(self, stix_data=None, _store=False): + """ + def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySink, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - def add(self, stix_data): - """add STIX objects to in-memory dictionary maintained by - the MemorySink (MemoryStore) + def add(self, stix_data, allow_custom=False): + _add(self, stix_data, allow_custom=allow_custom) + add.__doc__ = _add.__doc__ - see "_add()" for args documentation - """ - _add(self, stix_data) - - def save_to_file(self, file_path): - """write SITX objects in in-memory dictionary to json file, as a STIX Bundle - - Args: - file_path (str): file path to write STIX data to - - """ + def save_to_file(self, file_path, allow_custom=False): file_path = os.path.abspath(file_path) if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(self._data.values()))) + f.write(str(Bundle(self._data.values(), allow_custom=allow_custom))) + save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ class MemorySource(DataSource): - """Provides an interface for searching/retrieving - STIX objects from an in-memory dictionary. + """Interface for searching/retrieving STIX objects from an in-memory + dictionary. Designed to be paired with a MemorySink, together as the two components of a MemoryStore. @@ -166,42 +178,44 @@ class MemorySource(DataSource): Args: stix_data (dict OR list OR STIX object): valid STIX 2.0 content in bundle or list. - _store (bool): if the MemorySource is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSink. Not user supplied + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. Attributes: _data (dict): the in-memory dict that holds STIX objects. If apart of a MemoryStore, dict is shared between with a MemorySink - """ - def __init__(self, stix_data=None, _store=False): + """ + def __init__(self, stix_data=None, _store=False, allow_custom=False): super(MemorySource, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) - def get(self, stix_id, _composite_filters=None): - """retrieve STIX object from in-memory dict via STIX ID + def get(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (dict OR STIX object): STIX object that has the supplied ID. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory as they are supplied (either as python dictionary or STIX object), it is returned in the same form as it as added - """ + """ if _composite_filters is None: # if get call is only based on 'id', no need to search, just retrieve from dict try: @@ -213,24 +227,28 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) - # reduce to most recent version - stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] + if all_data: + # reduce to most recent version + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] - return stix_obj + return stix_obj + else: + return None - def all_versions(self, stix_id, _composite_filters=None): - """retrieve STIX objects from in-memory dict via STIX ID, all versions of it + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it Note: Since Memory sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Translate call to get(). Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that has the supplied ID. As the @@ -239,26 +257,27 @@ class MemorySource(DataSource): is returned in the same form as it as added """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] - def query(self, query=None, _composite_filters=None): - """search and retrieve STIX objects based on the complete query + def query(self, query=None, _composite_filters=None, allow_custom=False): + """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters - attached to MemorySource, and any filters passed from a - CompositeDataSource (i.e. _composite_filters) + attached to this MemorySource, and any filters passed from a + CompositeDataSource (i.e. _composite_filters). Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that matches the supplied query. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory as they are supplied (either as python dictionary or STIX object), it - is returned in the same form as it as added + is returned in the same form as it as added. """ if query is None: @@ -267,7 +286,7 @@ class MemorySource(DataSource): if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters @@ -281,15 +300,8 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path): - """load STIX data from json file - - File format is expected to be a single json - STIX object or json STIX bundle - - Args: - file_path (str): file path to load STIX data from - """ + def load_from_file(self, file_path, allow_custom=False): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - _add(self, stix_data) + _add(self, stix_data, allow_custom=allow_custom) + load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 7ecedca..414e27f 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,9 +1,5 @@ """ -Python STIX 2.0 TAXII Source/Sink - -TODO: - Test everything - +Python STIX 2.x TaxiiCollectionStore """ from stix2.base import _STIXBase @@ -21,7 +17,7 @@ class TAXIICollectionStore(DataStore): around a paired TAXIICollectionSink and TAXIICollectionSource. Args: - collection (taxii2.Collection): TAXII Collection instance + collection (taxii2.Collection): TAXII Collection instance """ def __init__(self, collection): super(TAXIICollectionStore, self).__init__() @@ -41,39 +37,40 @@ class TAXIICollectionSink(DataSink): super(TAXIICollectionSink, self).__init__() self.collection = collection - def add(self, stix_data): - """add/push STIX content to TAXII Collection endpoint + def add(self, stix_data, allow_custom=False): + """Add/push STIX content to TAXII Collection endpoint Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0 json encoded string, or list of any of the following + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. """ - if isinstance(stix_data, _STIXBase): # adding python STIX object - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) elif isinstance(stix_data, dict): # adding python dict (of either Bundle or STIX obj) if stix_data["type"] == "bundle": bundle = stix_data else: - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj) + self.add(obj, allow_custom=allow_custom) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data) + stix_data = parse(stix_data, allow_custom=allow_custom) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: - bundle = dict(Bundle(stix_data)) + bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) else: raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") @@ -93,22 +90,22 @@ class TAXIICollectionSource(DataSource): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None): - """retrieve STIX object from local/remote STIX Collection + def get(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from local/remote STIX Collection endpoint. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (STIX object): STIX object that has the supplied STIX ID. The STIX object is received from TAXII has dict, parsed into a python STIX object and then returned - """ # combine all query filters query = set() @@ -124,22 +121,25 @@ class TAXIICollectionSource(DataSource): stix_obj = list(apply_common_filters(stix_objs, query)) if len(stix_obj): - stix_obj = stix_obj[0] - stix_obj = parse(stix_obj) + stix_obj = parse(stix_obj[0], allow_custom=allow_custom) + if stix_obj.id != stix_id: + # check - was added to handle erroneous TAXII servers + stix_obj = None else: stix_obj = None return stix_obj - def all_versions(self, stix_id, _composite_filters=None): - """retrieve STIX object from local/remote TAXII Collection + def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (see query() as all_versions() is just a wrapper) @@ -151,12 +151,18 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) - return all_data + # parse STIX objects from TAXII returned json + all_data = [parse(stix_obj) for stix_obj in all_data] - def query(self, query=None, _composite_filters=None): - """search and retreive STIX objects based on the complete query + # check - was added to handle erroneous TAXII servers + all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] + + return all_data_clean + + def query(self, query=None, _composite_filters=None, allow_custom=False): + """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters attached to MemorySource, and any filters passed from a @@ -164,9 +170,10 @@ class TAXIICollectionSource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. Returns: (list): list of STIX objects that matches the supplied @@ -174,14 +181,13 @@ class TAXIICollectionSource(DataSource): parsed into python STIX objects and then returned. """ - if query is None: query = set() else: if not isinstance(query, list): # make sure dont make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) - query = list(query) + query = [query] query = set(query) # combine all query filters @@ -194,7 +200,7 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters)["objects"] + all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) @@ -203,7 +209,7 @@ class TAXIICollectionSource(DataSource): all_data = list(apply_common_filters(all_data, query)) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] return stix_objs @@ -225,14 +231,13 @@ class TAXIICollectionSource(DataSource): for 'requests.get()'. """ - params = {} for filter_ in query: - if filter_.field in TAXII_FILTERS: - if filter_.field == "added_after": - params[filter_.field] = filter_.value + if filter_.property in TAXII_FILTERS: + if filter_.property == "added_after": + params[filter_.property] = filter_.value else: - taxii_field = "match[%s]" % filter_.field + taxii_field = "match[%s]" % filter_.property params[taxii_field] = filter_.value return params diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json index 5bfb8bb..cb9cfe2 100755 --- a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json @@ -1,16 +1,9 @@ { - "id": "bundle--2ed6ab6a-ca68-414f-8493-e4db8b75dd51", - "objects": [ - { - "created": "2017-05-31T21:30:41.022744Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", - "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", - "modified": "2017-05-31T21:30:41.022744Z", - "name": "Data from Network Shared Drive Mitigation", - "type": "course-of-action" - } - ], - "spec_version": "2.0", - "type": "bundle" + "created": "2017-05-31T21:30:41.022744Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", + "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", + "modified": "2017-05-31T21:30:41.022744Z", + "name": "Data from Network Shared Drive Mitigation", + "type": "course-of-action" } \ No newline at end of file diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index c7c95a8..b1cffd0 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -132,8 +132,9 @@ def test_create_bundle_invalid(indicator, malware, relationship): assert excinfo.value.reason == 'This property may not contain a Bundle object' -def test_parse_bundle(): - bundle = stix2.parse(EXPECTED_BUNDLE) +@pytest.mark.parametrize("version", ["2.0"]) +def test_parse_bundle(version): + bundle = stix2.parse(EXPECTED_BUNDLE, version=version) assert bundle.type == "bundle" assert bundle.id.startswith("bundle--") @@ -158,3 +159,10 @@ def test_parse_unknown_type(): with pytest.raises(stix2.exceptions.ParseError) as excinfo: stix2.parse(unknown) assert str(excinfo.value) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator." + + +def test_stix_object_property(): + prop = stix2.core.STIXObjectProperty() + + identity = stix2.Identity(name="test", identity_class="individual") + assert prop.clean(identity) is identity diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 48529b9..92d5d4c 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -91,6 +91,7 @@ def test_custom_property_in_bundled_object(): bundle = stix2.Bundle(identity, allow_custom=True) assert bundle.objects[0].x_foo == "bar" + assert '"x_foo": "bar"' in str(bundle) @stix2.sdo.CustomObject('x-new-type', [ @@ -483,3 +484,13 @@ def test_parse_observable_with_unregistered_custom_extension(): with pytest.raises(ValueError) as excinfo: stix2.parse_observable(input_str) assert "Can't parse Unknown extension type" in str(excinfo.value) + + +def test_register_custom_object(): + # Not the way to register custom object. + class CustomObject2(object): + _type = 'awesome-object' + + stix2._register_type(CustomObject2) + # Note that we will always check against newest OBJ_MAP. + assert (CustomObject2._type, CustomObject2) in stix2.OBJ_MAP.items() diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 689fe8c..3327ca9 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,17 +1,13 @@ -import os - import pytest from taxii2client import Collection -from stix2 import (Campaign, FileSystemSink, FileSystemSource, FileSystemStore, - Filter, MemorySource, MemoryStore) +from stix2 import Filter, MemorySource from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters from stix2.utils import deduplicate COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' -FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") class MockTAXIIClient(object): @@ -148,28 +144,6 @@ def test_ds_abstract_class_smoke(): ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) -def test_memory_store_smoke(): - # Initialize MemoryStore with dict - ms = MemoryStore(STIX_OBJS1) - - # Add item to sink - ms.add(dict(id="bundle--%s" % make_id(), - objects=STIX_OBJS2, - spec_version="2.0", - type="bundle")) - - resp = ms.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") - assert len(resp) == 1 - - resp = ms.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") - assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" - - query = [Filter('type', '=', 'malware')] - - resp = ms.query(query) - assert len(resp) == 0 - - def test_ds_taxii(collection): ds = taxii.TAXIICollectionSource(collection) assert ds.collection is not None @@ -205,7 +179,7 @@ def test_parse_taxii_filters(): def test_add_get_remove_filter(ds): - # First 3 filters are valid, remaining fields are erroneous in some way + # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ Filter('type', '=', 'malware'), Filter('id', '!=', 'stix object id'), @@ -219,14 +193,14 @@ def test_add_get_remove_filter(ds): with pytest.raises(ValueError) as excinfo: # create Filter that has an operator that is not allowed Filter('modified', '*', 'not supported operator - just place holder') - assert str(excinfo.value) == "Filter operator '*' not supported for specified field: 'modified'" + assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'" with pytest.raises(TypeError) as excinfo: # create Filter that has a value type that is not allowed Filter('created', '=', object()) # On Python 2, the type of object() is `` On Python 3, it's ``. assert str(excinfo.value).startswith("Filter value type") - assert str(excinfo.value).endswith("is not supported. The type must be a python immutable type or dictionary") + assert str(excinfo.value).endswith("is not supported. The type must be a Python immutable type or dictionary") assert len(ds.filters) == 0 @@ -433,7 +407,7 @@ def test_filters4(ds): with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") assert str(excinfo.value) == ("Filter operator '?' not supported " - "for specified field: 'modified'") + "for specified property: 'modified'") def test_filters5(ds): @@ -443,6 +417,52 @@ def test_filters5(ds): assert len(resp) == 1 +def test_filters6(ds): + # Test filtering on non-common property + resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) + assert resp[0]['id'] == STIX_OBJS2[0]['id'] + assert len(resp) == 3 + + +def test_filters7(ds): + # Test filtering on embedded property + stix_objects = list(STIX_OBJS2) + [{ + "type": "observed-data", + "id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-04-06T19:58:16.000Z", + "modified": "2016-04-06T19:58:16.000Z", + "first_observed": "2015-12-21T19:00:00Z", + "last_observed": "2015-12-21T19:00:00Z", + "number_observed": 50, + "objects": { + "0": { + "type": "file", + "hashes": { + "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + }, + "extensions": { + "pdf-ext": { + "version": "1.7", + "document_info_dict": { + "Title": "Sample document", + "Author": "Adobe Systems Incorporated", + "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh", + "Producer": "Acrobat Distiller 3.01 for Power Macintosh", + "CreationDate": "20070412090123-02" + }, + "pdfid0": "DFCE52BD827ECF765649852119D", + "pdfid1": "57A1E0F9ED2AE523E313C" + } + } + } + } + }] + resp = list(apply_common_filters(stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) + assert resp[0]['id'] == stix_objects[3]['id'] + assert len(resp) == 1 + + def test_deduplicate(ds): unique = deduplicate(STIX_OBJS1) @@ -512,207 +532,3 @@ def test_composite_datasource_operations(): # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 - - -def test_filesytem_source(): - # creation - fs_source = FileSystemSource(FS_PATH) - assert fs_source.stix_dir == FS_PATH - - # get object - mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") - assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" - assert mal.name == "Rover" - - # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) - id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") - assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" - assert id_.name == "The MITRE Corporation" - assert id_.type == "identity" - - # query - intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) - assert len(intrusion_sets) == 2 - assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] - assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] - - is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] - assert "DragonOK" in is_1.aliases - assert len(is_1.external_references) == 4 - - # query2 - is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) - assert len(is_2) == 1 - - is_2 = is_2[0] - assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" - assert is_2.type == "attack-pattern" - - -def test_filesystem_sink(): - # creation - fs_sink = FileSystemSink(FS_PATH) - assert fs_sink.stix_dir == FS_PATH - - fs_source = FileSystemSource(FS_PATH) - - # Test all the ways stix objects can be added (via different supplied forms) - - # add python stix object - camp1 = Campaign(name="Hannibal", - objective="Targeting Italian and Spanish Diplomat internet accounts", - aliases=["War Elephant"]) - - fs_sink.add(camp1) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) - - camp1_r = fs_source.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == "Hannibal" - assert "War Elephant" in camp1_r.aliases - - # add stix object dict - camp2 = { - "name": "Aurelius", - "type": "campaign", - "objective": "German and French Intelligence Services", - "aliases": ["Purple Robes"], - "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add(camp2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) - - camp2_r = fs_source.get(camp2["id"]) - assert camp2_r.id == camp2["id"] - assert camp2_r.name == camp2["name"] - assert "Purple Robes" in camp2_r.aliases - - # add stix bundle dict - bund = { - "type": "bundle", - "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", - "spec_version": "2.0", - "objects": [ - { - "name": "Atilla", - "type": "campaign", - "objective": "Bulgarian, Albanian and Romanian Intelligence Services", - "aliases": ["Huns"], - "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - ] - } - - fs_sink.add(bund) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) - - camp3_r = fs_source.get(bund["objects"][0]["id"]) - assert camp3_r.id == bund["objects"][0]["id"] - assert camp3_r.name == bund["objects"][0]["name"] - assert "Huns" in camp3_r.aliases - - # add json-encoded stix obj - camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' - - fs_sink.add(camp4) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") - assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" - assert camp4_r.name == "Ghengis Khan" - - # add json-encoded stix bundle - bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ - ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' - fs_sink.add(bund2) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") - assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" - assert camp5_r.name == "Spartacus" - - # add list of objects - camp6 = Campaign(name="Comanche", - objective="US Midwest manufacturing firms, oil refineries, and businesses", - aliases=["Horse Warrior"]) - - camp7 = { - "name": "Napolean", - "type": "campaign", - "objective": "Central and Eastern Europe military commands and departments", - "aliases": ["The Frenchmen"], - "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" - } - - fs_sink.add([camp6, camp7]) - - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) - - camp6_r = fs_source.get(camp6.id) - assert camp6_r.id == camp6.id - assert "Horse Warrior" in camp6_r.aliases - - camp7_r = fs_source.get(camp7["id"]) - assert camp7_r.id == camp7["id"] - assert "The Frenchmen" in camp7_r.aliases - - # remove all added objects - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) - - # remove campaign dir (that was added in course of testing) - os.rmdir(os.path.join(FS_PATH, "campaign")) - - -def test_filesystem_store(): - # creation - fs_store = FileSystemStore(FS_PATH) - - # get() - coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") - assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" - assert coa.type == "course-of-action" - - # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) - rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] - assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" - assert rel.type == "relationship" - - # query() - tools = fs_store.query([Filter("labels", "in", "tool")]) - assert len(tools) == 2 - assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] - assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] - - # add() - camp1 = Campaign(name="Great Heathen Army", - objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", - aliases=["Ragnar"]) - fs_store.add(camp1) - - camp1_r = fs_store.get(camp1.id) - assert camp1_r.id == camp1.id - assert camp1_r.name == camp1.name - - # remove - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) - - # remove campaign dir - os.rmdir(os.path.join(FS_PATH, "campaign")) diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index 81f2cda..c669a33 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -184,3 +184,35 @@ def test_parse_malware(): assert mal.modified == FAKE_TIME assert mal.labels == ['ransomware'] assert mal.name == "Cryptolocker" + + +def test_created_by(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) + env.add(identity) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + creator = env.creator_of(ind) + assert creator is identity + + +def test_created_by_no_datasource(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(factory=factory) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + with pytest.raises(AttributeError) as excinfo: + env.creator_of(ind) + assert 'Environment has no data source' in str(excinfo.value) + + +def test_created_by_not_found(): + identity = stix2.Identity(**IDENTITY_KWARGS) + factory = stix2.ObjectFactory(created_by_ref=identity.id) + env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) + + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + creator = env.creator_of(ind) + assert creator is None diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py new file mode 100644 index 0000000..7aaa3f5 --- /dev/null +++ b/stix2/test/test_filesystem.py @@ -0,0 +1,377 @@ +import os +import shutil + +import pytest + +from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, + FileSystemSource, FileSystemStore, Filter, properties) + +FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") + + +@pytest.fixture +def fs_store(): + # create + yield FileSystemStore(FS_PATH) + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_source(): + # create + fs = FileSystemSource(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +@pytest.fixture +def fs_sink(): + # create + fs = FileSystemSink(FS_PATH) + assert fs.stix_dir == FS_PATH + yield fs + + # remove campaign dir + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +def test_filesystem_source_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSource('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + +def test_filesystem_sink_nonexistent_folder(): + with pytest.raises(ValueError) as excinfo: + FileSystemSink('nonexistent-folder') + assert "for STIX data does not exist" in str(excinfo) + + +def test_filesytem_source_get_object(fs_source): + # get object + mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" + assert mal.name == "Rover" + + +def test_filesytem_source_get_nonexistent_object(fs_source): + ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38") + assert ind is None + + +def test_filesytem_source_all_versions(fs_source): + # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) + id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") + assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" + assert id_.name == "The MITRE Corporation" + assert id_.type == "identity" + + +def test_filesytem_source_query_single(fs_source): + # query2 + is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) + assert len(is_2) == 1 + + is_2 = is_2[0] + assert is_2.id == "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a" + assert is_2.type == "attack-pattern" + + +def test_filesytem_source_query_multiple(fs_source): + # query + intrusion_sets = fs_source.query([Filter("type", '=', "intrusion-set")]) + assert len(intrusion_sets) == 2 + assert "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064" in [is_.id for is_ in intrusion_sets] + assert "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a" in [is_.id for is_ in intrusion_sets] + + is_1 = [is_ for is_ in intrusion_sets if is_.id == "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a"][0] + assert "DragonOK" in is_1.aliases + assert len(is_1.external_references) == 4 + + +def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): + # add python stix object + camp1 = Campaign(name="Hannibal", + objective="Targeting Italian and Spanish Diplomat internet accounts", + aliases=["War Elephant"]) + + fs_sink.add(camp1) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) + + camp1_r = fs_source.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == "Hannibal" + assert "War Elephant" in camp1_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): + # add stix object dict + camp2 = { + "name": "Aurelius", + "type": "campaign", + "objective": "German and French Intelligence Services", + "aliases": ["Purple Robes"], + "id": "campaign--111111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add(camp2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) + + camp2_r = fs_source.get(camp2["id"]) + assert camp2_r.id == camp2["id"] + assert camp2_r.name == camp2["name"] + assert "Purple Robes" in camp2_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) + + +def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): + # add stix bundle dict + bund = { + "type": "bundle", + "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", + "spec_version": "2.0", + "objects": [ + { + "name": "Atilla", + "type": "campaign", + "objective": "Bulgarian, Albanian and Romanian Intelligence Services", + "aliases": ["Huns"], + "id": "campaign--133111b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + ] + } + + fs_sink.add(bund) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) + + camp3_r = fs_source.get(bund["objects"][0]["id"]) + assert camp3_r.id == bund["objects"][0]["id"] + assert camp3_r.name == bund["objects"][0]["name"] + assert "Huns" in camp3_r.aliases + + os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) + + +def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): + # add json-encoded stix obj + camp4 = '{"type": "campaign", "id":"campaign--144111b6-1112-4fb0-111b-b111107ca70a",'\ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' + + fs_sink.add(camp4) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp4_r = fs_source.get("campaign--144111b6-1112-4fb0-111b-b111107ca70a") + assert camp4_r.id == "campaign--144111b6-1112-4fb0-111b-b111107ca70a" + assert camp4_r.name == "Ghengis Khan" + + os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) + + +def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): + # add json-encoded stix bundle + bund2 = '{"type": "bundle", "id": "bundle--332211b6-1132-4fb0-111b-b111107ca70a",' \ + ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--155155b6-1112-4fb0-111b-b111107ca70a",' \ + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' + fs_sink.add(bund2) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp5_r = fs_source.get("campaign--155155b6-1112-4fb0-111b-b111107ca70a") + assert camp5_r.id == "campaign--155155b6-1112-4fb0-111b-b111107ca70a" + assert camp5_r.name == "Spartacus" + + os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) + + +def test_filesystem_sink_add_objects_list(fs_sink, fs_source): + # add list of objects + camp6 = Campaign(name="Comanche", + objective="US Midwest manufacturing firms, oil refineries, and businesses", + aliases=["Horse Warrior"]) + + camp7 = { + "name": "Napolean", + "type": "campaign", + "objective": "Central and Eastern Europe military commands and departments", + "aliases": ["The Frenchmen"], + "id": "campaign--122818b6-1112-4fb0-111b-b111107ca70a", + "created": "2017-05-31T21:31:53.197755Z" + } + + fs_sink.add([camp6, camp7]) + + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-111b-b111107ca70a" + ".json")) + + camp6_r = fs_source.get(camp6.id) + assert camp6_r.id == camp6.id + assert "Horse Warrior" in camp6_r.aliases + + camp7_r = fs_source.get(camp7["id"]) + assert camp7_r.id == camp7["id"] + assert "The Frenchmen" in camp7_r.aliases + + # remove all added objects + os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) + os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) + + +def test_filesystem_store_get_stored_as_bundle(fs_store): + coa = fs_store.get("course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f") + assert coa.id == "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f" + assert coa.type == "course-of-action" + + +def test_filesystem_store_get_stored_as_object(fs_store): + coa = fs_store.get("course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd") + assert coa.id == "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd" + assert coa.type == "course-of-action" + + +def test_filesystem_store_all_versions(fs_store): + # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) + rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] + assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" + assert rel.type == "relationship" + + +def test_filesystem_store_query(fs_store): + # query() + tools = fs_store.query([Filter("labels", "in", "tool")]) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_query_single_filter(fs_store): + query = Filter("labels", "in", "tool") + tools = fs_store.query(query) + assert len(tools) == 2 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [tool.id for tool in tools] + assert "tool--03342581-f790-4f03-ba41-e82e67392e23" in [tool.id for tool in tools] + + +def test_filesystem_store_empty_query(fs_store): + results = fs_store.query() # returns all + assert len(results) == 26 + assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] + assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] + + +def test_filesystem_store_query_multiple_filters(fs_store): + fs_store.source.filters.add(Filter("labels", "in", "tool")) + tools = fs_store.query(Filter("id", "=", "tool--242f3da3-4425-4d11-8f5c-b842886da966")) + assert len(tools) == 1 + assert tools[0].id == "tool--242f3da3-4425-4d11-8f5c-b842886da966" + + +def test_filesystem_store_query_dont_include_type_folder(fs_store): + results = fs_store.query(Filter("type", "!=", "tool")) + assert len(results) == 24 + + +def test_filesystem_store_add(fs_store): + # add() + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + # remove + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + + +def test_filesystem_store_add_as_bundle(): + fs_store = FileSystemStore(FS_PATH, bundlify=True) + + camp1 = Campaign(name="Great Heathen Army", + objective="Targeting the government of United Kingdom and insitutions affiliated with the Church Of England", + aliases=["Ragnar"]) + fs_store.add(camp1) + + with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: + assert '"type": "bundle"' in bundle_file.read() + + camp1_r = fs_store.get(camp1.id) + assert camp1_r.id == camp1.id + assert camp1_r.name == camp1.name + + shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) + + +def test_filesystem_add_bundle_object(fs_store): + bundle = Bundle() + fs_store.add(bundle) + + +def test_filesystem_store_add_invalid_object(fs_store): + ind = ('campaign', 'campaign--111111b6-1112-4fb0-111b-b111107ca70a') # tuple isn't valid + with pytest.raises(TypeError) as excinfo: + fs_store.add(ind) + assert 'stix_data must be' in str(excinfo.value) + assert 'a STIX object' in str(excinfo.value) + assert 'JSON formatted STIX' in str(excinfo.value) + assert 'JSON formatted STIX bundle' in str(excinfo.value) + + +def test_filesystem_object_with_custom_property(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + fs_store.add(camp, True) + + camp_r = fs_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_object_with_custom_property_in_bundle(fs_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + bundle = Bundle(camp, allow_custom=True) + fs_store.add(bundle, True) + + camp_r = fs_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_filesystem_custom_object(fs_store): + @CustomObject('x-new-obj', [ + ('property1', properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + newobj = NewObj(property1='something') + fs_store.add(newobj, True) + + newobj_r = fs_store.get(newobj.id, True) + assert newobj_r.id == newobj.id + assert newobj_r.property1 == 'something' + + # remove dir + shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py new file mode 100644 index 0000000..0603bf7 --- /dev/null +++ b/stix2/test/test_memory.py @@ -0,0 +1,270 @@ +import pytest + +from stix2 import (Bundle, Campaign, CustomObject, Filter, MemorySource, + MemoryStore, properties) +from stix2.sources import make_id + +IND1 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND2 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND3 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.936Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND4 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND5 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND6 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-31T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND7 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} +IND8 = { + "created": "2017-01-27T13:49:53.935Z", + "id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" +} + +STIX_OBJS2 = [IND6, IND7, IND8] +STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] + + +@pytest.fixture +def mem_store(): + yield MemoryStore(STIX_OBJS1) + + +@pytest.fixture +def mem_source(): + yield MemorySource(STIX_OBJS1) + + +def test_memory_source_get(mem_source): + resp = mem_source.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" + + +def test_memory_source_get_nonexistant_object(mem_source): + resp = mem_source.get("tool--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert resp is None + + +def test_memory_store_all_versions(mem_store): + # Add bundle of items to sink + mem_store.add(dict(id="bundle--%s" % make_id(), + objects=STIX_OBJS2, + spec_version="2.0", + type="bundle")) + + resp = mem_store.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(resp) == 1 # MemoryStore can only store 1 version of each object + + +def test_memory_store_query(mem_store): + query = [Filter('type', '=', 'malware')] + resp = mem_store.query(query) + assert len(resp) == 0 + + +def test_memory_store_query_single_filter(mem_store): + query = Filter('id', '=', 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f') + resp = mem_store.query(query) + assert len(resp) == 1 + + +def test_memory_store_query_empty_query(mem_store): + resp = mem_store.query() + # sort since returned in random order + resp = sorted(resp, key=lambda k: k['id']) + assert len(resp) == 2 + assert resp[0]['id'] == 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f' + assert resp[0]['modified'] == '2017-01-27T13:49:53.935Z' + assert resp[1]['id'] == 'indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f' + assert resp[1]['modified'] == '2017-01-27T13:49:53.936Z' + + +def test_memory_store_query_multiple_filters(mem_store): + mem_store.source.filters.add(Filter('type', '=', 'indicator')) + query = Filter('id', '=', 'indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f') + resp = mem_store.query(query) + assert len(resp) == 1 + + +def test_memory_store_add_stix_object_str(mem_store): + # add stix object string + camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" + camp_name = "Aurelius" + camp_alias = "Purple Robes" + camp = """{ + "name": "%s", + "type": "campaign", + "objective": "German and French Intelligence Services", + "aliases": ["%s"], + "id": "%s", + "created": "2017-05-31T21:31:53.197755Z" + }""" % (camp_name, camp_alias, camp_id) + + mem_store.add(camp) + + camp_r = mem_store.get(camp_id) + assert camp_r["id"] == camp_id + assert camp_r["name"] == camp_name + assert camp_alias in camp_r["aliases"] + + +def test_memory_store_add_stix_bundle_str(mem_store): + # add stix bundle string + camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" + camp_name = "Atilla" + camp_alias = "Huns" + bund = """{ + "type": "bundle", + "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", + "spec_version": "2.0", + "objects": [ + { + "name": "%s", + "type": "campaign", + "objective": "Bulgarian, Albanian and Romanian Intelligence Services", + "aliases": ["%s"], + "id": "%s", + "created": "2017-05-31T21:31:53.197755Z" + } + ] + }""" % (camp_name, camp_alias, camp_id) + + mem_store.add(bund) + + camp_r = mem_store.get(camp_id) + assert camp_r["id"] == camp_id + assert camp_r["name"] == camp_name + assert camp_alias in camp_r["aliases"] + + +def test_memory_store_add_invalid_object(mem_store): + ind = ('indicator', IND1) # tuple isn't valid + with pytest.raises(TypeError) as excinfo: + mem_store.add(ind) + assert 'stix_data must be' in str(excinfo.value) + assert 'a STIX object' in str(excinfo.value) + assert 'JSON formatted STIX' in str(excinfo.value) + assert 'JSON formatted STIX bundle' in str(excinfo.value) + + +def test_memory_store_object_with_custom_property(mem_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + mem_store.add(camp, True) + + camp_r = mem_store.get(camp.id, True) + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_memory_store_object_with_custom_property_in_bundle(mem_store): + camp = Campaign(name="Scipio Africanus", + objective="Defeat the Carthaginians", + x_empire="Roman", + allow_custom=True) + + bundle = Bundle(camp, allow_custom=True) + mem_store.add(bundle, True) + + bundle_r = mem_store.get(bundle.id, True) + camp_r = bundle_r['objects'][0] + assert camp_r.id == camp.id + assert camp_r.x_empire == camp.x_empire + + +def test_memory_store_custom_object(mem_store): + @CustomObject('x-new-obj', [ + ('property1', properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + newobj = NewObj(property1='something') + mem_store.add(newobj, True) + + newobj_r = mem_store.get(newobj.id, True) + assert newobj_r.id == newobj.id + assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py index 7d03b9e..6bd1888 100644 --- a/stix2/test/test_properties.py +++ b/stix2/test/test_properties.py @@ -1,8 +1,7 @@ import pytest -from stix2 import TCPExt +from stix2 import EmailMIMEComponent, ExtensionsProperty, TCPExt from stix2.exceptions import AtLeastOnePropertyError, DictionaryKeyError -from stix2.observables import EmailMIMEComponent, ExtensionsProperty from stix2.properties import (BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty, FloatProperty, HashesProperty, diff --git a/stix2/utils.py b/stix2/utils.py index 32aba72..cb02b15 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -34,7 +34,7 @@ class STIXdatetime(dt.datetime): def deduplicate(stix_obj_list): - """Deduplicate a list of STIX objects to a unique set + """Deduplicate a list of STIX objects to a unique set. Reduces a set of STIX objects to unique set by looking at 'id' and 'modified' fields - as a unique object version @@ -44,7 +44,6 @@ def deduplicate(stix_obj_list): of deduplicate(),that if the "stix_obj_list" argument has multiple STIX objects of the same version, the last object version found in the list will be the one that is returned. - () Args: stix_obj_list (list): list of STIX objects (dicts) @@ -56,7 +55,11 @@ def deduplicate(stix_obj_list): unique_objs = {} for obj in stix_obj_list: - unique_objs[(obj['id'], obj['modified'])] = obj + try: + unique_objs[(obj['id'], obj['modified'])] = obj + except KeyError: + # Handle objects with no `modified` property, e.g. marking-definition + unique_objs[(obj['id'], obj['created'])] = obj return list(unique_objs.values()) @@ -248,3 +251,11 @@ def revoke(data): if data.get("revoked"): raise RevokeError("revoke") return new_version(data, revoked=True) + + +def get_class_hierarchy_names(obj): + """Given an object, return the names of the class hierarchy.""" + names = [] + for cls in obj.__class__.__mro__: + names.append(cls.__name__) + return names diff --git a/stix2/v20/__init__.py b/stix2/v20/__init__.py new file mode 100644 index 0000000..888e1ca --- /dev/null +++ b/stix2/v20/__init__.py @@ -0,0 +1,43 @@ + +# flake8: noqa + +from ..core import Bundle +from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, + ExternalReference, GranularMarking, KillChainPhase, + MarkingDefinition, StatementMarking, TLPMarking) +from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, + AutonomousSystem, CustomExtension, CustomObservable, + Directory, DomainName, EmailAddress, EmailMessage, + EmailMIMEComponent, ExtensionsProperty, File, + HTTPRequestExt, ICMPExt, IPv4Address, IPv6Address, + MACAddress, Mutex, NetworkTraffic, NTFSExt, PDFExt, + Process, RasterImageExt, SocketExt, Software, TCPExt, + UNIXAccountExt, UserAccount, WindowsPEBinaryExt, + WindowsPEOptionalHeaderType, WindowsPESection, + WindowsProcessExt, WindowsRegistryKey, + WindowsRegistryValueType, WindowsServiceExt, + X509Certificate, X509V3ExtenstionsType, + parse_observable) +from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, + Identity, Indicator, IntrusionSet, Malware, ObservedData, + Report, ThreatActor, Tool, Vulnerability) +from .sro import Relationship, Sighting + +OBJ_MAP = { + 'attack-pattern': AttackPattern, + 'bundle': Bundle, + 'campaign': Campaign, + 'course-of-action': CourseOfAction, + 'identity': Identity, + 'indicator': Indicator, + 'intrusion-set': IntrusionSet, + 'malware': Malware, + 'marking-definition': MarkingDefinition, + 'observed-data': ObservedData, + 'report': Report, + 'relationship': Relationship, + 'threat-actor': ThreatActor, + 'tool': Tool, + 'sighting': Sighting, + 'vulnerability': Vulnerability, +} diff --git a/stix2/v20/common.py b/stix2/v20/common.py new file mode 100644 index 0000000..51ded05 --- /dev/null +++ b/stix2/v20/common.py @@ -0,0 +1,189 @@ +"""STIX 2 Common Data Types and Properties.""" + +from collections import OrderedDict + +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (HashesProperty, IDProperty, ListProperty, Property, + ReferenceProperty, SelectorProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW, get_dict + + +class ExternalReference(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('source_name', StringProperty(required=True)), + ('description', StringProperty()), + ('url', StringProperty()), + ('hashes', HashesProperty()), + ('external_id', StringProperty()), + ]) + + def _check_object_constraints(self): + super(ExternalReference, self)._check_object_constraints() + self._check_at_least_one_property(["description", "external_id", "url"]) + + +class KillChainPhase(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('kill_chain_name', StringProperty(required=True)), + ('phase_name', StringProperty(required=True)), + ]) + + +class GranularMarking(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('marking_ref', ReferenceProperty(required=True, type="marking-definition")), + ('selectors', ListProperty(SelectorProperty, required=True)), + ]) + + +class TLPMarking(_STIXBase): + + _type = 'tlp' + _properties = OrderedDict() + _properties.update([ + ('tlp', Property(required=True)) + ]) + + +class StatementMarking(_STIXBase): + + _type = 'statement' + _properties = OrderedDict() + _properties.update([ + ('statement', StringProperty(required=True)) + ]) + + def __init__(self, statement=None, **kwargs): + # Allow statement as positional args. + if statement and not kwargs.get('statement'): + kwargs['statement'] = statement + + super(StatementMarking, self).__init__(**kwargs) + + +class MarkingProperty(Property): + """Represent the marking objects in the ``definition`` property of + marking-definition objects. + """ + + def clean(self, value): + if type(value) in OBJ_MAP_MARKING.values(): + return value + else: + raise ValueError("must be a Statement, TLP Marking or a registered marking.") + + +class MarkingDefinition(_STIXBase, _MarkingsMixin): + + _type = 'marking-definition' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ('definition_type', StringProperty(required=True)), + ('definition', MarkingProperty(required=True)), + ]) + + def __init__(self, **kwargs): + if set(('definition_type', 'definition')).issubset(kwargs.keys()): + # Create correct marking type object + try: + marking_type = OBJ_MAP_MARKING[kwargs['definition_type']] + except KeyError: + raise ValueError("definition_type must be a valid marking type") + + if not isinstance(kwargs['definition'], marking_type): + defn = get_dict(kwargs['definition']) + kwargs['definition'] = marking_type(**defn) + + super(MarkingDefinition, self).__init__(**kwargs) + + +OBJ_MAP_MARKING = { + 'tlp': TLPMarking, + 'statement': StatementMarking, +} + + +def _register_marking(cls): + """Register a custom STIX Marking Definition type. + """ + OBJ_MAP_MARKING[cls._type] = cls + return cls + + +def CustomMarking(type='x-custom-marking', properties=None): + """Custom STIX Marking decorator. + + Example: + >>> @CustomMarking('x-custom-marking', [ + ... ('property1', StringProperty(required=True)), + ... ('property2', IntegerProperty()), + ... ]) + ... class MyNewMarkingObjectType(): + ... pass + + """ + def custom_builder(cls): + + class _Custom(cls, _STIXBase): + _type = type + _properties = OrderedDict() + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update(properties) + + def __init__(self, **kwargs): + _STIXBase.__init__(self, **kwargs) + cls.__init__(self, **kwargs) + + _register_marking(_Custom) + return _Custom + + return custom_builder + + +# TODO: don't allow the creation of any other TLPMarkings than the ones below + +TLP_WHITE = MarkingDefinition( + id="marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="white") +) + +TLP_GREEN = MarkingDefinition( + id="marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="green") +) + +TLP_AMBER = MarkingDefinition( + id="marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="amber") +) + +TLP_RED = MarkingDefinition( + id="marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", + created="2017-01-20T00:00:00.000Z", + definition_type="tlp", + definition=TLPMarking(tlp="red") +) diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py new file mode 100644 index 0000000..a874df9 --- /dev/null +++ b/stix2/v20/observables.py @@ -0,0 +1,948 @@ +"""STIX 2.0 Cyber Observable Objects. + +Embedded observable object types, such as Email MIME Component, which is +embedded in Email Message objects, inherit from ``_STIXBase`` instead of +Observable and do not have a ``_type`` attribute. +""" + +from collections import OrderedDict + +from ..base import _Extension, _Observable, _STIXBase +from ..exceptions import (AtLeastOnePropertyError, DependentPropertiesError, + ParseError) +from ..properties import (BinaryProperty, BooleanProperty, DictionaryProperty, + EmbeddedObjectProperty, EnumProperty, FloatProperty, + HashesProperty, HexProperty, IntegerProperty, + ListProperty, ObjectReferenceProperty, Property, + StringProperty, TimestampProperty, TypeProperty) +from ..utils import get_dict + + +class ObservableProperty(Property): + """Property for holding Cyber Observable Objects. + """ + + def clean(self, value): + try: + dictified = get_dict(value) + except ValueError: + raise ValueError("The observable property must contain a dictionary") + if dictified == {}: + raise ValueError("The observable property must contain a non-empty dictionary") + + valid_refs = dict((k, v['type']) for (k, v) in dictified.items()) + + for key, obj in dictified.items(): + parsed_obj = parse_observable(obj, valid_refs) + dictified[key] = parsed_obj + + return dictified + + +class ExtensionsProperty(DictionaryProperty): + """Property for representing extensions on Observable objects. + """ + + def __init__(self, enclosing_type=None, required=False): + self.enclosing_type = enclosing_type + super(ExtensionsProperty, self).__init__(required) + + def clean(self, value): + try: + dictified = get_dict(value) + except ValueError: + raise ValueError("The extensions property must contain a dictionary") + if dictified == {}: + raise ValueError("The extensions property must contain a non-empty dictionary") + + if self.enclosing_type in EXT_MAP: + specific_type_map = EXT_MAP[self.enclosing_type] + for key, subvalue in dictified.items(): + if key in specific_type_map: + cls = specific_type_map[key] + if type(subvalue) is dict: + dictified[key] = cls(**subvalue) + elif type(subvalue) is cls: + dictified[key] = subvalue + else: + raise ValueError("Cannot determine extension type.") + else: + raise ValueError("The key used in the extensions dictionary is not an extension type name") + else: + raise ValueError("The enclosing type '%s' has no extensions defined" % self.enclosing_type) + return dictified + + +class Artifact(_Observable): + + _type = 'artifact' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('mime_type', StringProperty()), + ('payload_bin', BinaryProperty()), + ('url', StringProperty()), + ('hashes', HashesProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + def _check_object_constraints(self): + super(Artifact, self)._check_object_constraints() + self._check_mutually_exclusive_properties(["payload_bin", "url"]) + self._check_properties_dependency(["hashes"], ["url"]) + + +class AutonomousSystem(_Observable): + + _type = 'autonomous-system' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('number', IntegerProperty(required=True)), + ('name', StringProperty()), + ('rir', StringProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class Directory(_Observable): + + _type = 'directory' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('path', StringProperty(required=True)), + ('path_enc', StringProperty()), + # these are not the created/modified timestamps of the object itself + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types=['file', 'directory']))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class DomainName(_Observable): + + _type = 'domain-name' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'domain-name']))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class EmailAddress(_Observable): + + _type = 'email-addr' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('display_name', StringProperty()), + ('belongs_to_ref', ObjectReferenceProperty(valid_types='user-account')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class EmailMIMEComponent(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('body', StringProperty()), + ('body_raw_ref', ObjectReferenceProperty(valid_types=['artifact', 'file'])), + ('content_type', StringProperty()), + ('content_disposition', StringProperty()), + ]) + + def _check_object_constraints(self): + super(EmailMIMEComponent, self)._check_object_constraints() + self._check_at_least_one_property(["body", "body_raw_ref"]) + + +class EmailMessage(_Observable): + + _type = 'email-message' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('is_multipart', BooleanProperty(required=True)), + ('date', TimestampProperty()), + ('content_type', StringProperty()), + ('from_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('sender_ref', ObjectReferenceProperty(valid_types='email-addr')), + ('to_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('cc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('bcc_refs', ListProperty(ObjectReferenceProperty(valid_types='email-addr'))), + ('subject', StringProperty()), + ('received_lines', ListProperty(StringProperty)), + ('additional_header_fields', DictionaryProperty()), + ('body', StringProperty()), + ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))), + ('raw_email_ref', ObjectReferenceProperty(valid_types='artifact')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + def _check_object_constraints(self): + super(EmailMessage, self)._check_object_constraints() + self._check_properties_dependency(["is_multipart"], ["body_multipart"]) + if self.get("is_multipart") is True and self.get("body"): + # 'body' MAY only be used if is_multipart is false. + raise DependentPropertiesError(self.__class__, [("is_multipart", "body")]) + + +class ArchiveExt(_Extension): + + _type = 'archive-ext' + _properties = OrderedDict() + _properties.update([ + ('contains_refs', ListProperty(ObjectReferenceProperty(valid_types='file'), required=True)), + ('version', StringProperty()), + ('comment', StringProperty()), + ]) + + +class AlternateDataStream(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ]) + + +class NTFSExt(_Extension): + + _type = 'ntfs-ext' + _properties = OrderedDict() + _properties.update([ + ('sid', StringProperty()), + ('alternate_data_streams', ListProperty(EmbeddedObjectProperty(type=AlternateDataStream))), + ]) + + +class PDFExt(_Extension): + + _type = 'pdf-ext' + _properties = OrderedDict() + _properties.update([ + ('version', StringProperty()), + ('is_optimized', BooleanProperty()), + ('document_info_dict', DictionaryProperty()), + ('pdfid0', StringProperty()), + ('pdfid1', StringProperty()), + ]) + + +class RasterImageExt(_Extension): + + _type = 'raster-image-ext' + _properties = OrderedDict() + _properties.update([ + ('image_height', IntegerProperty()), + ('image_weight', IntegerProperty()), + ('bits_per_pixel', IntegerProperty()), + ('image_compression_algorithm', StringProperty()), + ('exif_tags', DictionaryProperty()), + ]) + + +class WindowsPEOptionalHeaderType(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('magic_hex', HexProperty()), + ('major_linker_version', IntegerProperty()), + ('minor_linker_version', IntegerProperty()), + ('size_of_code', IntegerProperty()), + ('size_of_initialized_data', IntegerProperty()), + ('size_of_uninitialized_data', IntegerProperty()), + ('address_of_entry_point', IntegerProperty()), + ('base_of_code', IntegerProperty()), + ('base_of_data', IntegerProperty()), + ('image_base', IntegerProperty()), + ('section_alignment', IntegerProperty()), + ('file_alignment', IntegerProperty()), + ('major_os_version', IntegerProperty()), + ('minor_os_version', IntegerProperty()), + ('major_image_version', IntegerProperty()), + ('minor_image_version', IntegerProperty()), + ('major_subsystem_version', IntegerProperty()), + ('minor_subsystem_version', IntegerProperty()), + ('win32_version_value_hex', HexProperty()), + ('size_of_image', IntegerProperty()), + ('size_of_headers', IntegerProperty()), + ('checksum_hex', HexProperty()), + ('subsystem_hex', HexProperty()), + ('dll_characteristics_hex', HexProperty()), + ('size_of_stack_reserve', IntegerProperty()), + ('size_of_stack_commit', IntegerProperty()), + ('size_of_heap_reserve', IntegerProperty()), + ('size_of_heap_commit', IntegerProperty()), + ('loader_flags_hex', HexProperty()), + ('number_of_rva_and_sizes', IntegerProperty()), + ('hashes', HashesProperty()), + ]) + + def _check_object_constraints(self): + super(WindowsPEOptionalHeaderType, self)._check_object_constraints() + self._check_at_least_one_property() + + +class WindowsPESection(_STIXBase): + + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('size', IntegerProperty()), + ('entropy', FloatProperty()), + ('hashes', HashesProperty()), + ]) + + +class WindowsPEBinaryExt(_Extension): + + _type = 'windows-pebinary-ext' + _properties = OrderedDict() + _properties.update([ + ('pe_type', StringProperty(required=True)), # open_vocab + ('imphash', StringProperty()), + ('machine_hex', HexProperty()), + ('number_of_sections', IntegerProperty()), + ('time_date_stamp', TimestampProperty(precision='second')), + ('pointer_to_symbol_table_hex', HexProperty()), + ('number_of_symbols', IntegerProperty()), + ('size_of_optional_header', IntegerProperty()), + ('characteristics_hex', HexProperty()), + ('file_header_hashes', HashesProperty()), + ('optional_header', EmbeddedObjectProperty(type=WindowsPEOptionalHeaderType)), + ('sections', ListProperty(EmbeddedObjectProperty(type=WindowsPESection))), + ]) + + +class File(_Observable): + + _type = 'file' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('hashes', HashesProperty()), + ('size', IntegerProperty()), + ('name', StringProperty()), + ('name_enc', StringProperty()), + ('magic_number_hex', HexProperty()), + ('mime_type', StringProperty()), + # these are not the created/modified timestamps of the object itself + ('created', TimestampProperty()), + ('modified', TimestampProperty()), + ('accessed', TimestampProperty()), + ('parent_directory_ref', ObjectReferenceProperty(valid_types='directory')), + ('is_encrypted', BooleanProperty()), + ('encryption_algorithm', StringProperty()), + ('decryption_key', StringProperty()), + ('contains_refs', ListProperty(ObjectReferenceProperty)), + ('content_ref', ObjectReferenceProperty(valid_types='artifact')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + def _check_object_constraints(self): + super(File, self)._check_object_constraints() + self._check_properties_dependency(["is_encrypted"], ["encryption_algorithm", "decryption_key"]) + self._check_at_least_one_property(["hashes", "name"]) + + +class IPv4Address(_Observable): + + _type = 'ipv4-addr' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class IPv6Address(_Observable): + + _type = 'ipv6-addr' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('resolves_to_refs', ListProperty(ObjectReferenceProperty(valid_types='mac-addr'))), + ('belongs_to_refs', ListProperty(ObjectReferenceProperty(valid_types='autonomous-system'))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class MACAddress(_Observable): + + _type = 'mac-addr' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class Mutex(_Observable): + + _type = 'mutex' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty(required=True)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class HTTPRequestExt(_Extension): + + _type = 'http-request-ext' + _properties = OrderedDict() + _properties.update([ + ('request_method', StringProperty(required=True)), + ('request_value', StringProperty(required=True)), + ('request_version', StringProperty()), + ('request_header', DictionaryProperty()), + ('message_body_length', IntegerProperty()), + ('message_body_data_ref', ObjectReferenceProperty(valid_types='artifact')), + ]) + + +class ICMPExt(_Extension): + + _type = 'icmp-ext' + _properties = OrderedDict() + _properties.update([ + ('icmp_type_hex', HexProperty(required=True)), + ('icmp_code_hex', HexProperty(required=True)), + ]) + + +class SocketExt(_Extension): + + _type = 'socket-ext' + _properties = OrderedDict() + _properties.update([ + ('address_family', EnumProperty(allowed=[ + "AF_UNSPEC", + "AF_INET", + "AF_IPX", + "AF_APPLETALK", + "AF_NETBIOS", + "AF_INET6", + "AF_IRDA", + "AF_BTH", + ], required=True)), + ('is_blocking', BooleanProperty()), + ('is_listening', BooleanProperty()), + ('protocol_family', EnumProperty(allowed=[ + "PF_INET", + "PF_IPX", + "PF_APPLETALK", + "PF_INET6", + "PF_AX25", + "PF_NETROM" + ])), + ('options', DictionaryProperty()), + ('socket_type', EnumProperty(allowed=[ + "SOCK_STREAM", + "SOCK_DGRAM", + "SOCK_RAW", + "SOCK_RDM", + "SOCK_SEQPACKET", + ])), + ('socket_descriptor', IntegerProperty()), + ('socket_handle', IntegerProperty()), + ]) + + +class TCPExt(_Extension): + + _type = 'tcp-ext' + _properties = OrderedDict() + _properties.update([ + ('src_flags_hex', HexProperty()), + ('dst_flags_hex', HexProperty()), + ]) + + +class NetworkTraffic(_Observable): + + _type = 'network-traffic' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('start', TimestampProperty()), + ('end', TimestampProperty()), + ('is_active', BooleanProperty()), + ('src_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('dst_ref', ObjectReferenceProperty(valid_types=['ipv4-addr', 'ipv6-addr', 'mac-addr', 'domain-name'])), + ('src_port', IntegerProperty()), + ('dst_port', IntegerProperty()), + ('protocols', ListProperty(StringProperty, required=True)), + ('src_byte_count', IntegerProperty()), + ('dst_byte_count', IntegerProperty()), + ('src_packets', IntegerProperty()), + ('dst_packets', IntegerProperty()), + ('ipfix', DictionaryProperty()), + ('src_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('dst_payload_ref', ObjectReferenceProperty(valid_types='artifact')), + ('encapsulates_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('encapsulates_by_ref', ObjectReferenceProperty(valid_types='network-traffic')), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + def _check_object_constraints(self): + super(NetworkTraffic, self)._check_object_constraints() + self._check_at_least_one_property(["src_ref", "dst_ref"]) + + +class WindowsProcessExt(_Extension): + + _type = 'windows-process-ext' + _properties = OrderedDict() + _properties.update([ + ('aslr_enabled', BooleanProperty()), + ('dep_enabled', BooleanProperty()), + ('priority', StringProperty()), + ('owner_sid', StringProperty()), + ('window_title', StringProperty()), + ('startup_info', DictionaryProperty()), + ]) + + +class WindowsServiceExt(_Extension): + + _type = 'windows-service-ext' + _properties = OrderedDict() + _properties.update([ + ('service_name', StringProperty(required=True)), + ('descriptions', ListProperty(StringProperty)), + ('display_name', StringProperty()), + ('group_name', StringProperty()), + ('start_type', EnumProperty(allowed=[ + "SERVICE_AUTO_START", + "SERVICE_BOOT_START", + "SERVICE_DEMAND_START", + "SERVICE_DISABLED", + "SERVICE_SYSTEM_ALERT", + ])), + ('service_dll_refs', ListProperty(ObjectReferenceProperty(valid_types='file'))), + ('service_type', EnumProperty(allowed=[ + "SERVICE_KERNEL_DRIVER", + "SERVICE_FILE_SYSTEM_DRIVER", + "SERVICE_WIN32_OWN_PROCESS", + "SERVICE_WIN32_SHARE_PROCESS", + ])), + ('service_status', EnumProperty(allowed=[ + "SERVICE_CONTINUE_PENDING", + "SERVICE_PAUSE_PENDING", + "SERVICE_PAUSED", + "SERVICE_RUNNING", + "SERVICE_START_PENDING", + "SERVICE_STOP_PENDING", + "SERVICE_STOPPED", + ])), + ]) + + +class Process(_Observable): + + _type = 'process' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('is_hidden', BooleanProperty()), + ('pid', IntegerProperty()), + ('name', StringProperty()), + # this is not the created timestamps of the object itself + ('created', TimestampProperty()), + ('cwd', StringProperty()), + ('arguments', ListProperty(StringProperty)), + ('command_line', StringProperty()), + ('environment_variables', DictionaryProperty()), + ('opened_connection_refs', ListProperty(ObjectReferenceProperty(valid_types='network-traffic'))), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('binary_ref', ObjectReferenceProperty(valid_types='file')), + ('parent_ref', ObjectReferenceProperty(valid_types='process')), + ('child_refs', ListProperty(ObjectReferenceProperty('process'))), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + def _check_object_constraints(self): + # no need to check windows-service-ext, since it has a required property + super(Process, self)._check_object_constraints() + try: + self._check_at_least_one_property() + if "windows-process-ext" in self.get('extensions', {}): + self.extensions["windows-process-ext"]._check_at_least_one_property() + except AtLeastOnePropertyError as enclosing_exc: + if 'extensions' not in self: + raise enclosing_exc + else: + if "windows-process-ext" in self.get('extensions', {}): + self.extensions["windows-process-ext"]._check_at_least_one_property() + + +class Software(_Observable): + + _type = 'software' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('name', StringProperty(required=True)), + ('cpe', StringProperty()), + ('languages', ListProperty(StringProperty)), + ('vendor', StringProperty()), + ('version', StringProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class URL(_Observable): + + _type = 'url' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('value', StringProperty(required=True)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class UNIXAccountExt(_Extension): + + _type = 'unix-account-ext' + _properties = OrderedDict() + _properties.update([ + ('gid', IntegerProperty()), + ('groups', ListProperty(StringProperty)), + ('home_dir', StringProperty()), + ('shell', StringProperty()), + ]) + + +class UserAccount(_Observable): + + _type = 'user-account' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('user_id', StringProperty(required=True)), + ('account_login', StringProperty()), + ('account_type', StringProperty()), # open vocab + ('display_name', StringProperty()), + ('is_service_account', BooleanProperty()), + ('is_privileged', BooleanProperty()), + ('can_escalate_privs', BooleanProperty()), + ('is_disabled', BooleanProperty()), + ('account_created', TimestampProperty()), + ('account_expires', TimestampProperty()), + ('password_last_changed', TimestampProperty()), + ('account_first_login', TimestampProperty()), + ('account_last_login', TimestampProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +class WindowsRegistryValueType(_STIXBase): + + _type = 'windows-registry-value-type' + _properties = OrderedDict() + _properties.update([ + ('name', StringProperty(required=True)), + ('data', StringProperty()), + ('data_type', EnumProperty(allowed=[ + 'REG_NONE', + 'REG_SZ', + 'REG_EXPAND_SZ', + 'REG_BINARY', + 'REG_DWORD', + 'REG_DWORD_BIG_ENDIAN', + 'REG_LINK', + 'REG_MULTI_SZ', + 'REG_RESOURCE_LIST', + 'REG_FULL_RESOURCE_DESCRIPTION', + 'REG_RESOURCE_REQUIREMENTS_LIST', + 'REG_QWORD', + 'REG_INVALID_TYPE', + ])), + ]) + + +class WindowsRegistryKey(_Observable): + + _type = 'windows-registry-key' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('key', StringProperty(required=True)), + ('values', ListProperty(EmbeddedObjectProperty(type=WindowsRegistryValueType))), + # this is not the modified timestamps of the object itself + ('modified', TimestampProperty()), + ('creator_user_ref', ObjectReferenceProperty(valid_types='user-account')), + ('number_of_subkeys', IntegerProperty()), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + @property + def values(self): + # Needed because 'values' is a property on collections.Mapping objects + return self._inner['values'] + + +class X509V3ExtenstionsType(_STIXBase): + + _type = 'x509-v3-extensions-type' + _properties = OrderedDict() + _properties.update([ + ('basic_constraints', StringProperty()), + ('name_constraints', StringProperty()), + ('policy_constraints', StringProperty()), + ('key_usage', StringProperty()), + ('extended_key_usage', StringProperty()), + ('subject_key_identifier', StringProperty()), + ('authority_key_identifier', StringProperty()), + ('subject_alternative_name', StringProperty()), + ('issuer_alternative_name', StringProperty()), + ('subject_directory_attributes', StringProperty()), + ('crl_distribution_points', StringProperty()), + ('inhibit_any_policy', StringProperty()), + ('private_key_usage_period_not_before', TimestampProperty()), + ('private_key_usage_period_not_after', TimestampProperty()), + ('certificate_policies', StringProperty()), + ('policy_mappings', StringProperty()), + ]) + + +class X509Certificate(_Observable): + + _type = 'x509-certificate' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('is_self_signed', BooleanProperty()), + ('hashes', HashesProperty()), + ('version', StringProperty()), + ('serial_number', StringProperty()), + ('signature_algorithm', StringProperty()), + ('issuer', StringProperty()), + ('validity_not_before', TimestampProperty()), + ('validity_not_after', TimestampProperty()), + ('subject', StringProperty()), + ('subject_public_key_algorithm', StringProperty()), + ('subject_public_key_modulus', StringProperty()), + ('subject_public_key_exponent', IntegerProperty()), + ('x509_v3_extensions', EmbeddedObjectProperty(type=X509V3ExtenstionsType)), + ('extensions', ExtensionsProperty(enclosing_type=_type)), + ]) + + +OBJ_MAP_OBSERVABLE = { + 'artifact': Artifact, + 'autonomous-system': AutonomousSystem, + 'directory': Directory, + 'domain-name': DomainName, + 'email-addr': EmailAddress, + 'email-message': EmailMessage, + 'file': File, + 'ipv4-addr': IPv4Address, + 'ipv6-addr': IPv6Address, + 'mac-addr': MACAddress, + 'mutex': Mutex, + 'network-traffic': NetworkTraffic, + 'process': Process, + 'software': Software, + 'url': URL, + 'user-account': UserAccount, + 'windows-registry-key': WindowsRegistryKey, + 'x509-certificate': X509Certificate, +} + + +EXT_MAP = { + 'file': { + 'archive-ext': ArchiveExt, + 'ntfs-ext': NTFSExt, + 'pdf-ext': PDFExt, + 'raster-image-ext': RasterImageExt, + 'windows-pebinary-ext': WindowsPEBinaryExt + }, + 'network-traffic': { + 'http-request-ext': HTTPRequestExt, + 'icmp-ext': ICMPExt, + 'socket-ext': SocketExt, + 'tcp-ext': TCPExt, + }, + 'process': { + 'windows-process-ext': WindowsProcessExt, + 'windows-service-ext': WindowsServiceExt, + }, + 'user-account': { + 'unix-account-ext': UNIXAccountExt, + }, +} + + +def parse_observable(data, _valid_refs=None, allow_custom=False): + """Deserialize a string or file-like object into a STIX Cyber Observable + object. + + Args: + data: The STIX 2 string to be parsed. + _valid_refs: A list of object references valid for the scope of the + object being parsed. Use empty list if no valid refs are present. + allow_custom: Whether to allow custom properties or not. + Default: False. + + Returns: + An instantiated Python STIX Cyber Observable object. + """ + + obj = get_dict(data) + obj['_valid_refs'] = _valid_refs or [] + + if 'type' not in obj: + raise ParseError("Can't parse observable with no 'type' property: %s" % str(obj)) + try: + obj_class = OBJ_MAP_OBSERVABLE[obj['type']] + except KeyError: + raise ParseError("Can't parse unknown observable type '%s'! For custom observables, " + "use the CustomObservable decorator." % obj['type']) + + if 'extensions' in obj and obj['type'] in EXT_MAP: + for name, ext in obj['extensions'].items(): + if name not in EXT_MAP[obj['type']]: + raise ParseError("Can't parse Unknown extension type '%s' for observable type '%s'!" % (name, obj['type'])) + ext_class = EXT_MAP[obj['type']][name] + obj['extensions'][name] = ext_class(allow_custom=allow_custom, **obj['extensions'][name]) + + return obj_class(allow_custom=allow_custom, **obj) + + +def _register_observable(new_observable): + """Register a custom STIX Cyber Observable type. + """ + + OBJ_MAP_OBSERVABLE[new_observable._type] = new_observable + + +def CustomObservable(type='x-custom-observable', properties=None): + """Custom STIX Cyber Observable Object type decorator. + + Example: + >>> @CustomObservable('x-custom-observable', [ + ... ('property1', StringProperty(required=True)), + ... ('property2', IntegerProperty()), + ... ]) + ... class MyNewObservableType(): + ... pass + """ + + def custom_builder(cls): + + class _Custom(cls, _Observable): + + _type = type + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ]) + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + # Check properties ending in "_ref/s" are ObjectReferenceProperties + for prop_name, prop in properties: + if prop_name.endswith('_ref') and not isinstance(prop, ObjectReferenceProperty): + raise ValueError("'%s' is named like an object reference property but " + "is not an ObjectReferenceProperty." % prop_name) + elif (prop_name.endswith('_refs') and (not isinstance(prop, ListProperty) + or not isinstance(prop.contained, ObjectReferenceProperty))): + raise ValueError("'%s' is named like an object reference list property but " + "is not a ListProperty containing ObjectReferenceProperty." % prop_name) + + _properties.update(properties) + + def __init__(self, **kwargs): + _Observable.__init__(self, **kwargs) + try: + cls.__init__(self, **kwargs) + except (AttributeError, TypeError) as e: + # Don't accidentally catch errors raised in a custom __init__() + if ("has no attribute '__init__'" in str(e) or + str(e) == "object.__init__() takes no parameters"): + return + raise e + + _register_observable(_Custom) + return _Custom + + return custom_builder + + +def _register_extension(observable, new_extension): + """Register a custom extension to a STIX Cyber Observable type. + """ + + try: + observable_type = observable._type + except AttributeError: + raise ValueError("Unknown observable type. Custom observables must be " + "created with the @CustomObservable decorator.") + + try: + EXT_MAP[observable_type][new_extension._type] = new_extension + except KeyError: + if observable_type not in OBJ_MAP_OBSERVABLE: + raise ValueError("Unknown observable type '%s'. Custom observables " + "must be created with the @CustomObservable decorator." + % observable_type) + else: + EXT_MAP[observable_type] = {new_extension._type: new_extension} + + +def CustomExtension(observable=None, type='x-custom-observable', properties=None): + """Decorator for custom extensions to STIX Cyber Observables. + """ + + if not observable or not issubclass(observable, _Observable): + raise ValueError("'observable' must be a valid Observable class!") + + def custom_builder(cls): + + class _Custom(cls, _Extension): + + _type = type + _properties = { + 'extensions': ExtensionsProperty(enclosing_type=_type), + } + + if not isinstance(properties, dict) or not properties: + raise ValueError("'properties' must be a dict!") + + _properties.update(properties) + + def __init__(self, **kwargs): + _Extension.__init__(self, **kwargs) + try: + cls.__init__(self, **kwargs) + except (AttributeError, TypeError) as e: + # Don't accidentally catch errors raised in a custom __init__() + if ("has no attribute '__init__'" in str(e) or + str(e) == "object.__init__() takes no parameters"): + return + raise e + + _register_extension(observable, _Custom) + return _Custom + + return custom_builder diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py new file mode 100644 index 0000000..1af0777 --- /dev/null +++ b/stix2/v20/sdo.py @@ -0,0 +1,364 @@ +"""STIX 2.0 Domain Objects""" + +from collections import OrderedDict + +import stix2 + +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, PatternProperty, ReferenceProperty, + StringProperty, TimestampProperty, TypeProperty) +from ..utils import NOW +from .common import ExternalReference, GranularMarking, KillChainPhase +from .observables import ObservableProperty + + +class STIXDomainObject(_STIXBase, _MarkingsMixin): + pass + + +class AttackPattern(STIXDomainObject): + + _type = 'attack-pattern' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Campaign(STIXDomainObject): + + _type = 'campaign' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('objective', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class CourseOfAction(STIXDomainObject): + + _type = 'course-of-action' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Identity(STIXDomainObject): + + _type = 'identity' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('identity_class', StringProperty(required=True)), + ('sectors', ListProperty(StringProperty)), + ('contact_information', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Indicator(STIXDomainObject): + + _type = 'indicator' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty()), + ('description', StringProperty()), + ('pattern', PatternProperty(required=True)), + ('valid_from', TimestampProperty(default=lambda: NOW)), + ('valid_until', TimestampProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class IntrusionSet(STIXDomainObject): + + _type = 'intrusion-set' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('first_seen', TimestampProperty()), + ('last_seen ', TimestampProperty()), + ('goals', ListProperty(StringProperty)), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Malware(STIXDomainObject): + + _type = 'malware' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class ObservedData(STIXDomainObject): + + _type = 'observed-data' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_observed', TimestampProperty(required=True)), + ('last_observed', TimestampProperty(required=True)), + ('number_observed', IntegerProperty(required=True)), + ('objects', ObservableProperty(required=True)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Report(STIXDomainObject): + + _type = 'report' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('published', TimestampProperty(required=True)), + ('object_refs', ListProperty(ReferenceProperty, required=True)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class ThreatActor(STIXDomainObject): + + _type = 'threat-actor' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('aliases', ListProperty(StringProperty)), + ('roles', ListProperty(StringProperty)), + ('goals', ListProperty(StringProperty)), + ('sophistication', StringProperty()), + ('resource_level', StringProperty()), + ('primary_motivation', StringProperty()), + ('secondary_motivations', ListProperty(StringProperty)), + ('personal_motivations', ListProperty(StringProperty)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Tool(STIXDomainObject): + + _type = 'tool' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('kill_chain_phases', ListProperty(KillChainPhase)), + ('tool_version', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty, required=True)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +class Vulnerability(STIXDomainObject): + + _type = 'vulnerability' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('name', StringProperty(required=True)), + ('description', StringProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + +def CustomObject(type='x-custom-type', properties=None): + """Custom STIX Object type decorator. + + Example: + >>> @CustomObject('x-type-name', [ + ... ('property1', StringProperty(required=True)), + ... ('property2', IntegerProperty()), + ... ]) + ... class MyNewObjectType(): + ... pass + + Supply an ``__init__()`` function to add any special validations to the custom + type. Don't call ``super().__init__()`` though - doing so will cause an error. + + Example: + >>> @CustomObject('x-type-name', [ + ... ('property1', StringProperty(required=True)), + ... ('property2', IntegerProperty()), + ... ]) + ... class MyNewObjectType(): + ... def __init__(self, property2=None, **kwargs): + ... if property2 and property2 < 10: + ... raise ValueError("'property2' is too small.") + """ + + def custom_builder(cls): + + class _Custom(cls, STIXDomainObject): + _type = type + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ]) + + if not properties or not isinstance(properties, list): + raise ValueError("Must supply a list, containing tuples. For example, [('property1', IntegerProperty())]") + + _properties.update([x for x in properties if not x[0].startswith("x_")]) + + # This is to follow the general properties structure. + _properties.update([ + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + # Put all custom properties at the bottom, sorted alphabetically. + _properties.update(sorted([x for x in properties if x[0].startswith("x_")], key=lambda x: x[0])) + + def __init__(self, **kwargs): + _STIXBase.__init__(self, **kwargs) + try: + cls.__init__(self, **kwargs) + except (AttributeError, TypeError) as e: + # Don't accidentally catch errors raised in a custom __init__() + if ("has no attribute '__init__'" in str(e) or + str(e) == "object.__init__() takes no parameters"): + return + raise e + + stix2._register_type(_Custom, version="2.0") + return _Custom + + return custom_builder diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py new file mode 100644 index 0000000..7f05f5e --- /dev/null +++ b/stix2/v20/sro.py @@ -0,0 +1,82 @@ +"""STIX 2.0 Relationship Objects.""" + +from collections import OrderedDict + +from ..base import _STIXBase +from ..markings import _MarkingsMixin +from ..properties import (BooleanProperty, IDProperty, IntegerProperty, + ListProperty, ReferenceProperty, StringProperty, + TimestampProperty, TypeProperty) +from ..utils import NOW +from .common import ExternalReference, GranularMarking + + +class STIXRelationshipObject(_STIXBase, _MarkingsMixin): + pass + + +class Relationship(STIXRelationshipObject): + + _type = 'relationship' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('relationship_type', StringProperty(required=True)), + ('description', StringProperty()), + ('source_ref', ReferenceProperty(required=True)), + ('target_ref', ReferenceProperty(required=True)), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + # Explicitly define the first three kwargs to make readable Relationship declarations. + def __init__(self, source_ref=None, relationship_type=None, + target_ref=None, **kwargs): + # Allow (source_ref, relationship_type, target_ref) as positional args. + if source_ref and not kwargs.get('source_ref'): + kwargs['source_ref'] = source_ref + if relationship_type and not kwargs.get('relationship_type'): + kwargs['relationship_type'] = relationship_type + if target_ref and not kwargs.get('target_ref'): + kwargs['target_ref'] = target_ref + + super(Relationship, self).__init__(**kwargs) + + +class Sighting(STIXRelationshipObject): + _type = 'sighting' + _properties = OrderedDict() + _properties.update([ + ('type', TypeProperty(_type)), + ('id', IDProperty(_type)), + ('created_by_ref', ReferenceProperty(type="identity")), + ('created', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('modified', TimestampProperty(default=lambda: NOW, precision='millisecond')), + ('first_seen', TimestampProperty()), + ('last_seen', TimestampProperty()), + ('count', IntegerProperty()), + ('sighting_of_ref', ReferenceProperty(required=True)), + ('observed_data_refs', ListProperty(ReferenceProperty(type="observed-data"))), + ('where_sighted_refs', ListProperty(ReferenceProperty(type="identity"))), + ('summary', BooleanProperty()), + ('revoked', BooleanProperty()), + ('labels', ListProperty(StringProperty)), + ('external_references', ListProperty(ExternalReference)), + ('object_marking_refs', ListProperty(ReferenceProperty(type="marking-definition"))), + ('granular_markings', ListProperty(GranularMarking)), + ]) + + # Explicitly define the first kwargs to make readable Sighting declarations. + def __init__(self, sighting_of_ref=None, **kwargs): + # Allow sighting_of_ref as a positional arg. + if sighting_of_ref and not kwargs.get('sighting_of_ref'): + kwargs['sighting_of_ref'] = sighting_of_ref + + super(Sighting, self).__init__(**kwargs) diff --git a/stix2/v21/__init__.py b/stix2/v21/__init__.py index dad0785..f0d51d9 100644 --- a/stix2/v21/__init__.py +++ b/stix2/v21/__init__.py @@ -9,10 +9,10 @@ from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomExtension, CustomObservable, Directory, DomainName, EmailAddress, EmailMessage, - EmailMIMEComponent, File, HTTPRequestExt, ICMPExt, - IPv4Address, IPv6Address, MACAddress, Mutex, - NetworkTraffic, NTFSExt, PDFExt, Process, - RasterImageExt, SocketExt, Software, TCPExt, + EmailMIMEComponent, ExtensionsProperty, File, + HTTPRequestExt, ICMPExt, IPv4Address, IPv6Address, + MACAddress, Mutex, NetworkTraffic, NTFSExt, PDFExt, + Process, RasterImageExt, SocketExt, Software, TCPExt, UNIXAccountExt, UserAccount, WindowsPEBinaryExt, WindowsPEOptionalHeaderType, WindowsPESection, WindowsProcessExt, WindowsRegistryKey, From b83c5ac7efaf389987edb48de6980b4c9dfb5334 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 09:59:42 -0400 Subject: [PATCH 44/90] Update README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index faacc53..baedfe8 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ For more in-depth documentation, please see `https://stix2.readthedocs.io/ Date: Thu, 2 Nov 2017 21:29:25 -0400 Subject: [PATCH 45/90] ABC for DataSink, DataStore and DataSource. Fixes across the concrete objects --- stix2/sources/__init__.py | 139 ++++++++++++++++-------------------- stix2/sources/filesystem.py | 103 ++++++++++++++++++++++---- stix2/sources/memory.py | 131 ++++++++++++++++++++++++--------- stix2/sources/taxii.py | 101 ++++++++++++++++++++++---- 4 files changed, 336 insertions(+), 138 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 1fe9391..231b777 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -11,8 +11,11 @@ | """ +from abc import ABCMeta, abstractmethod import uuid +from six import with_metaclass + from stix2.utils import deduplicate @@ -20,95 +23,94 @@ def make_id(): return str(uuid.uuid4()) -class DataStore(object): +class DataStore(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataStore. Args: source (DataSource): An existing DataSource to use as this DataStore's DataSource component - sink (DataSink): An existing DataSink to use as this DataStore's DataSink component Attributes: id (str): A unique UUIDv4 to identify this DataStore. - source (DataSource): An object that implements DataSource class. - sink (DataSink): An object that implements DataSink class. """ def __init__(self, source=None, sink=None): + super(DataStore, self).__init__() self.id = make_id() self.source = source self.sink = sink - def get(self, stix_id, allow_custom=False): + @abstractmethod + def get(self, stix_id): # pragma: no cover """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the single most recent version of the STIX object specified by the "id". """ - return self.source.get(stix_id, allow_custom=allow_custom) + return NotImplementedError() - def all_versions(self, stix_id, allow_custom=False): + @abstractmethod + def all_versions(self, stix_id): # pragma: no cover """Retrieve all versions of a single STIX object by ID. - Implement: Translate all_versions() call to the appropriate DataSource call + Implement: Define a function that performs any custom behavior before + calling the associated DataSource all_versions() method. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id, allow_custom=allow_custom) + return NotImplementedError() - def query(self, query=None, allow_custom=False): + @abstractmethod + def query(self, query=None): # pragma: no cover """Retrieve STIX objects matching a set of filters. Implement: Specific data source API calls, processing, functionality required for retrieving query from the data source. + Define custom behavior before calling the associated DataSource query() + Args: query (list): a list of filters (which collectively are the query) to conduct search on. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.query(query=query) + return NotImplementedError() - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + @abstractmethod + def add(self, stix_objs): # pragma: no cover + """Method for storing STIX objects. - Translates add() to the appropriate DataSink call. + Define custom behavior before storing STIX objects using the associated + DataSink. Translates add() to the appropriate DataSink call. Args: stix_objs (list): a list of STIX objects - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + """ - return self.sink.add(stix_objs, allow_custom=allow_custom) + return NotImplementedError() -class DataSink(object): +class DataSink(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSink. @@ -117,10 +119,12 @@ class DataSink(object): """ def __init__(self): + super(DataSink, self).__init__() self.id = make_id() - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + @abstractmethod + def add(self, stix_objs): # pragma: no cover + """Method for storing STIX objects. Implement: Specific data sink API calls, processing, functionality required for adding data to the sink @@ -128,28 +132,27 @@ class DataSink(object): Args: stix_objs (list): a list of STIX objects (where each object is a STIX object) - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. """ raise NotImplementedError() -class DataSource(object): +class DataSource(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSource. Attributes: id (str): A unique UUIDv4 to identify this DataSource. - - _filters (set): A collection of filters attached to this DataSource. + filters (set): A collection of filters attached to this DataSource. """ def __init__(self): + super(DataSource, self).__init__() self.id = make_id() self.filters = set() - def get(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def get(self, stix_id): # pragma: no cover """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -158,10 +161,6 @@ class DataSource(object): stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the STIX object @@ -169,10 +168,11 @@ class DataSource(object): """ raise NotImplementedError() - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def all_versions(self, stix_id): # pragma: no cover """ - Implement: Similar to get() except returns list of all object versions of - the specified "id". In addition, implement the specific data + Implement: Similar to get() except returns list of all object versions + of the specified "id". In addition, implement the specific data source API calls, processing, functionality required for retrieving data from the data source. @@ -180,10 +180,6 @@ class DataSource(object): stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -191,18 +187,15 @@ class DataSource(object): """ raise NotImplementedError() - def query(self, query=None, _composite_filters=None, allow_custom=False): + @abstractmethod + def query(self, query=None): # pragma: no cover """ - Implement:Implement the specific data source API calls, processing, + Implement: The specific data source API calls, processing, functionality required for retrieving query from the data source Args: query (list): a list of filters (which collectively are the query) - to conduct search on - _composite_filters (set): a set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to conduct search on. Returns: stix_objs (list): a list of STIX objects @@ -224,7 +217,7 @@ class CompositeDataSource(DataSource): Attributes: - data_sources (dict): A dictionary of DataSource objects; to be + data_sources (list): A dictionary of DataSource objects; to be controlled and used by the Data Source Controller object. """ @@ -237,7 +230,7 @@ class CompositeDataSource(DataSource): super(CompositeDataSource, self).__init__() self.data_sources = [] - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object by STIX ID Federated retrieve method, iterates through all DataSources @@ -253,9 +246,7 @@ class CompositeDataSource(DataSource): stix_id (str): the id of the STIX object to retrieve. _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached - to another parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to another parent CompositeDataSource), not user supplied. Returns: stix_obj: the STIX object to be returned. @@ -273,7 +264,7 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds in self.data_sources: - data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.get(stix_id=stix_id, _composite_filters=all_filters) if data: all_data.append(data) @@ -288,22 +279,20 @@ class CompositeDataSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects by STIX ID + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve all versions of a STIX object by STIX ID. - Federated all_versions retrieve method - iterates through all DataSources - defined in "data_sources" + Federated all_versions retrieve method - iterates through all + DataSources defined in "data_sources". A composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + each configured data source, pushing filtering to them to handle. Args: - stix_id (str): id of the STIX objects to retrieve + stix_id (str): id of the STIX objects to retrieve. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects that have the specified id @@ -322,7 +311,7 @@ class CompositeDataSource(DataSource): # retrieve STIX objects from all configured data sources for ds in self.data_sources: - data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -332,19 +321,17 @@ class CompositeDataSource(DataSource): return all_data - def query(self, query=None, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects that match query + def query(self, query=None, _composite_filters=None): + """Retrieve STIX objects that match a query. Federate the query to all DataSources attached to the Composite Data Source. Args: - query (list): list of filters to search on + query (list): list of filters to search on. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects to be returned @@ -354,7 +341,7 @@ class CompositeDataSource(DataSource): raise AttributeError('CompositeDataSource has no data sources') if not query: - # dont mess with the query (i.e. convert to a set, as thats done + # don't mess with the query (i.e. convert to a set, as that's done # within the specific DataSources that are called) query = [] @@ -369,7 +356,7 @@ class CompositeDataSource(DataSource): # federate query to all attached data sources, # pass composite filters to id for ds in self.data_sources: - data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.query(query=query, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index eb83d8c..4287cbf 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -26,7 +26,7 @@ class FileSystemStore(DataStore): Default: False. Attributes: - source (FileSystemSource): FuleSystemSource + source (FileSystemSource): FileSystemSource sink (FileSystemSink): FileSystemSink """ @@ -35,6 +35,85 @@ class FileSystemStore(DataStore): self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve all versions of a single STIX object by ID. + + Implement: Translate all_versions() call to the appropriate DataSource + call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + """Retrieve STIX objects matching a set of filters. + + Implement: Specific data source API calls, processing, + functionality required for retrieving query from the data source. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translates add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + class FileSystemSink(DataSink): """Interface for adding/pushing STIX objects to file directory of STIX @@ -99,11 +178,11 @@ class FileSystemSink(DataSink): self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): - stix_data = parse(stix_data, allow_custom, version) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # extract STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: # adding json-formatted STIX self._check_path_and_write(stix_data) @@ -111,12 +190,12 @@ class FileSystemSink(DataSink): elif isinstance(stix_data, Bundle): # recursively add individual STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects for stix_obj in stix_data: - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), " @@ -146,7 +225,7 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: @@ -166,8 +245,7 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version) + all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -176,7 +254,7 @@ class FileSystemSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions @@ -197,10 +275,9 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version)] + return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -305,7 +382,7 @@ class FileSystemSource(DataSource): all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2d1705d..0179c45 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,7 +24,7 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None, allow_custom=False): +def _add(store, stix_data=None, allow_custom=False, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. @@ -34,6 +34,8 @@ def _add(store, stix_data=None, allow_custom=False): stix_data (list OR dict OR STIX object): STIX objects to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -44,25 +46,25 @@ def _add(store, stix_data=None, allow_custom=False): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # recurse on each STIX object in bundle for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: - _add(store, stix_data) + _add(store, stix_data, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") @@ -81,6 +83,8 @@ class MemoryStore(DataStore): stix_data (list OR dict OR STIX object): STIX content to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Attributes: _data (dict): the in-memory dict that holds STIX objects @@ -88,15 +92,15 @@ class MemoryStore(DataStore): sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None): super(MemoryStore, self).__init__() self._data = {} if stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) - self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) + self.source = MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + self.sink = MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) def save_to_file(self, file_path, allow_custom=False): """Write SITX objects from in-memory dictionary to JSON file, as a STIX @@ -110,7 +114,7 @@ class MemoryStore(DataStore): """ return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, file_path, allow_custom=False, version=None): """Load STIX data from JSON file. File format is expected to be a single JSON @@ -120,9 +124,72 @@ class MemoryStore(DataStore): file_path (str): file path to load STIX data from allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ - return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) + return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) + + def get(self, stix_id, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, _composite_filters=_composite_filters) + + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve all versions of a single STIX object by ID. + + Translate all_versions() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id, _composite_filters=_composite_filters) + + def query(self, query=None, _composite_filters=None): + """Retrieve STIX objects matching a set of filters. + + Translates query() to appropriate DataStore call. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query, _composite_filters=_composite_filters) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translates add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) class MemorySink(DataSink): @@ -146,17 +213,17 @@ class MemorySink(DataSink): a MemorySource """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySink, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def add(self, stix_data, allow_custom=False): - _add(self, stix_data, allow_custom=allow_custom) + def add(self, stix_data, allow_custom=False, version=None): + _add(self, stix_data, allow_custom=allow_custom, version=version) add.__doc__ = _add.__doc__ def save_to_file(self, file_path, allow_custom=False): @@ -190,24 +257,22 @@ class MemorySource(DataSource): a MemorySink """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySource, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (dict OR STIX object): STIX object that has the supplied @@ -227,7 +292,7 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, _composite_filters=_composite_filters) if all_data: # reduce to most recent version @@ -237,7 +302,7 @@ class MemorySource(DataSource): else: return None - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, _composite_filters=None): """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it Note: Since Memory sources/sinks don't handle multiple versions of a @@ -245,10 +310,8 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that has the supplied ID. As the @@ -257,9 +320,9 @@ class MemorySource(DataSource): is returned in the same form as it as added """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -268,10 +331,8 @@ class MemorySource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that matches the supplied @@ -284,7 +345,7 @@ class MemorySource(DataSource): query = set() else: if not isinstance(query, list): - # make sure dont make set from a Filter object, + # make sure don't make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) query = [query] query = set(query) @@ -300,8 +361,8 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, file_path, allow_custom=False, version=None): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 414e27f..3fd5eff 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,5 +1,5 @@ """ -Python STIX 2.x TaxiiCollectionStore +Python STIX 2.x TAXIICollectionStore """ from stix2.base import _STIXBase @@ -24,6 +24,71 @@ class TAXIICollectionStore(DataStore): self.source = TAXIICollectionSource(collection) self.sink = TAXIICollectionSink(collection) + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def all_versions(self, stix_id): + """Retrieve all versions of a single STIX object by ID. + + Translate all_versions() to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id) + + def query(self, query=None): + """Retrieve STIX objects matching a set of filters. + + Translate query() to the appropriate DataSource call. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translate add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + class TAXIICollectionSink(DataSink): """Provides an interface for pushing STIX objects to a local/remote @@ -37,7 +102,7 @@ class TAXIICollectionSink(DataSink): super(TAXIICollectionSink, self).__init__() self.collection = collection - def add(self, stix_data, allow_custom=False): + def add(self, stix_data, allow_custom=False, version=None): """Add/push STIX content to TAXII Collection endpoint Args: @@ -46,6 +111,8 @@ class TAXIICollectionSink(DataSink): json encoded string, or list of any of the following allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -62,11 +129,11 @@ class TAXIICollectionSink(DataSink): elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj, allow_custom=allow_custom) + self.add(obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: @@ -90,16 +157,18 @@ class TAXIICollectionSource(DataSource): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): """Retrieve STIX object from local/remote STIX Collection endpoint. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -121,7 +190,7 @@ class TAXIICollectionSource(DataSource): stix_obj = list(apply_common_filters(stix_objs, query)) if len(stix_obj): - stix_obj = parse(stix_obj[0], allow_custom=allow_custom) + stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) if stix_obj.id != stix_id: # check - was added to handle erroneous TAXII servers stix_obj = None @@ -130,16 +199,18 @@ class TAXIICollectionSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (see query() as all_versions() is just a wrapper) @@ -154,14 +225,14 @@ class TAXIICollectionSource(DataSource): all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) # parse STIX objects from TAXII returned json - all_data = [parse(stix_obj) for stix_obj in all_data] + all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -170,10 +241,12 @@ class TAXIICollectionSource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): list of STIX objects that matches the supplied @@ -200,7 +273,7 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) @@ -209,7 +282,7 @@ class TAXIICollectionSource(DataSource): all_data = list(apply_common_filters(all_data, query)) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs From cd6c7982afc5c430ca7302535904d26279b396b7 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 21:31:56 -0400 Subject: [PATCH 46/90] Update tests related to Datastores --- stix2/test/test_data_sources.py | 58 ++++++++++++++------------------- stix2/test/test_filesystem.py | 6 ++-- stix2/test/test_memory.py | 6 ++-- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 3327ca9..d798a0c 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,7 @@ import pytest from taxii2client import Collection -from stix2 import Filter, MemorySource +from stix2 import Filter, MemorySink, MemorySource from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters @@ -20,11 +20,6 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) -@pytest.fixture -def ds(): - return DataSource() - - IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -127,21 +122,17 @@ STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] def test_ds_abstract_class_smoke(): - ds1 = DataSource() - ds2 = DataSink() - ds3 = DataStore(source=ds1, sink=ds2) + with pytest.raises(TypeError): + DataStore() - with pytest.raises(NotImplementedError): - ds3.add(None) + with pytest.raises(TypeError): + DataStore.get() - with pytest.raises(NotImplementedError): - ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + with pytest.raises(TypeError): + DataSource() - with pytest.raises(NotImplementedError): - ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") - - with pytest.raises(NotImplementedError): - ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + with pytest.raises(TypeError): + DataSink() def test_ds_taxii(collection): @@ -177,7 +168,8 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -def test_add_get_remove_filter(ds): +def test_add_get_remove_filter(): + ds = taxii.TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ @@ -226,7 +218,7 @@ def test_add_get_remove_filter(ds): ds.filters.update(valid_filters) -def test_apply_common_filters(ds): +def test_apply_common_filters(): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -374,35 +366,35 @@ def test_apply_common_filters(ds): assert len(resp) == 0 -def test_filters0(ds): +def test_filters0(): # "Return any object modified before 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters1(ds): +def test_filters1(): # "Return any object modified after 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters2(ds): +def test_filters2(): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters3(ds): +def test_filters3(): # "Return any object modified before or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters4(ds): +def test_filters4(): # Assert invalid Filter cannot be created with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") @@ -410,21 +402,21 @@ def test_filters4(ds): "for specified property: 'modified'") -def test_filters5(ds): +def test_filters5(): # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters6(ds): +def test_filters6(): # Test filtering on non-common property resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters7(ds): +def test_filters7(): # Test filtering on embedded property stix_objects = list(STIX_OBJS2) + [{ "type": "observed-data", @@ -463,7 +455,7 @@ def test_filters7(ds): assert len(resp) == 1 -def test_deduplicate(ds): +def test_deduplicate(): unique = deduplicate(STIX_OBJS1) # Only 3 objects are unique @@ -483,14 +475,14 @@ def test_deduplicate(ds): def test_add_remove_composite_datasource(): cds = CompositeDataSource() - ds1 = DataSource() - ds2 = DataSource() - ds3 = DataSink() + ds1 = MemorySource() + ds2 = MemorySource() + ds3 = MemorySink() with pytest.raises(TypeError) as excinfo: cds.add_data_sources([ds1, ds2, ds1, ds3]) assert str(excinfo.value) == ("DataSource (to be added) is not of type " - "stix2.DataSource. DataSource type is ''") + "stix2.DataSource. DataSource type is ''") cds.add_data_sources([ds1, ds2, ds1]) diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 7aaa3f5..1610615 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -340,7 +340,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -354,7 +354,7 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): bundle = Bundle(camp, allow_custom=True) fs_store.add(bundle, True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -369,7 +369,7 @@ def test_filesystem_custom_object(fs_store): newobj = NewObj(property1='something') fs_store.add(newobj, True) - newobj_r = fs_store.get(newobj.id, True) + newobj_r = fs_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index 0603bf7..fbdcaf7 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -235,7 +235,7 @@ def test_memory_store_object_with_custom_property(mem_store): mem_store.add(camp, True) - camp_r = mem_store.get(camp.id, True) + camp_r = mem_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -249,7 +249,7 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store): bundle = Bundle(camp, allow_custom=True) mem_store.add(bundle, True) - bundle_r = mem_store.get(bundle.id, True) + bundle_r = mem_store.get(bundle.id) camp_r = bundle_r['objects'][0] assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -265,6 +265,6 @@ def test_memory_store_custom_object(mem_store): newobj = NewObj(property1='something') mem_store.add(newobj, True) - newobj_r = mem_store.get(newobj.id, True) + newobj_r = mem_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' From 8c2af813b29b68b253075f4aec082379318732ad Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 21:56:18 -0400 Subject: [PATCH 47/90] Define CustomProperty. Make sure to update _properties dict when allow_custom=True --- stix2/base.py | 16 ++++++++-------- stix2/properties.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index b0cf6ff..b26f044 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,14 +40,7 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - props = set(self._properties.keys()) - custom_props = list(set(self._inner.keys()) - props) - custom_props.sort() - - all_properties = list(self._properties.keys()) - all_properties.extend(custom_props) # Any custom properties to the bottom - - return all_properties + return list(self._properties.keys()) def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: @@ -109,6 +102,13 @@ class _STIXBase(collections.Mapping): extra_kwargs = list(set(kwargs) - set(cls._properties)) if extra_kwargs: raise ExtraPropertiesError(cls, extra_kwargs) + else: + from .properties import CustomProperty + + # The custom properties will get added to the bottom. + # Matched with a CustomProperty. + extra_kwargs = list(set(kwargs) - set(cls._properties)) + self._properties.update([(x, CustomProperty()) for x in extra_kwargs]) # Remove any keyword arguments whose value is None setting_kwargs = {} diff --git a/stix2/properties.py b/stix2/properties.py index ca7f04c..5b480ac 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -393,3 +393,15 @@ class PatternProperty(StringProperty): raise ValueError(str(errors[0])) return self.string_type(value) + + +class CustomProperty(Property): + """ + The custom property class can be used as a base to extend further + functionality of a custom property. + + Note: + This class is used internally to signal the use of any custom property + that is parsed by any object or `parse()` method and allow_custom=True. + """ + pass From d6c14139f3e94cce76e5abad0bad8244d00d418f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 08:02:32 -0400 Subject: [PATCH 48/90] Extend object serialization options for _STIXBase --- stix2/base.py | 42 +++++++++++++++++++++++++------- stix2/test/test_bundle.py | 51 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index b26f044..d7c7553 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -153,15 +153,7 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - properties = self.object_properties() - - def sort_by(element): - return find_property_index(self, properties, element) - - # separators kwarg -> don't include spaces after commas. - return json.dumps(self, indent=4, cls=STIXJSONEncoder, - item_sort_key=sort_by, - separators=(",", ": ")) + return self.serialize(pretty=True) def __repr__(self): props = [(k, self[k]) for k in self.object_properties() if self.get(k)] @@ -185,6 +177,38 @@ class _STIXBase(collections.Mapping): def revoke(self): return _revoke(self) + def serialize(self, pretty=False, **kwargs): + """ + Serialize a STIX object. + + Args: + pretty (bool): If True, output properties following the STIX specs + formatting. This includes indentation. Refer to notes for more + details. + **kwargs: The arguments for a json.dumps() call. + + Returns: + dict: The serialized JSON object. + + Note: + The argument ``pretty=True`` will output the STIX object following + spec order. Using this argument greatly impacts object serialization + performance. If your use case is centered across machine-to-machine + operation it is recommended to set ``pretty=False``. + + When ``pretty=True`` the following key-value pairs will be added or + overridden: indent=4, separators=(",", ": "), item_sort_key=sort_by. + """ + if pretty: + properties = self.object_properties() + + def sort_by(element): + return find_property_index(self, properties, element) + + kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by}) + + return json.dumps(self, cls=STIXJSONEncoder, **kwargs) + class _Observable(_STIXBase): diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b1cffd0..262b050 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -1,8 +1,9 @@ +import json + import pytest import stix2 - EXPECTED_BUNDLE = """{ "type": "bundle", "id": "bundle--00000000-0000-0000-0000-000000000004", @@ -41,6 +42,44 @@ EXPECTED_BUNDLE = """{ ] }""" +EXPECTED_BUNDLE_DICT = { + "type": "bundle", + "id": "bundle--00000000-0000-0000-0000-000000000004", + "spec_version": "2.0", + "objects": [ + { + "type": "indicator", + "id": "indicator--00000000-0000-0000-0000-000000000001", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "valid_from": "2017-01-01T12:34:56Z", + "labels": [ + "malicious-activity" + ] + }, + { + "type": "malware", + "id": "malware--00000000-0000-0000-0000-000000000002", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Cryptolocker", + "labels": [ + "ransomware" + ] + }, + { + "type": "relationship", + "id": "relationship--00000000-0000-0000-0000-000000000003", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "relationship_type": "indicates", + "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" + } + ] +} + def test_empty_bundle(): bundle = stix2.Bundle() @@ -82,10 +121,18 @@ def test_bundle_with_wrong_spec_version(): assert str(excinfo.value) == "Invalid value for Bundle 'spec_version': must equal '2.0'." -def test_create_bundle(indicator, malware, relationship): +def test_create_bundle1(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE + + +def test_create_bundle2(indicator, malware, relationship): + bundle = stix2.Bundle(objects=[indicator, malware, relationship]) + + print(repr(bundle)) + assert json.loads(bundle.serialize()) == EXPECTED_BUNDLE_DICT def test_create_bundle_with_positional_args(indicator, malware, relationship): From d31b110330fd5b6fa5dc4ef89479975e71101817 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 08:58:47 -0400 Subject: [PATCH 49/90] Update TAXII datastores --- stix2/sources/taxii.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 3fd5eff..9242ed2 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -45,21 +45,27 @@ class TAXIICollectionStore(DataStore): """ return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - def all_versions(self, stix_id): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve all versions of a single STIX object by ID. Translate all_versions() to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id) + return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - def query(self, query=None): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX objects matching a set of filters. Translate query() to the appropriate DataSource call. @@ -67,12 +73,18 @@ class TAXIICollectionStore(DataStore): Args: query (list): a list of filters (which collectively are the query) to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: stix_objs (list): a list of STIX objects """ - return self.source.query(query=query) + return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) def add(self, stix_objs, allow_custom=False, version=None): """Store STIX objects. @@ -157,7 +169,7 @@ class TAXIICollectionSource(DataSource): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote STIX Collection endpoint. @@ -222,7 +234,7 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] From 79475586d86ec6741d651db0dff6a556b992064e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 14:17:36 -0400 Subject: [PATCH 50/90] Revert object_properties() to #85 fix. Update tests accordingly --- stix2/base.py | 16 +++++++-------- stix2/test/test_data_sources.py | 35 +++++++++++++++++++++++++-------- stix2/test/test_filesystem.py | 10 +++++----- stix2/test/test_identity.py | 11 +++++++++++ 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index d7c7553..76b07b8 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,7 +40,14 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - return list(self._properties.keys()) + props = set(self._properties.keys()) + custom_props = list(set(self._inner.keys()) - props) + custom_props.sort() + + all_properties = list(self._properties.keys()) + all_properties.extend(custom_props) # Any custom properties to the bottom + + return all_properties def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: @@ -102,13 +109,6 @@ class _STIXBase(collections.Mapping): extra_kwargs = list(set(kwargs) - set(cls._properties)) if extra_kwargs: raise ExtraPropertiesError(cls, extra_kwargs) - else: - from .properties import CustomProperty - - # The custom properties will get added to the bottom. - # Matched with a CustomProperty. - extra_kwargs = list(set(kwargs) - set(cls._properties)) - self._properties.update([(x, CustomProperty()) for x in extra_kwargs]) # Remove any keyword arguments whose value is None setting_kwargs = {} diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index d798a0c..514ba40 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -498,29 +498,48 @@ def test_composite_datasource_operations(): objects=STIX_OBJS1, spec_version="2.0", type="bundle") - cds = CompositeDataSource() - ds1 = MemorySource(stix_data=BUNDLE1) - ds2 = MemorySource(stix_data=STIX_OBJS2) + cds1 = CompositeDataSource() + ds1_1 = MemorySource(stix_data=BUNDLE1) + ds1_2 = MemorySource(stix_data=STIX_OBJS2) - cds.add_data_sources([ds1, ds2]) + cds2 = CompositeDataSource() + ds2_1 = MemorySource(stix_data=BUNDLE1) + ds2_2 = MemorySource(stix_data=STIX_OBJS2) - indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + cds1.add_data_sources([ds1_1, ds1_2]) + cds2.add_data_sources([ds2_1, ds2_2]) + + indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") # In STIX_OBJS2 changed the 'modified' property to a later time... assert len(indicators) == 2 - indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + cds1.add_data_sources([cds2]) + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" - query = [ + query1 = [ Filter("type", "=", "indicator") ] - results = cds.query(query) + query2 = [ + Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") + ] + + cds1.filters.update(query2) + + results = cds1.query(query1) # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 1610615..85f6966 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -340,7 +340,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -352,9 +352,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): allow_custom=True) bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle, True) + fs_store.add(bundle, allow_custom=True) - camp_r = fs_store.get(camp.id) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -367,9 +367,9 @@ def test_filesystem_custom_object(fs_store): pass newobj = NewObj(property1='something') - fs_store.add(newobj, True) + fs_store.add(newobj, allow_custom=True) - newobj_r = fs_store.get(newobj.id) + newobj_r = fs_store.get(newobj.id, allow_custom=True) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py index a9415fe..8e3dd42 100644 --- a/stix2/test/test_identity.py +++ b/stix2/test/test_identity.py @@ -62,4 +62,15 @@ def test_parse_no_type(): "identity_class": "individual" }""") + +def test_identity_with_custom(): + identity = stix2.Identity( + name="John Smith", + identity_class="individual", + custom_properties={'x_foo': 'bar'} + ) + + assert identity.x_foo == "bar" + assert "x_foo" in identity.object_properties() + # TODO: Add other examples From 6f1ab52aa5a465837151f9a3487bd3c1402f07dc Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 14:22:15 -0400 Subject: [PATCH 51/90] Update Jupyter Notebook documentation --- docs/guide/custom.ipynb | 111 ++++++------------------------------ docs/guide/ts_support.ipynb | 13 ++++- 2 files changed, 29 insertions(+), 95 deletions(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 2254fa8..48b9ffe 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -99,101 +99,28 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
{\n",
-       "    "x_foo": "bar",\n",
-       "    "type": "identity",\n",
-       "    "id": "identity--8d7f0697-e589-4e3b-aa57-cae798d2d138",\n",
-       "    "created": "2017-09-26T21:02:19.465Z",\n",
-       "    "modified": "2017-09-26T21:02:19.465Z",\n",
-       "    "name": "John Smith",\n",
-       "    "identity_class": "individual"\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"identity\",\n", + " \"id\": \"identity--10761df2-93f6-4eb4-9d02-4fccfe5dc91d\",\n", + " \"created\": \"2017-11-03T18:20:48.145Z\",\n", + " \"modified\": \"2017-11-03T18:20:48.145Z\",\n", + " \"name\": \"John Smith\",\n", + " \"identity_class\": \"individual\",\n", + " \"x_foo\": \"bar\"\n", + "}\n" + ] } ], "source": [ + "from stix2 import Identity\n", + "\n", "identity = Identity(name=\"John Smith\",\n", " identity_class=\"individual\",\n", " custom_properties={\n", @@ -923,14 +850,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index f98d7b5..0c03548 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -114,7 +114,6 @@ "import stix2\n", "\n", "stix2.v20.Indicator()\n", - "\n", "stix2.v21.Indicator()" ] }, @@ -169,9 +168,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n \"type\": \"indicator\",\n \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n \"created\": \"2017-09-26T23:33:39.829Z\",\n \"modified\": \"2017-09-26T23:33:39.829Z\",\n \"name\": \"File hash for malware variant\",\n \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n \"valid_from\": \"2017-09-26T23:33:39.829952Z\",\n \"labels\": [\n \"malicious-activity\"\n ]\n}\n" + ] + } + ], "source": [ "from stix2 import parse\n", "\n", From 0d5859b9062adf62f19c56f40aa958c89fcbeeb6 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 15:13:00 -0400 Subject: [PATCH 52/90] Test other CompositeDataSource operations --- stix2/test/test_data_sources.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 514ba40..0bbc858 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -543,3 +543,13 @@ def test_composite_datasource_operations(): assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" + + # There is only one indicator with different ID. Since we use the same data + # when deduplicated, only two indicators (one with different modified). + results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(results) == 2 + + # Since we have filters already associated with our CompositeSource providing + # nothing returns the same as cds1.query(query1) (the associated query is query2) + results = cds1.query([]) + assert len(results) == 3 From 19818c85736b01ce0c19df6a793f1f7f50e49f94 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 6 Nov 2017 14:21:29 -0500 Subject: [PATCH 53/90] Remove stray print() --- stix2/test/test_bundle.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 262b050..8b14172 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -131,7 +131,6 @@ def test_create_bundle1(indicator, malware, relationship): def test_create_bundle2(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) - print(repr(bundle)) assert json.loads(bundle.serialize()) == EXPECTED_BUNDLE_DICT From da66f101473cf20f5b56957ca18d783d2c5c0d28 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 8 Nov 2017 13:53:21 -0500 Subject: [PATCH 54/90] Make DataStore a regular class, remove unwanted overrides, update tests. Remove CustomProperty since it is no longer needed --- stix2/properties.py | 12 ----- stix2/sources/__init__.py | 46 +++++++----------- stix2/sources/filesystem.py | 86 ++------------------------------- stix2/sources/memory.py | 68 ++------------------------ stix2/sources/taxii.py | 84 ++------------------------------ stix2/test/test_data_sources.py | 10 +--- 6 files changed, 31 insertions(+), 275 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 5b480ac..ca7f04c 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -393,15 +393,3 @@ class PatternProperty(StringProperty): raise ValueError(str(errors[0])) return self.string_type(value) - - -class CustomProperty(Property): - """ - The custom property class can be used as a base to extend further - functionality of a custom property. - - Note: - This class is used internally to signal the use of any custom property - that is parsed by any object or `parse()` method and allow_custom=True. - """ - pass diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 231b777..b3e8a29 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -23,9 +23,9 @@ def make_id(): return str(uuid.uuid4()) -class DataStore(with_metaclass(ABCMeta)): - """An implementer will create a concrete subclass from - this class for the specific DataStore. +class DataStore(object): + """An implementer can subclass to create custom behavior from + this class for the specific DataStores. Args: source (DataSource): An existing DataSource to use @@ -45,8 +45,7 @@ class DataStore(with_metaclass(ABCMeta)): self.source = source self.sink = sink - @abstractmethod - def get(self, stix_id): # pragma: no cover + def get(self, *args, **kwargs): """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. @@ -59,14 +58,12 @@ class DataStore(with_metaclass(ABCMeta)): object specified by the "id". """ - return NotImplementedError() + return self.source.get(*args, **kwargs) - @abstractmethod - def all_versions(self, stix_id): # pragma: no cover + def all_versions(self, *args, **kwargs): """Retrieve all versions of a single STIX object by ID. - Implement: Define a function that performs any custom behavior before - calling the associated DataSource all_versions() method. + Translate all_versions() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. @@ -75,16 +72,12 @@ class DataStore(with_metaclass(ABCMeta)): stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.source.all_versions(*args, **kwargs) - @abstractmethod - def query(self, query=None): # pragma: no cover + def query(self, *args, **kwargs): """Retrieve STIX objects matching a set of filters. - Implement: Specific data source API calls, processing, - functionality required for retrieving query from the data source. - - Define custom behavior before calling the associated DataSource query() + Translate query() call to the appropriate DataSource call. Args: query (list): a list of filters (which collectively are the query) @@ -94,10 +87,9 @@ class DataStore(with_metaclass(ABCMeta)): stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.source.query(*args, **kwargs) - @abstractmethod - def add(self, stix_objs): # pragma: no cover + def add(self, *args, **kwargs): """Method for storing STIX objects. Define custom behavior before storing STIX objects using the associated @@ -107,7 +99,7 @@ class DataStore(with_metaclass(ABCMeta)): stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.sink.add(*args, **kwargs) class DataSink(with_metaclass(ABCMeta)): @@ -123,7 +115,7 @@ class DataSink(with_metaclass(ABCMeta)): self.id = make_id() @abstractmethod - def add(self, stix_objs): # pragma: no cover + def add(self, stix_objs): """Method for storing STIX objects. Implement: Specific data sink API calls, processing, @@ -134,7 +126,6 @@ class DataSink(with_metaclass(ABCMeta)): STIX object) """ - raise NotImplementedError() class DataSource(with_metaclass(ABCMeta)): @@ -152,7 +143,7 @@ class DataSource(with_metaclass(ABCMeta)): self.filters = set() @abstractmethod - def get(self, stix_id): # pragma: no cover + def get(self, stix_id): """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -166,10 +157,9 @@ class DataSource(with_metaclass(ABCMeta)): stix_obj: the STIX object """ - raise NotImplementedError() @abstractmethod - def all_versions(self, stix_id): # pragma: no cover + def all_versions(self, stix_id): """ Implement: Similar to get() except returns list of all object versions of the specified "id". In addition, implement the specific data @@ -185,10 +175,9 @@ class DataSource(with_metaclass(ABCMeta)): stix_objs (list): a list of STIX objects """ - raise NotImplementedError() @abstractmethod - def query(self, query=None): # pragma: no cover + def query(self, query=None): """ Implement: The specific data source API calls, processing, functionality required for retrieving query from the data source @@ -201,7 +190,6 @@ class DataSource(with_metaclass(ABCMeta)): stix_objs (list): a list of STIX objects """ - raise NotImplementedError() class CompositeDataSource(DataSource): diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 4287cbf..e92c525 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -31,88 +31,10 @@ class FileSystemStore(DataStore): """ def __init__(self, stix_dir, bundlify=False): - super(FileSystemStore, self).__init__() - self.source = FileSystemSource(stix_dir=stix_dir) - self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) - - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Implement: Translate all_versions() call to the appropriate DataSource - call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Implement: Specific data source API calls, processing, - functionality required for retrieving query from the data source. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translates add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + super(FileSystemStore, self).__init__( + source=FileSystemSource(stix_dir=stix_dir), + sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + ) class FileSystemSink(DataSink): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 5b44edb..a069747 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -93,14 +93,15 @@ class MemoryStore(DataStore): """ def __init__(self, stix_data=None, allow_custom=False, version=None): - super(MemoryStore, self).__init__() self._data = {} if stix_data: _add(self, stix_data, allow_custom=allow_custom, version=version) - self.source = MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) - self.sink = MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + super(MemoryStore, self).__init__( + source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True), + sink=MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + ) def save_to_file(self, file_path, allow_custom=False): """Write SITX objects from in-memory dictionary to JSON file, as a STIX @@ -130,67 +131,6 @@ class MemoryStore(DataStore): """ return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) - def get(self, stix_id, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Translate all_versions() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, _composite_filters=_composite_filters) - - def query(self, query=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Translates query() to appropriate DataStore call. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translates add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) - class MemorySink(DataSink): """Interface for adding/pushing STIX objects to an in-memory dictionary. diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 9242ed2..8eb5069 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -20,86 +20,10 @@ class TAXIICollectionStore(DataStore): collection (taxii2.Collection): TAXII Collection instance """ def __init__(self, collection): - super(TAXIICollectionStore, self).__init__() - self.source = TAXIICollectionSource(collection) - self.sink = TAXIICollectionSink(collection) - - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Translate all_versions() to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Translate query() to the appropriate DataSource call. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translate add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + super(TAXIICollectionStore, self).__init__( + source=TAXIICollectionSource(collection), + sink=TAXIICollectionSink(collection) + ) class TAXIICollectionSink(DataSink): diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0bbc858..ef0cf26 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -2,8 +2,8 @@ import pytest from taxii2client import Collection from stix2 import Filter, MemorySink, MemorySource -from stix2.sources import (CompositeDataSource, DataSink, DataSource, - DataStore, make_id, taxii) +from stix2.sources import (CompositeDataSource, DataSink, DataSource, make_id, + taxii) from stix2.sources.filters import apply_common_filters from stix2.utils import deduplicate @@ -122,12 +122,6 @@ STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] def test_ds_abstract_class_smoke(): - with pytest.raises(TypeError): - DataStore() - - with pytest.raises(TypeError): - DataStore.get() - with pytest.raises(TypeError): DataSource() From 258ea4b82caf9728da1d79e62c82bc68a3fccb4c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 8 Nov 2017 14:01:56 -0500 Subject: [PATCH 55/90] Some cleanup in MemoryStore --- stix2/sources/memory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index a069747..308d0d0 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -103,7 +103,7 @@ class MemoryStore(DataStore): sink=MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) ) - def save_to_file(self, file_path, allow_custom=False): + def save_to_file(self, *args, **kwargs): """Write SITX objects from in-memory dictionary to JSON file, as a STIX Bundle. @@ -113,9 +113,9 @@ class MemoryStore(DataStore): not. Default: False. """ - return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) + return self.sink.save_to_file(*args, **kwargs) - def load_from_file(self, file_path, allow_custom=False, version=None): + def load_from_file(self, *args, **kwargs): """Load STIX data from JSON file. File format is expected to be a single JSON @@ -129,7 +129,7 @@ class MemoryStore(DataStore): None, use latest version. """ - return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) + return self.source.load_from_file(*args, **kwargs) class MemorySink(DataSink): From c38483b63169c26bbaf9101809615e606e760d66 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 9 Nov 2017 10:10:19 -0500 Subject: [PATCH 56/90] Improve markings docstrings Including a note that the markings functions can be used as methods on SDO/SRO/MarkingDefinition objects. --- stix2/markings/__init__.py | 37 ++++++++++++++++------------- stix2/markings/granular_markings.py | 26 ++++++++++---------- stix2/markings/object_markings.py | 10 ++++---- stix2/markings/utils.py | 8 +++---- 4 files changed, 43 insertions(+), 38 deletions(-) diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 4038a5e..c8dbdbc 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -1,9 +1,14 @@ """ -Functions and classes for working with STIX 2 Data Markings. +Functions for working with STIX 2 Data Markings. -These high level functions will operate on both object level markings and +These high level functions will operate on both object-level markings and granular markings unless otherwise noted in each of the functions. +Note: + These functions are also available as methods on SDOs, SROs, and Marking + Definitions. The corresponding methods on those classes are identical to + these functions except that the `obj` parameter is omitted. + .. autosummary:: :toctree: markings @@ -20,7 +25,7 @@ from stix2.markings import granular_markings, object_markings def get_markings(obj, selectors=None, inherited=False, descendants=False): """ - Get all markings associated to the field(s). + Get all markings associated to the field(s) specified by selectors. Args: obj: An SDO or SRO object. @@ -57,15 +62,15 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False): def set_markings(obj, marking, selectors=None): """ - Removes all markings associated with selectors and appends a new granular + Remove all markings associated with selectors and appends a new granular marking. Refer to `clear_markings` and `add_markings` for details. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Returns: A new version of the given SDO or SRO with specified markings removed @@ -84,14 +89,14 @@ def set_markings(obj, marking, selectors=None): def remove_markings(obj, marking, selectors=None): """ - Removes granular_marking from the granular_markings collection. + Remove a marking from this object. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -114,14 +119,14 @@ def remove_markings(obj, marking, selectors=None): def add_markings(obj, marking, selectors=None): """ - Appends a granular_marking to the granular_markings collection. + Append a marking to this object. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -142,7 +147,7 @@ def add_markings(obj, marking, selectors=None): def clear_markings(obj, selectors=None): """ - Removes all granular_marking associated with the selectors. + Remove all markings associated with the selectors. Args: obj: An SDO or SRO object. @@ -170,14 +175,14 @@ def clear_markings(obj, selectors=None): def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ - Checks if field(s) is marked by any marking or by specific marking(s). + Check if field(s) is marked by any marking or by specific marking(s). Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the field(s) appear(s). marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). inherited: If True, include object level markings and granular markings inherited to determine if the properties is/are marked. descendants: If True, include granular markings applied to any children diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 3fe3a48..be5d258 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -8,7 +8,7 @@ from stix2.utils import new_version def get_markings(obj, selectors, inherited=False, descendants=False): """ - Get all markings associated to with the properties. + Get all granular markings associated to with the properties. Args: obj: An SDO or SRO object. @@ -50,8 +50,8 @@ def get_markings(obj, selectors, inherited=False, descendants=False): def set_markings(obj, marking, selectors): """ - Removes all markings associated with selectors and appends a new granular - marking. Refer to `clear_markings` and `add_markings` for details. + Remove all granular markings associated with selectors and append a new + granular marking. Refer to `clear_markings` and `add_markings` for details. Args: obj: An SDO or SRO object. @@ -71,14 +71,14 @@ def set_markings(obj, marking, selectors): def remove_markings(obj, marking, selectors): """ - Removes granular_marking from the granular_markings collection. + Remove a granular marking from the granular_markings collection. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -123,14 +123,14 @@ def remove_markings(obj, marking, selectors): def add_markings(obj, marking, selectors): """ - Appends a granular_marking to the granular_markings collection. + Append a granular marking to the granular_markings collection. Args: obj: An SDO or SRO object. - selectors: list of type string, selectors must be relative to the TLO - in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: list of type string, selectors must be relative to the TLO + in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -157,7 +157,7 @@ def add_markings(obj, marking, selectors): def clear_markings(obj, selectors): """ - Removes all granular_markings associated with the selectors. + Remove all granular markings associated with the selectors. Args: obj: An SDO or SRO object. @@ -214,14 +214,14 @@ def clear_markings(obj, selectors): def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ - Checks if field is marked by any marking or by specific marking(s). + Check if field is marked by any marking or by specific marking(s). Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. inherited: If True, return markings inherited from the given selector. descendants: If True, return granular markings applied to any children of the given selector. diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index cb78294..c0375c3 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -23,7 +23,7 @@ def get_markings(obj): def add_markings(obj, marking): """ - Appends an object level marking to the object_marking_refs collection. + Append an object level marking to the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -42,7 +42,7 @@ def add_markings(obj, marking): def remove_markings(obj, marking): """ - Removes object level marking from the object_marking_refs collection. + Remove an object level marking from the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -76,7 +76,7 @@ def remove_markings(obj, marking): def set_markings(obj, marking): """ - Removes all object level markings and appends new object level markings to + Remove all object level markings and append new object level markings to the collection. Refer to `clear_markings` and `add_markings` for details. Args: @@ -94,7 +94,7 @@ def set_markings(obj, marking): def clear_markings(obj): """ - Removes all object level markings from the object_marking_refs collection. + Remove all object level markings from the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -108,7 +108,7 @@ def clear_markings(obj): def is_marked(obj, marking=None): """ - Checks if SDO or SRO is marked by any marking or by specific marking(s). + Check if SDO or SRO is marked by any marking or by specific marking(s). Args: obj: A SDO or SRO object. diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 3024fe8..429311b 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -9,7 +9,7 @@ from stix2 import exceptions def _evaluate_expression(obj, selector): - """Walks an SDO or SRO generating selectors to match against ``selector``. + """Walk an SDO or SRO generating selectors to match against ``selector``. If a match is found and the the value of this property is present in the objects. Matching value of the property will be returned. @@ -32,7 +32,7 @@ def _evaluate_expression(obj, selector): def _validate_selector(obj, selector): - """Internal method to evaluate each selector.""" + """Evaluate each selector against an object.""" results = list(_evaluate_expression(obj, selector)) if len(results) >= 1: @@ -132,7 +132,7 @@ def compress_markings(granular_markings): def expand_markings(granular_markings): - """Expands granular markings list. + """Expand granular markings list. If there is more than one selector per granular marking. It will be expanded using the same marking_ref. @@ -187,7 +187,7 @@ def expand_markings(granular_markings): def build_granular_marking(granular_marking): - """Returns a dictionary with the required structure for a granular marking. + """Return a dictionary with the required structure for a granular marking. """ return {"granular_markings": expand_markings(granular_marking)} From 7495ea658875070055f8797a960a69b241694619 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Thu, 9 Nov 2017 15:42:59 -0500 Subject: [PATCH 57/90] Add inter-documentation links So when users read the guide docs, they can click a link to go to a relevant section in the API reference. --- docs/guide/creating.ipynb | 10 +- docs/guide/custom.ipynb | 123 ++++++++++++++++---- docs/guide/datastore.ipynb | 53 +++------ docs/guide/environment.ipynb | 28 ++--- docs/guide/filesystem.ipynb | 22 ++-- docs/guide/markings.ipynb | 14 ++- docs/guide/memory.ipynb | 16 +-- docs/guide/parsing.ipynb | 6 +- docs/guide/serializing.ipynb | 123 ++++++++++++++++++-- docs/guide/taxii.ipynb | 8 +- docs/guide/ts_support.ipynb | 220 +++++++++++++++++++++++++++++++---- docs/guide/versioning.ipynb | 6 +- requirements.txt | 2 +- stix2/environment.py | 7 +- 14 files changed, 486 insertions(+), 152 deletions(-) diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 6cfea7f..f9ff11d 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -381,7 +381,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To update the properties of an object, see the **Versioning** section." + "To update the properties of an object, see the [Versioning](versioning.ipynb) section." ] }, { @@ -500,7 +500,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided." + "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided.\n", + "\n", + "You can see the full list of SDO classes [here](../api/stix2.sdo.rst)." ] }, { @@ -869,9 +871,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 48b9ffe..9badeed 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "collapsed": true, "nbsphinx": "hidden" @@ -99,23 +99,98 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"identity\",\n", - " \"id\": \"identity--10761df2-93f6-4eb4-9d02-4fccfe5dc91d\",\n", - " \"created\": \"2017-11-03T18:20:48.145Z\",\n", - " \"modified\": \"2017-11-03T18:20:48.145Z\",\n", - " \"name\": \"John Smith\",\n", - " \"identity_class\": \"individual\",\n", - " \"x_foo\": \"bar\"\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "identity",\n",
+       "    "id": "identity--00c5743f-2d5e-4d66-88f1-1842584f4519",\n",
+       "    "created": "2017-11-09T16:17:44.596Z",\n",
+       "    "modified": "2017-11-09T16:17:44.596Z",\n",
+       "    "name": "John Smith",\n",
+       "    "identity_class": "individual",\n",
+       "    "x_foo": "bar"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -244,7 +319,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to ``parse()``:" + "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.core.rst#stix2.core.parse):" ] }, { @@ -282,7 +357,7 @@ "source": [ "### Custom STIX Object Types\n", "\n", - "To create a custom STIX object type, define a class with the ``@CustomObject`` decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n", + "To create a custom STIX object type, define a class with the @[CustomObject](../api/stix2.sdo.rst#stix2.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n", "\n", "Let's say zoo animals have become a serious cyber threat and we want to model them in STIX using a custom object type. Let's use a ``species`` property to store the kind of animal, and make that property required. We also want a property to store the class of animal, such as \"mammal\" or \"bird\" but only want to allow specific values in it. We can add some logic to validate this property in ``__init__``." ] @@ -513,7 +588,7 @@ "source": [ "### Custom Cyber Observable Types\n", "\n", - "Similar to custom STIX object types, use a decorator to create custom Cyber Observable types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary." + "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/stix2.observables.rst#stix2.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary." ] }, { @@ -679,7 +754,7 @@ "source": [ "### Custom Cyber Observable Extensions\n", "\n", - "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the ``@CustomExtension`` decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:" + "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/stix2.observables.rst#stix2.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:" ] }, { @@ -843,21 +918,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index f6d360b..7fc0997 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -58,9 +58,9 @@ "source": [ "# DataStore API\n", "\n", - "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of DataStore, DataSource and DataSink constructs: a DataSource for pulling STIX2 content, a DataSink for pushing STIX2 content, and a DataStore for pulling/pushing.\n", + "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) constructs: a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) for both pulling and pushing.\n", "\n", - "The DataStore, DataSource, DataSink (referred to as \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then sublcassed into real DataStore suite(s). CTI Python STIX2 provides for the DataStore suites of **FileSystem**, **Memory**, and **TAXII**. Users are also encrouraged subclassing the base Data suite and creating their own custom DataStore suites." + "The DataStore, [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource), [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then sublcassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites." ] }, { @@ -69,13 +69,13 @@ "source": [ "## CompositeDataSource\n", "\n", - "**CompositeDataSource** is an available controller that can be used as a single interface to a set of defined DataSources. The purpose of this controller is allow for the grouping of **DataSources** and making get/query calls to a set of DataSources in one API call. **CompositeDataSource** can be used to organize/group **DataSources**, federate get()/all_versions()/query() calls, and reduce user code.\n", + "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is an available controller that can be used as a single interface to a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). The purpose of this controller is allow for the grouping of [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) and making `get()`/`query()` calls to a set of DataSources in one API call. [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can be used to organize/group [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource), federate `get()`/`all_versions()`/`query()` calls, and reduce user code.\n", "\n", - "**CompositeDataSource** is just a wrapper around a set of defined **DataSources** (e.g. FileSystemSource) that federates get()/all_versions()/query() calls individually to each of the attached **DataSources** , collects the results from each **DataSource** and returns them.\n", + "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is just a wrapper around a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) (e.g. [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource)) that federates `get()`/`all_versions()`/`query()` calls individually to each of the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) , collects the results from each [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and returns them.\n", "\n", - "Filters can be attached to **CompositeDataSources** just as they can be done to **DataStores** and **DataSources**. When get()/all_versions()/query() calls are made to the **CompositeDataSource**, it will pass along any query filters from the call and any of its own filters to the attached **DataSources**. To which, those attached **DataSources** may have their own attached filters as well. The effect is that all the filters are eventually combined when the get()/all_versions()/query() call is actually executed within a **DataSource**. \n", + "Filters can be attached to [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore) and [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). In addition, those [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource). \n", "\n", - "A **CompositeDataSource** can also be attached to a **CompositeDataSource** for multiple layers of grouped **DataSources**.\n", + "A [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can also be attached to a [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) for multiple layers of grouped [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource).\n", "\n", "\n", "### CompositeDataSource API\n", @@ -161,37 +161,18 @@ "source": [ "## Filters\n", "\n", - "The CTI Python STIX2 **DataStore** suites - **FileSystem**, **Memory** and **TAXII** - all use the **Filters** module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to **query()**, and/or attached to a **DataStore** so that every future query placed to that **DataStore** is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from **DataStores**.\n", + "The CTI Python STIX2 DataStore suites - [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst) - all use the [Filters](../api/sources/stix2.sources.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) so that every future query placed to that [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore).\n", "\n", - "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). Currently, CTI Python STIX2 supports **ONLY** STIX 2 object common properties and TAXII2 Filtering parameters for fields to filter on:\n", + "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX2 objects can be filtered on. In addition, TAXII2 Filtering parameters for fields can also be used in filters.\n", "\n", - "Fields\n", - "\n", - "(STIX2 Object Common Properties)\n", - "\n", - "* created\n", - "* created_by_ref\n", - "* external_references.source_name\n", - "* external_references.description\n", - "* external_references.url\n", - "* external_references.external_id\n", - "* granular_markings.marking_ref\n", - "* granular_markings.selectors\n", - "* id\n", - "* labels\n", - "* modified\n", - "* object_marking_refs\n", - "* revoked\n", - "* type\n", - "\n", - "(TAXII2 filter fields)\n", + "TAXII2 filter fields:\n", "\n", "* added_after\n", "* match[id]\n", "* match[type]\n", "* match[version]\n", "\n", - "Supported operators on above properties:\n", + "Supported operators:\n", "\n", "* =\n", "* !=\n", @@ -201,7 +182,7 @@ "* ```>=```\n", "* <=\n", "\n", - "Value types of the common property values must be one of these (python) types:\n", + "Value types of the property values must be one of these (Python) types:\n", "\n", "* bool\n", "* dict\n", @@ -245,7 +226,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For Filters to be applied to a query, they must be either supplied with the query call or attached a **DataStore**, more specifically to **DataSource** whether that **DataSource** is a part of a **DataStore** or stands by itself. " + "For Filters to be applied to a query, they must be either supplied with the query call or attached a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), more specifically to a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) whether that [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) is a part of a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) or stands by itself. " ] }, { @@ -285,21 +266,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb index a9184a1..2d85911 100644 --- a/docs/guide/environment.ipynb +++ b/docs/guide/environment.ipynb @@ -58,11 +58,11 @@ "source": [ "## Using Environments\n", "\n", - "An ``Environment`` object makes it easier to use STIX 2 content as part of a larger application or ecosystem. It allows you to abstract away the nasty details of sending and receiving STIX data, and to create STIX objects with default values for common properties.\n", + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) object makes it easier to use STIX 2 content as part of a larger application or ecosystem. It allows you to abstract away the nasty details of sending and receiving STIX data, and to create STIX objects with default values for common properties.\n", "\n", "### Storing and Retrieving STIX Content\n", "\n", - "An ``Environment`` can be set up with a ``DataStore`` if you want to store and retrieve STIX content from the same place. " + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) if you want to store and retrieve STIX content from the same place. " ] }, { @@ -82,7 +82,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If desired, you can instead set up an ``Environment`` with different data sources and sinks. In the following example we set up an environment that retrieves objects from memory and a directory on the filesystem, and stores objects in a different directory on the filesystem." + "If desired, you can instead set up an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with different data sources and sinks. In the following example we set up an environment that retrieves objects from [memory](../api/sources/stix2.sources.memory.rst) and a directory on the [filesystem](../api/sources/stix2.sources.filesystem.rst), and stores objects in a different directory on the filesystem." ] }, { @@ -105,13 +105,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once you have an ``Environment`` you can store some STIX content in its DataSinks with ``add()``:" + "Once you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) you can store some STIX content in its [DataSinks](../api/stix2.sources.rst#stix2.sources.DataSink) with [add()](../api/stix2.environment.rst#stix2.environment.Environment.add):" ] }, { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import Indicator\n", @@ -126,7 +128,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can retrieve STIX objects from the DataSources in the Environment with ``get()``, ``query()``, and ``all_versions()``, just as you would for a DataSource." + "You can retrieve STIX objects from the [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), and [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), just as you would for a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource)." ] }, { @@ -237,7 +239,7 @@ "source": [ "### Creating STIX Objects With Defaults\n", "\n", - "To create STIX objects with default values for certain properties, use an ``ObjectFactory``. For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization." + "To create STIX objects with default values for certain properties, use an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory). For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization." ] }, { @@ -259,7 +261,7 @@ "collapsed": true }, "source": [ - "Once you've set up the Object Factory, use its ``create()`` method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object." + "Once you've set up the [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory), use its [create()](../api/stix2.environment.rst#stix2.environment.ObjectFactory.create) method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object." ] }, { @@ -374,14 +376,14 @@ "collapsed": true }, "source": [ - "All objects we create with that ``ObjectFactory`` will automatically get the default value for ``created_by_ref``. These are the properties for which defaults can be set:\n", + "All objects we create with that [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory) will automatically get the default value for ``created_by_ref``. These are the properties for which defaults can be set:\n", "\n", "- ``created_by_ref``\n", "- ``created``\n", "- ``external_references``\n", "- ``object_marking_refs``\n", "\n", - "These defaults can be bypassed. For example, say you have an ``Environment`` with multiple default values but want to create an object with a different value for ``created_by_ref``, or none at all." + "These defaults can be bypassed. For example, say you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with multiple default values but want to create an object with a different value for ``created_by_ref``, or none at all." ] }, { @@ -607,7 +609,7 @@ "collapsed": true }, "source": [ - "For the full power of the Environment layer, create an Environment with both a DataStore/Source/Sink and an Object Factory:" + "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore)/[Source](../api/stix2.sources.rst#stix2.sources.DataSource)/[Sink](../api/stix2.sources.rst#stix2.sources.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" ] }, { @@ -723,9 +725,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index c2e21b4..4b5bd6f 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -58,7 +58,7 @@ "source": [ "## FileSystem \n", "\n", - "The FileSystem suite contains **FileSystemStore **, **FileSystemSource** and **FileSystemSink**. Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", + "The FileSystem suite contains [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore), [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", "\n", "The directory and file structure of the intended STIX2 content should be:\n", "\n", @@ -82,7 +82,7 @@ " /STIX2 Domain Object type\n", "```\n", "\n", - "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are json files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", + "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", "\n", "```\n", "stix2_content/\n", @@ -107,15 +107,15 @@ " /vulnerability\n", "```\n", "\n", - "**FileSystemStore** is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As **FileSystemStore** is just a wrapper around a paired **FileSystemSource** and **FileSystemSink** that point the same file directory.\n", + "[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n", "\n", - "Use cases where STIX2 content will only be retrieved or pushed, then a **FileSystemSource** and **FileSystemSink** can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", + "Use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", "\n", "### FileSystem API\n", "\n", - "A note on **get()**, **all_versions()**, and **query()**. The format of the STIX2 content targeted by the FileSystem suite is json files. When STIX2 content (in json) is retrieved by the **FileSystemStore** from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", + "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query). The format of the STIX2 content targeted by the FileSystem suite is JSON files. When STIX2 content (in JSON) is retrieved by the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", "\n", - "A note on **add()**. When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: python STIX objects, python dicts (of valid STIX objects or Bundles), json-encoded strings (of valid STIX objects or Bundles), or a (python)list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX json object (in a STIX Bundle) and written to disk. \n", + "A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", "\n", "### FileSystem Examples\n", "\n", @@ -532,21 +532,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index cb8f762..5df8f54 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -696,7 +696,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Both object markings and granular markings can also be added to STIX objects which have already been created.\n", + "[Several functions](../api/stix2.markings.rst) exist to support working with data markings.\n", + "\n", + "Both object markings and granular markings can be added to STIX objects which have already been created.\n", "\n", "**Note**: Doing so will create a new version of the object (note the updated ``modified`` time)." ] @@ -1041,7 +1043,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "STIX objects can also be cleared of all markings:" + "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):" ] }, { @@ -1188,7 +1190,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To get a list of the granular markings on an object, pass the object and a list of selectors to ``get_markings``:" + "To get a list of the granular markings on an object, pass the object and a list of selectors to [get_markings()](../api/stix2.markings.rst#stix2.markings.get_markings):" ] }, { @@ -1215,7 +1217,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can also call ``get_markings()`` as a method on the STIX object." + "You can also call [get_markings()](../api/stix2.markings.rst#stix2.markings.get_markings) as a method on the STIX object." ] }, { @@ -1310,9 +1312,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index 570425e..75c0475 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -58,14 +58,14 @@ "source": [ "## Memory\n", "\n", - "The Memory suite consists of **MemoryStore**, **MemorySource**, and **MemorySink**. Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the **MemoryStore** is a just a wrapper around a paired **MemorySource** and **MemorySink**; as there is quite limited uses for just a **MemorySource** or a **MemorySink**, it is recommended to always use **MemoryStore**. The **MemoryStore** is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is ecompassed within the **FileSystemStore**. However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. **MemoryStore.save_to_file()** allows for saving all the STIX content that is in memory to a json file. **MemoryStore.load_from_file()** allows for loading STIX content from a json-formatted file. \n", + "The Memory suite consists of [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore), [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource), and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink). Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is a just a wrapper around a paired [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink); as there is quite limited uses for just a [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) or a [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink), it is recommended to always use [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). The [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is encompassed within the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore). However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. [MemoryStore.save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file) allows for saving all the STIX content that is in memory to a json file. [MemoryStore.load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) allows for loading STIX content from a JSON-formatted file. \n", "\n", "\n", "### Memory API\n", "\n", - "A note on **load_from_file()** and **save()**. These methods both add STIX content to an internal dictionary (maintained by MemoryStore). STIX content that is to be added can be in the following forms: python STIX objects, python dicts (of valid STIX objects or Bundles), json-encoded strings (of valid STIX objects or Bundles), or a (python)list of any of the previously listed types. **MemoryStore** actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as , is what it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dicts (of STIX objects) are stored as python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the **MemoryStore**. Also, for **load_from_file()**, STIX content is assumed to be in json form within the file, individually or in a Bundle. \n", + "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) and [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). These methods both add STIX content to an internal dictionary (maintained by [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore)). STIX content that is to be added can be in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as, is how it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dictionaries (of STIX objects) are stored as Python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). Also, for [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, individually or in a Bundle. \n", "\n", - "A note on **save_to_file()**. This method dumps all STIX content that is in MemoryStore to the specified file. The file format will be json, and the STIX content will be within a STIX Bundle. ntoe, the the output form will be a json STIX Bundle regardless of the form that the individual STIX objects are stored(i.e. supplied) to the MemoryStore. \n", + "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", "\n", "### Memory Examples\n", "\n", @@ -288,21 +288,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb index e991e0c..d24f994 100644 --- a/docs/guide/parsing.ipynb +++ b/docs/guide/parsing.ipynb @@ -63,7 +63,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Parsing STIX content is as easy as calling the `parse()` function on a JSON string. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well." + "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well." ] }, { @@ -109,9 +109,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb index 8fcc19d..4499677 100644 --- a/docs/guide/serializing.ipynb +++ b/docs/guide/serializing.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "collapsed": true, "nbsphinx": "hidden" @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -144,15 +144,15 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--81b0644d-5e9d-48fb-bb83-aabe77918305",\n",
-       "    "created": "2017-09-26T23:38:55.476Z",\n",
-       "    "modified": "2017-09-26T23:38:55.476Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674",\n",
+       "    "created": "2017-11-09T19:21:06.285Z",\n",
+       "    "modified": "2017-11-09T19:21:06.285Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-09-26T23:38:55.476436Z"\n",
+       "    "valid_from": "2017-11-09T19:21:06.285451Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
        "}\n",
        "
\n" ], @@ -160,7 +160,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -174,13 +174,112 @@ "\n", "print(str(indicator))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's serialize() function:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{"valid_from": "2017-11-09T19:21:06.285451Z", "name": "File hash for malware variant", "created": "2017-11-09T19:21:06.285Z", "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "labels": ["malicious-activity"], "modified": "2017-11-09T19:21:06.285Z", "type": "indicator", "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674"}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(indicator.serialize())" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 016f9d8..2f8905b 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -58,9 +58,9 @@ "source": [ "## TAXIICollection\n", "\n", - "The TAXIICollection suite contains **TAXIICollectionStore**, **TAXIICollectionSource**, and **TAXIICollectionSink**. **TAXIICollectionStore** for pushing and retrieving STIX content to local/remote TAXII Collection(s). **TAXIICollectionSource** for retrieving STIX content to local/remote TAXII Collection(s). **TAXIICollectionSink** for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be binded to a Collection from the taxii2client library (taxii2client.Collection), where all **TAXIICollection** API calls will be executed through that Collection instance.\n", + "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) for pushing and retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) for retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", "\n", - "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the **TAXIICollection** suite supports searching on all STIX2 common object properties (see **Filters** documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. **TAXIICollection** will seperate any supplied queries into TAXII supported filters and non-supported filters. During a **TAXIICollection** API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the **TAXIICollection** API call. \n", + "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", "\n", "### TAXIICollection API\n", "\n", @@ -265,9 +265,9 @@ ], "metadata": { "kernelspec": { - "display_name": "cti-python-stix2", + "display_name": "Python 2", "language": "python", - "name": "cti-python-stix2" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index 0c03548..092f91d 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -1,5 +1,57 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# Delete this cell to re-enable tracebacks\n", + "import sys\n", + "ipython = get_ipython()\n", + "\n", + "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", + " exception_only=False, running_compiled_code=False):\n", + " etype, value, tb = sys.exc_info()\n", + " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", + "\n", + "ipython.showtraceback = hide_traceback" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true, + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# JSON output syntax highlighting\n", + "from __future__ import print_function\n", + "from pygments import highlight\n", + "from pygments.lexers import JsonLexer\n", + "from pygments.formatters import HtmlFormatter\n", + "from IPython.display import HTML\n", + "\n", + "original_print = print\n", + "\n", + "def json_print(inpt):\n", + " string = str(inpt)\n", + " if string[0] == '{':\n", + " formatter = HtmlFormatter()\n", + " return HTML('{}'.format(\n", + " formatter.get_style_defs('.highlight'),\n", + " highlight(string, JsonLexer(), formatter)))\n", + " else:\n", + " original_print(inpt)\n", + "\n", + "print = json_print" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -10,13 +62,15 @@ "\n", "Imports can be used in different ways depending on the use case and support levels.\n", "\n", - "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" + "People who want to support the latest version of STIX 2.X without having to make changes, can implicitly use the latest version:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", @@ -34,7 +88,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import Indicator\n", @@ -46,13 +102,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "People who want to use an explicit version" + "People who want to use an explicit version:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2.v20\n", @@ -70,7 +128,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2.v20 import Indicator\n", @@ -88,7 +148,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2.v20 as stix2\n", @@ -108,7 +170,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", @@ -127,7 +191,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import v20, v21\n", @@ -146,7 +212,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2.v20 import Indicator as Indicator_v20\n", @@ -163,20 +231,106 @@ "### How parsing will work\n", "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", "\n", - "You can lock your `parse()` method to a specific STIX version by" + "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n \"type\": \"indicator\",\n \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n \"created\": \"2017-09-26T23:33:39.829Z\",\n \"modified\": \"2017-09-26T23:33:39.829Z\",\n \"name\": \"File hash for malware variant\",\n \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n \"valid_from\": \"2017-09-26T23:33:39.829952Z\",\n \"labels\": [\n \"malicious-activity\"\n ]\n}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
+       "    "created": "2017-09-26T23:33:39.829Z",\n",
+       "    "modified": "2017-09-26T23:33:39.829Z",\n",
+       "    "name": "File hash for malware variant",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2017-09-26T23:33:39.829952Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -208,17 +362,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### How will custom work\n", + "### How will custom content work\n", "\n", - "CustomObject, CustomObservable, CustomMarking and CustomExtension must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "[CustomObject](../api/stix2.sdo.rst#stix2.sdo.CustomObject), [CustomObservable](../api/stix2.observables.rst#stix2.observables.CustomObservable), [CustomMarking](../api/stix2.common.rst#stix2.common.CustomMarking) and [CustomExtension](../api/stix2.observables.rst#stix2.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", "\n", - "You can perform this by," + "You can perform this by:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", @@ -238,7 +394,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } diff --git a/docs/guide/versioning.ipynb b/docs/guide/versioning.ipynb index 30ceb69..fb3b866 100644 --- a/docs/guide/versioning.ipynb +++ b/docs/guide/versioning.ipynb @@ -182,7 +182,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The modified time will be updated to the current time unless you provide a specific value as a keyword argument. Note that you can’t change the type, id, or created properties." + "The modified time will be updated to the current time unless you provide a specific value as a keyword argument. Note that you can’t change the `type`, `id`, or `created` properties." ] }, { @@ -322,9 +322,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/requirements.txt b/requirements.txt index 93328de..0de31a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bumpversion -nbsphinx +nbsphinx>=0.2.15 pre-commit pytest pytest-cov diff --git a/stix2/environment.py b/stix2/environment.py index 4919335..64a73b1 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -76,7 +76,7 @@ class ObjectFactory(object): class Environment(object): - """ + """Abstract away some of the nasty details of working with STIX content. Args: factory (ObjectFactory, optional): Factory for creating objects with common @@ -161,9 +161,8 @@ class Environment(object): up. Returns: - The STIX object's creator, or - None, if the object contains no `created_by_ref` property or the - object's creator cannot be found. + The STIX object's creator, or None, if the object contains no + `created_by_ref` property or the object's creator cannot be found. """ creator_id = obj.get('created_by_ref', '') From 60c3aa0167e8196919c9f527ada541f19d2566e9 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Mon, 13 Nov 2017 09:44:00 -0500 Subject: [PATCH 58/90] Fix redundant Data Markings example To show both uses of get_markings() --- docs/guide/markings.ipynb | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index 5df8f54..fcacf47 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -146,14 +146,14 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T14:42:54.685Z",\n",
+       "    "id": "indicator--65ff0082-bb92-4812-9b74-b144b858297f",\n",
+       "    "created": "2017-11-13T14:42:14.641Z",\n",
+       "    "modified": "2017-11-13T14:42:14.641Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2017-11-13T14:42:14.641818Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -187,7 +187,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -263,8 +263,8 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "marking-definition",\n",
-       "    "id": "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53",\n",
-       "    "created": "2017-10-04T14:43:04.090873Z",\n",
+       "    "id": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
+       "    "created": "2017-11-13T14:43:30.558058Z",\n",
        "    "definition_type": "statement",\n",
        "    "definition": {\n",
        "        "statement": "Copyright 2017, Example Corp"\n",
@@ -276,7 +276,7 @@
        ""
       ]
      },
-     "execution_count": 4,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -523,7 +523,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -599,9 +599,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--9f8970eb-b398-41b6-b8c8-8a607ad3a2c5",\n",
-       "    "created": "2017-10-04T14:43:26.129Z",\n",
-       "    "modified": "2017-10-04T14:43:26.129Z",\n",
+       "    "id": "malware--f7128008-f6ab-4d43-a8a2-a681651268f8",\n",
+       "    "created": "2017-11-13T14:43:34.857Z",\n",
+       "    "modified": "2017-11-13T14:43:34.857Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "description": "A ransomware related to ...",\n",
        "    "labels": [\n",
@@ -609,7 +609,7 @@
        "    ],\n",
        "    "granular_markings": [\n",
        "        {\n",
-       "            "marking_ref": "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53",\n",
+       "            "marking_ref": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
        "            "selectors": [\n",
        "                "description"\n",
        "            ]\n",
@@ -628,7 +628,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1195,7 +1195,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -1204,13 +1204,15 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "malware.get_markings('name')"
+    "from stix2 import get_markings\n",
+    "\n",
+    "get_markings(malware, 'name')"
    ]
   },
   {

From f9ad7ceb6516bd7ff998c5bb12442fab5b113dbc Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 2 Nov 2017 16:18:41 -0400
Subject: [PATCH 59/90] Add relationships() function to Environment

---
 stix2/environment.py           | 40 ++++++++++++++++++
 stix2/test/constants.py        |  7 ++++
 stix2/test/test_environment.py | 75 +++++++++++++++++++++++++++++++++-
 3 files changed, 120 insertions(+), 2 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 4919335..6a8250f 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -2,6 +2,7 @@ import copy
 
 from .core import parse as _parse
 from .sources import CompositeDataSource, DataStore
+from .sources.filters import Filter
 
 
 class ObjectFactory(object):
@@ -171,3 +172,42 @@ class Environment(object):
             return self.get(creator_id)
         else:
             return None
+
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve Relationships involving the given STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            List of Relationship objects involving the given STIX object.
+
+        """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        if not target_only:
+            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
+        if not source_only:
+            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
diff --git a/stix2/test/constants.py b/stix2/test/constants.py
index 839b547..d8b66fe 100644
--- a/stix2/test/constants.py
+++ b/stix2/test/constants.py
@@ -29,6 +29,13 @@ MARKING_IDS = [
     "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f",
 ]
 
+# All required args for a Campaign instance
+CAMPAIGN_KWARGS = dict(
+    name="Green Group Attacks Against Finance",
+    description="Campaign by Green Group against a series of targets in the financial services sector.",
+)
+
+
 # All required args for a Campaign instance, plus some optional args
 CAMPAIGN_MORE_KWARGS = dict(
     type='campaign',
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index c669a33..086e446 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -2,8 +2,26 @@ import pytest
 
 import stix2
 
-from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID,
-                        INDICATOR_KWARGS, MALWARE_ID)
+from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, FAKE_TIME, IDENTITY_ID,
+                        IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
+                        MALWARE_ID, MALWARE_KWARGS)
+
+RELATIONSHIP_ID1 = 'relationship--06520621-5352-4e6a-b976-e8fa3d437ffd'
+RELATIONSHIP_ID2 = 'relationship--181c9c09-43e6-45dd-9374-3bec192f05ef'
+RELATIONSHIP_ID3 = 'relationship--a0cbb21c-8daf-4a7f-96aa-7155a4ef8f70'
+
+
+@pytest.fixture
+def ds():
+    cam = stix2.Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    idy = stix2.Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
+    mal = stix2.Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    rel1 = stix2.Relationship(ind, 'indicates', mal, id=RELATIONSHIP_ID1)
+    rel2 = stix2.Relationship(mal, 'targets', idy, id=RELATIONSHIP_ID2)
+    rel3 = stix2.Relationship(cam, 'uses', mal, id=RELATIONSHIP_ID3)
+    stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
+    yield stix2.MemoryStore(stix_objs)
 
 
 def test_object_factory_created_by_ref_str():
@@ -216,3 +234,56 @@ def test_created_by_not_found():
     ind = env.create(stix2.Indicator, **INDICATOR_KWARGS)
     creator = env.creator_of(ind)
     assert creator is None
+
+
+def test_relationships(ds):
+    env = stix2.Environment(store=ds)
+    mal = env.get(MALWARE_ID)
+    resp = env.relationships(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == RELATIONSHIP_ID1 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_ID2 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+
+
+def test_relationships_by_type(ds):
+    env = stix2.Environment(store=ds)
+    mal = env.get(MALWARE_ID)
+    resp = env.relationships(mal, relationship_type='indicates')
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_ID1
+
+
+def test_relationships_by_source(ds):
+    env = stix2.Environment(store=ds)
+    resp = env.relationships(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_ID2
+
+
+def test_relationships_by_target(ds):
+    env = stix2.Environment(store=ds)
+    resp = env.relationships(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == RELATIONSHIP_ID1 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+
+
+def test_relationships_by_target_and_type(ds):
+    env = stix2.Environment(store=ds)
+    resp = env.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+
+
+def test_relationships_by_target_and_source(ds):
+    env = stix2.Environment(store=ds)
+    with pytest.raises(ValueError) as excinfo:
+        env.relationships(MALWARE_ID, target_only=True, source_only=True)
+
+    assert 'not both' in str(excinfo.value)

From 2dc8a97a51889c46521713638164436a5c49f30c Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 13 Nov 2017 14:14:24 -0500
Subject: [PATCH 60/90] Fix documentation links for v20

---
 docs/api/stix2.common.rst          | 5 -----
 docs/api/stix2.observables.rst     | 5 -----
 docs/api/stix2.sdo.rst             | 5 -----
 docs/api/stix2.sro.rst             | 5 -----
 docs/api/stix2.v20.common.rst      | 5 +++++
 docs/api/stix2.v20.observables.rst | 5 +++++
 docs/api/stix2.v20.sdo.rst         | 5 +++++
 docs/api/stix2.v20.sro.rst         | 5 +++++
 docs/api_ref.rst                   | 5 +++++
 docs/guide/creating.ipynb          | 2 +-
 docs/guide/custom.ipynb            | 6 +++---
 docs/guide/ts_support.ipynb        | 2 +-
 stix2/__init__.py                  | 8 ++++----
 13 files changed, 34 insertions(+), 29 deletions(-)
 delete mode 100644 docs/api/stix2.common.rst
 delete mode 100644 docs/api/stix2.observables.rst
 delete mode 100644 docs/api/stix2.sdo.rst
 delete mode 100644 docs/api/stix2.sro.rst
 create mode 100644 docs/api/stix2.v20.common.rst
 create mode 100644 docs/api/stix2.v20.observables.rst
 create mode 100644 docs/api/stix2.v20.sdo.rst
 create mode 100644 docs/api/stix2.v20.sro.rst

diff --git a/docs/api/stix2.common.rst b/docs/api/stix2.common.rst
deleted file mode 100644
index 9e90b03..0000000
--- a/docs/api/stix2.common.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-common
-============
-
-.. automodule:: stix2.common
-   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.observables.rst b/docs/api/stix2.observables.rst
deleted file mode 100644
index 2d19996..0000000
--- a/docs/api/stix2.observables.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-observables
-=================
-
-.. automodule:: stix2.observables
-   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.sdo.rst b/docs/api/stix2.sdo.rst
deleted file mode 100644
index 9448ff4..0000000
--- a/docs/api/stix2.sdo.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-sdo
-=========
-
-.. automodule:: stix2.sdo
-   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.sro.rst b/docs/api/stix2.sro.rst
deleted file mode 100644
index a63af76..0000000
--- a/docs/api/stix2.sro.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-sro
-=========
-
-.. automodule:: stix2.sro
-   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.v20.common.rst b/docs/api/stix2.v20.common.rst
new file mode 100644
index 0000000..0c7a296
--- /dev/null
+++ b/docs/api/stix2.v20.common.rst
@@ -0,0 +1,5 @@
+common
+================
+
+.. automodule:: stix2.v20.common
+   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.v20.observables.rst b/docs/api/stix2.v20.observables.rst
new file mode 100644
index 0000000..d31f75f
--- /dev/null
+++ b/docs/api/stix2.v20.observables.rst
@@ -0,0 +1,5 @@
+observables
+=====================
+
+.. automodule:: stix2.v20.observables
+   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.v20.sdo.rst b/docs/api/stix2.v20.sdo.rst
new file mode 100644
index 0000000..c4c97f8
--- /dev/null
+++ b/docs/api/stix2.v20.sdo.rst
@@ -0,0 +1,5 @@
+sdo
+=============
+
+.. automodule:: stix2.v20.sdo
+   :members:
\ No newline at end of file
diff --git a/docs/api/stix2.v20.sro.rst b/docs/api/stix2.v20.sro.rst
new file mode 100644
index 0000000..379ed18
--- /dev/null
+++ b/docs/api/stix2.v20.sro.rst
@@ -0,0 +1,5 @@
+sro
+=============
+
+.. automodule:: stix2.v20.sro
+   :members:
\ No newline at end of file
diff --git a/docs/api_ref.rst b/docs/api_ref.rst
index 11bf278..bb60081 100644
--- a/docs/api_ref.rst
+++ b/docs/api_ref.rst
@@ -4,4 +4,9 @@ API Reference
 This section of documentation contains information on all of the classes and
 functions in the ``stix2`` API, as given by the package's docstrings.
 
+.. note::
+    All the classes and functions detailed in the pages below are importable
+    directly from `stix2`. See also:
+    :ref:`How imports will work `.
+
 .. automodule:: stix2
diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb
index f9ff11d..5836d1d 100644
--- a/docs/guide/creating.ipynb
+++ b/docs/guide/creating.ipynb
@@ -502,7 +502,7 @@
    "source": [
     "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided.\n",
     "\n",
-    "You can see the full list of SDO classes [here](../api/stix2.sdo.rst)."
+    "You can see the full list of SDO classes [here](../api/stix2.v20.sdo.rst)."
    ]
   },
   {
diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb
index 9badeed..7f30401 100644
--- a/docs/guide/custom.ipynb
+++ b/docs/guide/custom.ipynb
@@ -357,7 +357,7 @@
    "source": [
     "### Custom STIX Object Types\n",
     "\n",
-    "To create a custom STIX object type, define a class with the @[CustomObject](../api/stix2.sdo.rst#stix2.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n",
+    "To create a custom STIX object type, define a class with the @[CustomObject](../api/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n",
     "\n",
     "Let's say zoo animals have become a serious cyber threat and we want to model them in STIX using a custom object type. Let's use a ``species`` property to store the kind of animal, and make that property required. We also want a property to store the class of animal, such as \"mammal\" or \"bird\" but only want to allow specific values in it. We can add some logic to validate this property in ``__init__``."
    ]
@@ -588,7 +588,7 @@
    "source": [
     "### Custom Cyber Observable Types\n",
     "\n",
-    "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/stix2.observables.rst#stix2.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary."
+    "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary."
    ]
   },
   {
@@ -754,7 +754,7 @@
    "source": [
     "### Custom Cyber Observable Extensions\n",
     "\n",
-    "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/stix2.observables.rst#stix2.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:"
+    "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:"
    ]
   },
   {
diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb
index 092f91d..6ceef99 100644
--- a/docs/guide/ts_support.ipynb
+++ b/docs/guide/ts_support.ipynb
@@ -364,7 +364,7 @@
    "source": [
     "### How will custom content work\n",
     "\n",
-    "[CustomObject](../api/stix2.sdo.rst#stix2.sdo.CustomObject), [CustomObservable](../api/stix2.observables.rst#stix2.observables.CustomObservable), [CustomMarking](../api/stix2.common.rst#stix2.common.CustomMarking) and [CustomExtension](../api/stix2.observables.rst#stix2.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n",
+    "[CustomObject](../api/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject), [CustomObservable](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable), [CustomMarking](../api/stix2.v20.common.rst#stix2.v20.common.CustomMarking) and [CustomExtension](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n",
     "\n",
     "You can perform this by:"
    ]
diff --git a/stix2/__init__.py b/stix2/__init__.py
index 7ba3e99..9ce9378 100644
--- a/stix2/__init__.py
+++ b/stix2/__init__.py
@@ -3,17 +3,17 @@
 .. autosummary::
    :toctree: api
 
-   common
+   v20.common
    core
    environment
    exceptions
    markings
-   observables
+   v20.observables
    patterns
    properties
-   sdo
+   v20.sdo
    sources
-   sro
+   v20.sro
    utils
 """
 

From 4aabcfcda26669c84ddbfbcde4f69a6fcd74bdad Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 13 Nov 2017 16:36:43 -0500
Subject: [PATCH 61/90] Update CHANGELOG for v0.4.0

---
 CHANGELOG | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 650d372..c5ffabf 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,15 @@
 CHANGELOG
 =========
 
+0.4.0 - 2017-11-13
+
+* Adds `creator_of` function to easily get the Identity that created an object,
+  and a `serialize` function for fast serialization without sorting properties.
+* Fixes bugs with DataStores and with adding custom STIX content to Bundles.
+* Supports filtering on any property, not just common properties.
+* Includes internal changes to make it easier to support multiple versions of
+  the STIX specification.
+
 0.3.0 - 2017-10-06
 
 * Adds data stores, object factory, and the environment layer.

From ef3ce9f6f0a678e604bda63f9eb5742ed53f8eff Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 13 Nov 2017 16:40:58 -0500
Subject: [PATCH 62/90] =?UTF-8?q?Bump=20version:=200.3.0=20=E2=86=92=200.4?=
 =?UTF-8?q?.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/conf.py     | 4 ++--
 setup.cfg        | 2 +-
 stix2/version.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 6155be6..c337e79 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -28,8 +28,8 @@ project = 'stix2'
 copyright = '2017, OASIS Open'
 author = 'OASIS Open'
 
-version = '0.3.0'
-release = '0.3.0'
+version = '0.4.0'
+release = '0.4.0'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints']
diff --git a/setup.cfg b/setup.cfg
index 5edf45c..76162b5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.4.0
 commit = True
 tag = True
 
diff --git a/stix2/version.py b/stix2/version.py
index 493f741..6a9beea 100644
--- a/stix2/version.py
+++ b/stix2/version.py
@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"

From 55cf00d7f0f5e0703c80a88844dddcfb4c1b4fdd Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 15 Nov 2017 10:37:17 -0500
Subject: [PATCH 63/90] Move relationships() to DataSources

---
 stix2/environment.py        | 41 ++---------------
 stix2/sources/__init__.py   | 88 +++++++++++++++++++++++++++++++++++++
 stix2/sources/filesystem.py | 39 ++++++++++++++++
 stix2/sources/memory.py     | 39 ++++++++++++++++
 stix2/sources/taxii.py      | 39 ++++++++++++++++
 5 files changed, 209 insertions(+), 37 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 6a8250f..b018c01 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -2,7 +2,6 @@ import copy
 
 from .core import parse as _parse
 from .sources import CompositeDataSource, DataStore
-from .sources.filters import Filter
 
 
 class ObjectFactory(object):
@@ -173,41 +172,9 @@ class Environment(object):
         else:
             return None
 
-    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
-        """Retrieve Relationships involving the given STIX object.
-
-        Only one of `source_only` and `target_only` may be `True`.
-
-        Args:
-            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
-                relationships will be looked up.
-            relationship_type (str): Only retrieve Relationships of this type.
-            source_only (bool): Only retrieve Relationships for which this
-                object is the source_ref. Default: False.
-            target_only (bool): Only retrieve Relationships for which this
-                object is the target_ref. Default: False.
-
-        Returns:
-            List of Relationship objects involving the given STIX object.
-
-        """
-        results = []
-        filters = [Filter('type', '=', 'relationship')]
-
+    def relationships(self, *args, **kwargs):
         try:
-            obj_id = obj.get('id', '')
+            return self.source.relationships(*args, **kwargs)
         except AttributeError:
-            obj_id = obj
-
-        if relationship_type:
-            filters.append(Filter('relationship_type', '=', relationship_type))
-
-        if source_only and target_only:
-            raise ValueError("Search either source only or target only, but not both")
-
-        if not target_only:
-            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
-        if not source_only:
-            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
-
-        return results
+            raise AttributeError('Environment has no data source')
+    relationships.__doc__ = DataStore.relationships.__doc__
diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index b3e8a29..22f4027 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -16,6 +16,7 @@ import uuid
 
 from six import with_metaclass
 
+from stix2.sources.filters import Filter
 from stix2.utils import deduplicate
 
 
@@ -89,6 +90,28 @@ class DataStore(object):
         """
         return self.source.query(*args, **kwargs)
 
+    def relationships(self, *args, **kwargs):
+        """Retrieve Relationships involving the given STIX object.
+
+        Translate relationships() call to the appropriate DataSource call.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+        return self.source.relationships(*args, **kwargs)
+
     def add(self, *args, **kwargs):
         """Method for storing STIX objects.
 
@@ -191,6 +214,28 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
+    @abstractmethod
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """
+        Implement: The specific data source API calls, processing,
+        functionality required for dereferencing relationships.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+
 
 class CompositeDataSource(DataSource):
     """Controller for all the attached DataSources.
@@ -354,6 +399,49 @@ class CompositeDataSource(DataSource):
 
         return all_data
 
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve Relationships involving the given STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Federated relationships retrieve method - iterates through all
+        DataSources defined in "data_sources".
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        for ds in self.data_sources:
+            if not target_only:
+                results.extend(ds.query(filters + [Filter('source_ref', '=', obj_id)]))
+            if not source_only:
+                results.extend(ds.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
+
     def add_data_source(self, data_source):
         """Attach a DataSource to CompositeDataSource instance
 
diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py
index e92c525..db22faa 100644
--- a/stix2/sources/filesystem.py
+++ b/stix2/sources/filesystem.py
@@ -308,6 +308,45 @@ class FileSystemSource(DataSource):
 
         return stix_objs
 
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve Relationships involving the given STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        if not target_only:
+            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
+        if not source_only:
+            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
+
     def _parse_file_filters(self, query):
         """Extract STIX common filters.
 
diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py
index 308d0d0..af0dd02 100644
--- a/stix2/sources/memory.py
+++ b/stix2/sources/memory.py
@@ -301,6 +301,45 @@ class MemorySource(DataSource):
 
         return all_data
 
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve Relationships involving the given STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        if not target_only:
+            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
+        if not source_only:
+            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
+
     def load_from_file(self, file_path, allow_custom=False, version=None):
         file_path = os.path.abspath(file_path)
         stix_data = json.load(open(file_path, "r"))
diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py
index 8eb5069..257bbd5 100644
--- a/stix2/sources/taxii.py
+++ b/stix2/sources/taxii.py
@@ -222,6 +222,45 @@ class TAXIICollectionSource(DataSource):
 
         return stix_objs
 
+    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve Relationships involving the given STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                relationships will be looked up.
+            relationship_type (str): Only retrieve Relationships of this type.
+            source_only (bool): Only retrieve Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only retrieve Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of Relationship objects involving the given STIX object.
+
+        """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        if not target_only:
+            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
+        if not source_only:
+            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
+
     def _parse_taxii_filters(self, query):
         """Parse out TAXII filters that the TAXII server can filter on.
 

From fbce8f15fe5278c06a79f0cadb6ac624d9fbe8cf Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Wed, 15 Nov 2017 12:55:34 -0500
Subject: [PATCH 64/90] Allow no custom __init__() on CustomMarking

---
 stix2/v20/common.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/stix2/v20/common.py b/stix2/v20/common.py
index 2d15529..ef45060 100644
--- a/stix2/v20/common.py
+++ b/stix2/v20/common.py
@@ -145,7 +145,14 @@ def CustomMarking(type='x-custom-marking', properties=None):
 
             def __init__(self, **kwargs):
                 _STIXBase.__init__(self, **kwargs)
-                cls.__init__(self, **kwargs)
+                try:
+                    cls.__init__(self, **kwargs)
+                except (AttributeError, TypeError) as e:
+                    # Don't accidentally catch errors raised in a custom __init__()
+                    if ("has no attribute '__init__'" in str(e) or
+                            str(e) == "object.__init__() takes no parameters"):
+                        return
+                    raise e
 
         _register_marking(_Custom)
         return _Custom

From e6a8b555d343af706a3ff0fef591fd1035a79adc Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Wed, 15 Nov 2017 13:12:00 -0500
Subject: [PATCH 65/90] Add test for CustomMarking. closes #109

---
 stix2/test/test_custom.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index 92d5d4c..ecee1cd 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -94,6 +94,26 @@ def test_custom_property_in_bundled_object():
     assert '"x_foo": "bar"' in str(bundle)
 
 
+def test_custom_marking_no_init():
+    @stix2.CustomMarking('x-new-obj', [
+        ('property1', stix2.properties.StringProperty(required=True)),
+    ])
+    class NewObj():
+        pass
+
+    no = NewObj(property1='something')
+    assert no.property1 == 'something'
+
+    @stix2.CustomMarking('x-new-obj2', [
+        ('property1', stix2.properties.StringProperty(required=True)),
+    ])
+    class NewObj2(object):
+        pass
+
+    no2 = NewObj2(property1='something')
+    assert no2.property1 == 'something'
+
+
 @stix2.sdo.CustomObject('x-new-type', [
     ('property1', stix2.properties.StringProperty(required=True)),
     ('property2', stix2.properties.IntegerProperty()),

From 1b816c3d84b2bfcc908241b801117d9c342a06cd Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 15 Nov 2017 16:34:07 -0500
Subject: [PATCH 66/90] Test relationships() functions

---
 stix2/test/constants.py        |  5 +++
 stix2/test/test_environment.py | 28 ++++++-------
 stix2/test/test_filesystem.py  | 73 +++++++++++++++++++++++++++++++++-
 stix2/test/test_memory.py      | 69 +++++++++++++++++++++++++++++++-
 4 files changed, 156 insertions(+), 19 deletions(-)

diff --git a/stix2/test/constants.py b/stix2/test/constants.py
index d8b66fe..3db39d6 100644
--- a/stix2/test/constants.py
+++ b/stix2/test/constants.py
@@ -28,6 +28,11 @@ MARKING_IDS = [
     "marking-definition--68520ae2-fefe-43a9-84ee-2c2a934d2c7d",
     "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f",
 ]
+RELATIONSHIP_IDS = [
+    'relationship--06520621-5352-4e6a-b976-e8fa3d437ffd',
+    'relationship--181c9c09-43e6-45dd-9374-3bec192f05ef',
+    'relationship--a0cbb21c-8daf-4a7f-96aa-7155a4ef8f70'
+]
 
 # All required args for a Campaign instance
 CAMPAIGN_KWARGS = dict(
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 086e446..e595eb9 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -4,11 +4,7 @@ import stix2
 
 from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, FAKE_TIME, IDENTITY_ID,
                         IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
-                        MALWARE_ID, MALWARE_KWARGS)
-
-RELATIONSHIP_ID1 = 'relationship--06520621-5352-4e6a-b976-e8fa3d437ffd'
-RELATIONSHIP_ID2 = 'relationship--181c9c09-43e6-45dd-9374-3bec192f05ef'
-RELATIONSHIP_ID3 = 'relationship--a0cbb21c-8daf-4a7f-96aa-7155a4ef8f70'
+                        MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
 
 
 @pytest.fixture
@@ -17,9 +13,9 @@ def ds():
     idy = stix2.Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
     ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
     mal = stix2.Malware(id=MALWARE_ID, **MALWARE_KWARGS)
-    rel1 = stix2.Relationship(ind, 'indicates', mal, id=RELATIONSHIP_ID1)
-    rel2 = stix2.Relationship(mal, 'targets', idy, id=RELATIONSHIP_ID2)
-    rel3 = stix2.Relationship(cam, 'uses', mal, id=RELATIONSHIP_ID3)
+    rel1 = stix2.Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
+    rel2 = stix2.Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
+    rel3 = stix2.Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
     stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
     yield stix2.MemoryStore(stix_objs)
 
@@ -242,9 +238,9 @@ def test_relationships(ds):
     resp = env.relationships(mal)
 
     assert len(resp) == 3
-    assert any(x['id'] == RELATIONSHIP_ID1 for x in resp)
-    assert any(x['id'] == RELATIONSHIP_ID2 for x in resp)
-    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
 
 
 def test_relationships_by_type(ds):
@@ -253,7 +249,7 @@ def test_relationships_by_type(ds):
     resp = env.relationships(mal, relationship_type='indicates')
 
     assert len(resp) == 1
-    assert resp[0]['id'] == RELATIONSHIP_ID1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[0]
 
 
 def test_relationships_by_source(ds):
@@ -261,7 +257,7 @@ def test_relationships_by_source(ds):
     resp = env.relationships(MALWARE_ID, source_only=True)
 
     assert len(resp) == 1
-    assert resp[0]['id'] == RELATIONSHIP_ID2
+    assert resp[0]['id'] == RELATIONSHIP_IDS[1]
 
 
 def test_relationships_by_target(ds):
@@ -269,8 +265,8 @@ def test_relationships_by_target(ds):
     resp = env.relationships(MALWARE_ID, target_only=True)
 
     assert len(resp) == 2
-    assert any(x['id'] == RELATIONSHIP_ID1 for x in resp)
-    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
 
 
 def test_relationships_by_target_and_type(ds):
@@ -278,7 +274,7 @@ def test_relationships_by_target_and_type(ds):
     resp = env.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
 
     assert len(resp) == 1
-    assert any(x['id'] == RELATIONSHIP_ID3 for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
 
 
 def test_relationships_by_target_and_source(ds):
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 85f6966..66ca47f 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -4,7 +4,12 @@ import shutil
 import pytest
 
 from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink,
-                   FileSystemSource, FileSystemStore, Filter, properties)
+                   FileSystemSource, FileSystemStore, Filter, Identity,
+                   Indicator, Malware, Relationship, properties)
+
+from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
+                        IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
+                        MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
 
 FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data")
 
@@ -40,6 +45,25 @@ def fs_sink():
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
 
+@pytest.fixture
+def rel_fs_store():
+    cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
+    mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
+    rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
+    rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
+    stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
+    fs = FileSystemStore(FS_PATH)
+    for o in stix_objs:
+        fs.add(o)
+    yield fs
+
+    for o in stix_objs:
+        os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
+
+
 def test_filesystem_source_nonexistent_folder():
     with pytest.raises(ValueError) as excinfo:
         FileSystemSource('nonexistent-folder')
@@ -375,3 +399,50 @@ def test_filesystem_custom_object(fs_store):
 
     # remove dir
     shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True)
+
+
+def test_relationships(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.relationships(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_type(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.relationships(mal, relationship_type='indicates')
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[0]
+
+
+def test_relationships_by_source(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[1]
+
+
+def test_relationships_by_target(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_type(rel_fs_store):
+    resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_source(rel_fs_store):
+    with pytest.raises(ValueError) as excinfo:
+        rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True)
+
+    assert 'not both' in str(excinfo.value)
diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index 6b1219e..d26b7d3 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -3,10 +3,15 @@ import shutil
 
 import pytest
 
-from stix2 import (Bundle, Campaign, CustomObject, Filter, MemorySource,
-                   MemoryStore, properties)
+from stix2 import (Bundle, Campaign, CustomObject, Filter, Identity, Indicator,
+                   Malware, MemorySource, MemoryStore, Relationship,
+                   properties)
 from stix2.sources import make_id
 
+from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID,
+                        IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS,
+                        MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS)
+
 IND1 = {
     "created": "2017-01-27T13:49:53.935Z",
     "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
@@ -118,6 +123,19 @@ def mem_source():
     yield MemorySource(STIX_OBJS1)
 
 
+@pytest.fixture
+def rel_mem_store():
+    cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
+    idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
+    ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
+    mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
+    rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
+    rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
+    rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
+    stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
+    yield MemoryStore(stix_objs)
+
+
 def test_memory_source_get(mem_source):
     resp = mem_source.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")
     assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"
@@ -287,3 +305,50 @@ def test_memory_store_custom_object(mem_store):
     newobj_r = mem_store.get(newobj.id)
     assert newobj_r.id == newobj.id
     assert newobj_r.property1 == 'something'
+
+
+def test_relationships(rel_mem_store):
+    mal = rel_mem_store.get(MALWARE_ID)
+    resp = rel_mem_store.relationships(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_type(rel_mem_store):
+    mal = rel_mem_store.get(MALWARE_ID)
+    resp = rel_mem_store.relationships(mal, relationship_type='indicates')
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[0]
+
+
+def test_relationships_by_source(rel_mem_store):
+    resp = rel_mem_store.relationships(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == RELATIONSHIP_IDS[1]
+
+
+def test_relationships_by_target(rel_mem_store):
+    resp = rel_mem_store.relationships(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp)
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_type(rel_mem_store):
+    resp = rel_mem_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
+
+
+def test_relationships_by_target_and_source(rel_mem_store):
+    with pytest.raises(ValueError) as excinfo:
+        rel_mem_store.relationships(MALWARE_ID, target_only=True, source_only=True)
+
+    assert 'not both' in str(excinfo.value)

From 86f28644f92a9df820e36a511aa5fe748cd73d90 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Wed, 15 Nov 2017 16:56:55 -0500
Subject: [PATCH 67/90] Fix Python2 bug when filter value is unicode

---
 stix2/sources/filters.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py
index 5772112..5af48cd 100644
--- a/stix2/sources/filters.py
+++ b/stix2/sources/filters.py
@@ -10,6 +10,11 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
 
 """Supported filter value types"""
 FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
+try:
+    FILTER_VALUE_TYPES.append(unicode)
+except NameError:
+    # Python 3 doesn't need to worry about unicode
+    pass
 
 
 def _check_filter_components(prop, op, value):

From 29dec997a06f000622d9957e4a7a7b15494b7e8a Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 16 Nov 2017 14:58:59 -0500
Subject: [PATCH 68/90] Add related_to()

Function for calling relationships() but instead of just returning the
Relationship objects, returns the STIX objects being refered to in those
Relationships.
---
 stix2/environment.py           |  7 ++++
 stix2/sources/__init__.py      | 66 ++++++++++++++++++++++++++++++++++
 stix2/test/test_environment.py | 28 +++++++++++++++
 stix2/test/test_filesystem.py  | 25 +++++++++++++
 stix2/test/test_memory.py      | 26 ++++++++++++++
 5 files changed, 152 insertions(+)

diff --git a/stix2/environment.py b/stix2/environment.py
index b018c01..3f0742b 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -178,3 +178,10 @@ class Environment(object):
         except AttributeError:
             raise AttributeError('Environment has no data source')
     relationships.__doc__ = DataStore.relationships.__doc__
+
+    def related_to(self, *args, **kwargs):
+        try:
+            return self.source.related_to(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('Environment has no data source')
+    related_to.__doc__ = DataStore.related_to.__doc__
diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index 22f4027..7afe974 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -112,6 +112,30 @@ class DataStore(object):
         """
         return self.source.relationships(*args, **kwargs)
 
+    def related_to(self, *args, **kwargs):
+        """Retrieve STIX Objects that have a Relationship involving the given
+        STIX object.
+
+        Translate related_to() call to the appropriate DataSource call.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                related objects will be looked up.
+            relationship_type (str): Only retrieve objects related by this
+                Relationships type.
+            source_only (bool): Only examine Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only examine Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of STIX objects related to the given STIX object.
+
+        """
+        return self.source.related_to(*args, **kwargs)
+
     def add(self, *args, **kwargs):
         """Method for storing STIX objects.
 
@@ -236,6 +260,48 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
+    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve STIX Objects that have a Relationship involving the given
+        STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                related objects will be looked up.
+            relationship_type (str): Only retrieve objects related by this
+                Relationships type.
+            source_only (bool): Only examine Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only examine Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of STIX objects related to the given STIX object.
+
+        """
+        results = []
+        rels = self.relationships(obj, relationship_type, source_only, target_only)
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        for r in rels:
+            if not source_only:
+                # relationships() found relationships where target_ref is obj_id
+                source_id = r.source_ref
+                if source_id != obj_id:  # needed if target_only is also false
+                    results.append(self.get(source_id))
+            if not target_only:
+                # relationships() found relationships where source_ref is obj_id
+                target_id = r.target_ref
+                if target_id != obj_id:  # needed if source_only is also false
+                    results.append(self.get(target_id))
+
+        return results
+
 
 class CompositeDataSource(DataSource):
     """Controller for all the attached DataSources.
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index e595eb9..46b49c6 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -283,3 +283,31 @@ def test_relationships_by_target_and_source(ds):
         env.relationships(MALWARE_ID, target_only=True, source_only=True)
 
     assert 'not both' in str(excinfo.value)
+
+
+def test_related_to(ds):
+    env = stix2.Environment(store=ds)
+    mal = env.get(MALWARE_ID)
+    resp = env.related_to(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_source(ds):
+    env = stix2.Environment(store=ds)
+    resp = env.related_to(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert resp[0]['id'] == IDENTITY_ID
+
+
+def test_related_to_by_target(ds):
+    env = stix2.Environment(store=ds)
+    resp = env.related_to(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 66ca47f..455132f 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -446,3 +446,28 @@ def test_relationships_by_target_and_source(rel_fs_store):
         rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True)
 
     assert 'not both' in str(excinfo.value)
+
+
+def test_related_to(rel_fs_store):
+    mal = rel_fs_store.get(MALWARE_ID)
+    resp = rel_fs_store.related_to(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_source(rel_fs_store):
+    resp = rel_fs_store.related_to(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_target(rel_fs_store):
+    resp = rel_fs_store.related_to(MALWARE_ID, target_only=True)
+
+    assert len(resp) == 2
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index d26b7d3..0096916 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -352,3 +352,29 @@ def test_relationships_by_target_and_source(rel_mem_store):
         rel_mem_store.relationships(MALWARE_ID, target_only=True, source_only=True)
 
     assert 'not both' in str(excinfo.value)
+
+
+def test_related_to(rel_mem_store):
+    mal = rel_mem_store.get(MALWARE_ID)
+    resp = rel_mem_store.related_to(mal)
+
+    assert len(resp) == 3
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_source(rel_mem_store):
+    resp = rel_mem_store.related_to(MALWARE_ID, source_only=True)
+
+    assert len(resp) == 1
+    assert any(x['id'] == IDENTITY_ID for x in resp)
+
+
+def test_related_to_by_target(rel_mem_store):
+    resp = rel_mem_store.related_to(MALWARE_ID, target_only=True)
+    print(resp)
+
+    assert len(resp) == 2
+    assert any(x['id'] == CAMPAIGN_ID for x in resp)
+    assert any(x['id'] == INDICATOR_ID for x in resp)

From f0331f8b9bb3ef0c6ac5ca9426614504b7774420 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 16 Nov 2017 16:25:57 -0500
Subject: [PATCH 69/90] Reorganize DataStore code for dereferencing

- Move `relationships()` to DataStore like `related_to()` is. If a
  DataStore implementation needs a different way to handle relationship
  dereferencing (e.g. TAXII in the future, or CompositeDataSource), it
  can overwrite these functions.
- Reduce code duplication.
- Check for presence of Data Source/Sink in all DataStores, not just in
  Environment.
---
 stix2/environment.py        |  51 ++---------------
 stix2/sources/__init__.py   | 108 ++++++++++++++++++++++++++++++++----
 stix2/sources/filesystem.py |  39 -------------
 stix2/sources/memory.py     |  39 -------------
 stix2/sources/taxii.py      |  39 -------------
 5 files changed, 104 insertions(+), 172 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 3f0742b..5bb876b 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -105,30 +105,12 @@ class Environment(object):
         return self.factory.create(*args, **kwargs)
     create.__doc__ = ObjectFactory.create.__doc__
 
-    def get(self, *args, **kwargs):
-        try:
-            return self.source.get(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source to query')
-    get.__doc__ = DataStore.get.__doc__
-
-    def all_versions(self, *args, **kwargs):
-        """Retrieve all versions of a single STIX object by ID.
-        """
-        try:
-            return self.source.all_versions(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source to query')
-    all_versions.__doc__ = DataStore.all_versions.__doc__
-
-    def query(self, *args, **kwargs):
-        """Retrieve STIX objects matching a set of filters.
-        """
-        try:
-            return self.source.query(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source to query')
-    query.__doc__ = DataStore.query.__doc__
+    get = DataStore.__dict__['get']
+    all_versions = DataStore.__dict__['all_versions']
+    query = DataStore.__dict__['query']
+    relationships = DataStore.__dict__['relationships']
+    related_to = DataStore.__dict__['related_to']
+    add = DataStore.__dict__['add']
 
     def add_filters(self, *args, **kwargs):
         try:
@@ -142,13 +124,6 @@ class Environment(object):
         except AttributeError:
             raise AttributeError('Environment has no data source')
 
-    def add(self, *args, **kwargs):
-        try:
-            return self.sink.add(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data sink to put objects in')
-    add.__doc__ = DataStore.add.__doc__
-
     def parse(self, *args, **kwargs):
         return _parse(*args, **kwargs)
     parse.__doc__ = _parse.__doc__
@@ -171,17 +146,3 @@ class Environment(object):
             return self.get(creator_id)
         else:
             return None
-
-    def relationships(self, *args, **kwargs):
-        try:
-            return self.source.relationships(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
-    relationships.__doc__ = DataStore.relationships.__doc__
-
-    def related_to(self, *args, **kwargs):
-        try:
-            return self.source.related_to(*args, **kwargs)
-        except AttributeError:
-            raise AttributeError('Environment has no data source')
-    related_to.__doc__ = DataStore.related_to.__doc__
diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index 7afe974..d2b85d0 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -59,7 +59,10 @@ class DataStore(object):
                 object specified by the "id".
 
         """
-        return self.source.get(*args, **kwargs)
+        try:
+            return self.source.get(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
     def all_versions(self, *args, **kwargs):
         """Retrieve all versions of a single STIX object by ID.
@@ -73,7 +76,10 @@ class DataStore(object):
             stix_objs (list): a list of STIX objects
 
         """
-        return self.source.all_versions(*args, **kwargs)
+        try:
+            return self.source.all_versions(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
     def query(self, *args, **kwargs):
         """Retrieve STIX objects matching a set of filters.
@@ -88,7 +94,10 @@ class DataStore(object):
             stix_objs (list): a list of STIX objects
 
         """
-        return self.source.query(*args, **kwargs)
+        try:
+            return self.source.query(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
     def relationships(self, *args, **kwargs):
         """Retrieve Relationships involving the given STIX object.
@@ -110,7 +119,10 @@ class DataStore(object):
             (list): List of Relationship objects involving the given STIX object.
 
         """
-        return self.source.relationships(*args, **kwargs)
+        try:
+            return self.source.relationships(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
     def related_to(self, *args, **kwargs):
         """Retrieve STIX Objects that have a Relationship involving the given
@@ -134,7 +146,10 @@ class DataStore(object):
             (list): List of STIX objects related to the given STIX object.
 
         """
-        return self.source.related_to(*args, **kwargs)
+        try:
+            return self.source.related_to(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
     def add(self, *args, **kwargs):
         """Method for storing STIX objects.
@@ -146,7 +161,10 @@ class DataStore(object):
             stix_objs (list): a list of STIX objects
 
         """
-        return self.sink.add(*args, **kwargs)
+        try:
+            return self.sink.add(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data sink to put objects in' % self.__class__.__name__)
 
 
 class DataSink(with_metaclass(ABCMeta)):
@@ -238,11 +256,8 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
-    @abstractmethod
     def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
-        """
-        Implement: The specific data source API calls, processing,
-        functionality required for dereferencing relationships.
+        """Retrieve Relationships involving the given STIX object.
 
         Only one of `source_only` and `target_only` may be `True`.
 
@@ -259,6 +274,26 @@ class DataSource(with_metaclass(ABCMeta)):
             (list): List of Relationship objects involving the given STIX object.
 
         """
+        results = []
+        filters = [Filter('type', '=', 'relationship')]
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        if relationship_type:
+            filters.append(Filter('relationship_type', '=', relationship_type))
+
+        if source_only and target_only:
+            raise ValueError("Search either source only or target only, but not both")
+
+        if not target_only:
+            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
+        if not source_only:
+            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
+
+        return results
 
     def related_to(self, obj, relationship_type=None, source_only=False, target_only=False):
         """Retrieve STIX Objects that have a Relationship involving the given
@@ -486,6 +521,9 @@ class CompositeDataSource(DataSource):
             (list): List of Relationship objects involving the given STIX object.
 
         """
+        if not self.has_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
         results = []
         filters = [Filter('type', '=', 'relationship')]
 
@@ -508,6 +546,56 @@ class CompositeDataSource(DataSource):
 
         return results
 
+    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False):
+        """Retrieve STIX Objects that have a Relationship involving the given
+        STIX object.
+
+        Only one of `source_only` and `target_only` may be `True`.
+
+        Federated related objects method - iterates through all
+        DataSources defined in "data_sources".
+
+        Args:
+            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
+                related objects will be looked up.
+            relationship_type (str): Only retrieve objects related by this
+                Relationships type.
+            source_only (bool): Only examine Relationships for which this
+                object is the source_ref. Default: False.
+            target_only (bool): Only examine Relationships for which this
+                object is the target_ref. Default: False.
+
+        Returns:
+            (list): List of STIX objects related to the given STIX object.
+
+        """
+        if not self.has_data_sources():
+            raise AttributeError('CompositeDataSource has no data sources')
+
+        results = []
+        for ds in self.data_sources:
+            rels = ds.relationships(obj, relationship_type, source_only, target_only)
+
+        try:
+            obj_id = obj.get('id', '')
+        except AttributeError:
+            obj_id = obj
+
+        for ds in self.data_sources:
+            for r in rels:
+                if not source_only:
+                    # relationships() found relationships where target_ref is obj_id
+                    source_id = r.source_ref
+                    if source_id != obj_id:  # needed if target_only is also false
+                        results.append(ds.get(source_id))
+                if not target_only:
+                    # relationships() found relationships where source_ref is obj_id
+                    target_id = r.target_ref
+                    if target_id != obj_id:  # needed if source_only is also false
+                        results.append(ds.get(target_id))
+
+        return results
+
     def add_data_source(self, data_source):
         """Attach a DataSource to CompositeDataSource instance
 
diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py
index db22faa..e92c525 100644
--- a/stix2/sources/filesystem.py
+++ b/stix2/sources/filesystem.py
@@ -308,45 +308,6 @@ class FileSystemSource(DataSource):
 
         return stix_objs
 
-    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
-        """Retrieve Relationships involving the given STIX object.
-
-        Only one of `source_only` and `target_only` may be `True`.
-
-        Args:
-            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
-                relationships will be looked up.
-            relationship_type (str): Only retrieve Relationships of this type.
-            source_only (bool): Only retrieve Relationships for which this
-                object is the source_ref. Default: False.
-            target_only (bool): Only retrieve Relationships for which this
-                object is the target_ref. Default: False.
-
-        Returns:
-            (list): List of Relationship objects involving the given STIX object.
-
-        """
-        results = []
-        filters = [Filter('type', '=', 'relationship')]
-
-        try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
-            obj_id = obj
-
-        if relationship_type:
-            filters.append(Filter('relationship_type', '=', relationship_type))
-
-        if source_only and target_only:
-            raise ValueError("Search either source only or target only, but not both")
-
-        if not target_only:
-            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
-        if not source_only:
-            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
-
-        return results
-
     def _parse_file_filters(self, query):
         """Extract STIX common filters.
 
diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py
index af0dd02..308d0d0 100644
--- a/stix2/sources/memory.py
+++ b/stix2/sources/memory.py
@@ -301,45 +301,6 @@ class MemorySource(DataSource):
 
         return all_data
 
-    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
-        """Retrieve Relationships involving the given STIX object.
-
-        Only one of `source_only` and `target_only` may be `True`.
-
-        Args:
-            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
-                relationships will be looked up.
-            relationship_type (str): Only retrieve Relationships of this type.
-            source_only (bool): Only retrieve Relationships for which this
-                object is the source_ref. Default: False.
-            target_only (bool): Only retrieve Relationships for which this
-                object is the target_ref. Default: False.
-
-        Returns:
-            (list): List of Relationship objects involving the given STIX object.
-
-        """
-        results = []
-        filters = [Filter('type', '=', 'relationship')]
-
-        try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
-            obj_id = obj
-
-        if relationship_type:
-            filters.append(Filter('relationship_type', '=', relationship_type))
-
-        if source_only and target_only:
-            raise ValueError("Search either source only or target only, but not both")
-
-        if not target_only:
-            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
-        if not source_only:
-            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
-
-        return results
-
     def load_from_file(self, file_path, allow_custom=False, version=None):
         file_path = os.path.abspath(file_path)
         stix_data = json.load(open(file_path, "r"))
diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py
index 257bbd5..8eb5069 100644
--- a/stix2/sources/taxii.py
+++ b/stix2/sources/taxii.py
@@ -222,45 +222,6 @@ class TAXIICollectionSource(DataSource):
 
         return stix_objs
 
-    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
-        """Retrieve Relationships involving the given STIX object.
-
-        Only one of `source_only` and `target_only` may be `True`.
-
-        Args:
-            obj (STIX object OR dict OR str): The STIX object (or its ID) whose
-                relationships will be looked up.
-            relationship_type (str): Only retrieve Relationships of this type.
-            source_only (bool): Only retrieve Relationships for which this
-                object is the source_ref. Default: False.
-            target_only (bool): Only retrieve Relationships for which this
-                object is the target_ref. Default: False.
-
-        Returns:
-            (list): List of Relationship objects involving the given STIX object.
-
-        """
-        results = []
-        filters = [Filter('type', '=', 'relationship')]
-
-        try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
-            obj_id = obj
-
-        if relationship_type:
-            filters.append(Filter('relationship_type', '=', relationship_type))
-
-        if source_only and target_only:
-            raise ValueError("Search either source only or target only, but not both")
-
-        if not target_only:
-            results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
-        if not source_only:
-            results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
-
-        return results
-
     def _parse_taxii_filters(self, query):
         """Parse out TAXII filters that the TAXII server can filter on.
 

From cfa18bfd0d30393dc104eafd755c3b26fb09be4e Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 16 Nov 2017 16:38:11 -0500
Subject: [PATCH 70/90] Improve Environment tests

---
 stix2/test/test_data_sources.py |  8 ++++++++
 stix2/test/test_environment.py  | 21 ++++++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py
index ef0cf26..d7f238a 100644
--- a/stix2/test/test_data_sources.py
+++ b/stix2/test/test_data_sources.py
@@ -547,3 +547,11 @@ def test_composite_datasource_operations():
     # nothing returns the same as cds1.query(query1) (the associated query is query2)
     results = cds1.query([])
     assert len(results) == 3
+
+
+def test_composite_datastore_no_datasource():
+    cds = CompositeDataSource()
+
+    with pytest.raises(AttributeError) as excinfo:
+        cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
+    assert 'CompositeDataSource has no data source' in str(excinfo.value)
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index 46b49c6..d9e8fff 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -164,6 +164,14 @@ def test_environment_no_datastore():
         env.query(INDICATOR_ID)
     assert 'Environment has no data source' in str(excinfo.value)
 
+    with pytest.raises(AttributeError) as excinfo:
+        env.relationships(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
+    with pytest.raises(AttributeError) as excinfo:
+        env.related_to(INDICATOR_ID)
+    assert 'Environment has no data source' in str(excinfo.value)
+
 
 def test_environment_add_filters():
     env = stix2.Environment(factory=stix2.ObjectFactory())
@@ -200,7 +208,7 @@ def test_parse_malware():
     assert mal.name == "Cryptolocker"
 
 
-def test_created_by():
+def test_creator_of():
     identity = stix2.Identity(**IDENTITY_KWARGS)
     factory = stix2.ObjectFactory(created_by_ref=identity.id)
     env = stix2.Environment(store=stix2.MemoryStore(), factory=factory)
@@ -211,7 +219,7 @@ def test_created_by():
     assert creator is identity
 
 
-def test_created_by_no_datasource():
+def test_creator_of_no_datasource():
     identity = stix2.Identity(**IDENTITY_KWARGS)
     factory = stix2.ObjectFactory(created_by_ref=identity.id)
     env = stix2.Environment(factory=factory)
@@ -222,7 +230,7 @@ def test_created_by_no_datasource():
     assert 'Environment has no data source' in str(excinfo.value)
 
 
-def test_created_by_not_found():
+def test_creator_of_not_found():
     identity = stix2.Identity(**IDENTITY_KWARGS)
     factory = stix2.ObjectFactory(created_by_ref=identity.id)
     env = stix2.Environment(store=stix2.MemoryStore(), factory=factory)
@@ -232,6 +240,13 @@ def test_created_by_not_found():
     assert creator is None
 
 
+def test_creator_of_no_created_by_ref():
+    env = stix2.Environment(store=stix2.MemoryStore())
+    ind = env.create(stix2.Indicator, **INDICATOR_KWARGS)
+    creator = env.creator_of(ind)
+    assert creator is None
+
+
 def test_relationships(ds):
     env = stix2.Environment(store=ds)
     mal = env.get(MALWARE_ID)

From 515ff1e53f39985f5481a12641dd5b19c1bf34fc Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Thu, 16 Nov 2017 16:50:27 -0500
Subject: [PATCH 71/90] Clean up test

---
 stix2/test/test_memory.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py
index 0096916..a7d88a8 100644
--- a/stix2/test/test_memory.py
+++ b/stix2/test/test_memory.py
@@ -373,7 +373,6 @@ def test_related_to_by_source(rel_mem_store):
 
 def test_related_to_by_target(rel_mem_store):
     resp = rel_mem_store.related_to(MALWARE_ID, target_only=True)
-    print(resp)
 
     assert len(resp) == 2
     assert any(x['id'] == CAMPAIGN_ID for x in resp)

From c03ecb5230645d7e6bf27cfa032b94863d43a3eb Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 17 Nov 2017 08:50:40 -0500
Subject: [PATCH 72/90] Update test modules

---
 stix2/test/test_custom.py   | 49 +++++++++++++++++++++++++++++++++----
 stix2/test/test_markings.py | 10 +++++++-
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py
index ecee1cd..7c1832b 100644
--- a/stix2/test/test_custom.py
+++ b/stix2/test/test_custom.py
@@ -94,7 +94,7 @@ def test_custom_property_in_bundled_object():
     assert '"x_foo": "bar"' in str(bundle)
 
 
-def test_custom_marking_no_init():
+def test_custom_marking_no_init_1():
     @stix2.CustomMarking('x-new-obj', [
         ('property1', stix2.properties.StringProperty(required=True)),
     ])
@@ -104,6 +104,8 @@ def test_custom_marking_no_init():
     no = NewObj(property1='something')
     assert no.property1 == 'something'
 
+
+def test_custom_marking_no_init_2():
     @stix2.CustomMarking('x-new-obj2', [
         ('property1', stix2.properties.StringProperty(required=True)),
     ])
@@ -122,6 +124,15 @@ class NewType(object):
     def __init__(self, property2=None, **kwargs):
         if property2 and property2 < 10:
             raise ValueError("'property2' is too small.")
+        if "property3" in kwargs and not isinstance(kwargs.get("property3"), int):
+            raise TypeError("Must be integer!")
+
+
+def test_custom_object_raises_exception():
+    with pytest.raises(TypeError) as excinfo:
+        NewType(property1='something', property3='something', allow_custom=True)
+
+    assert str(excinfo.value) == "Must be integer!"
 
 
 def test_custom_object_type():
@@ -137,7 +148,7 @@ def test_custom_object_type():
     assert "'property2' is too small." in str(excinfo.value)
 
 
-def test_custom_object_no_init():
+def test_custom_object_no_init_1():
     @stix2.sdo.CustomObject('x-new-obj', [
         ('property1', stix2.properties.StringProperty(required=True)),
     ])
@@ -147,6 +158,8 @@ def test_custom_object_no_init():
     no = NewObj(property1='something')
     assert no.property1 == 'something'
 
+
+def test_custom_object_no_init_2():
     @stix2.sdo.CustomObject('x-new-obj2', [
         ('property1', stix2.properties.StringProperty(required=True)),
     ])
@@ -190,23 +203,36 @@ class NewObservable():
     def __init__(self, property2=None, **kwargs):
         if property2 and property2 < 10:
             raise ValueError("'property2' is too small.")
+        if "property3" in kwargs and not isinstance(kwargs.get("property3"), int):
+            raise TypeError("Must be integer!")
 
 
-def test_custom_observable_object():
+def test_custom_observable_object_1():
     no = NewObservable(property1='something')
     assert no.property1 == 'something'
 
+
+def test_custom_observable_object_2():
     with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo:
         NewObservable(property2=42)
     assert excinfo.value.properties == ['property1']
     assert "No values for required properties" in str(excinfo.value)
 
+
+def test_custom_observable_object_3():
     with pytest.raises(ValueError) as excinfo:
         NewObservable(property1='something', property2=4)
     assert "'property2' is too small." in str(excinfo.value)
 
 
-def test_custom_observable_object_no_init():
+def test_custom_observable_raises_exception():
+    with pytest.raises(TypeError) as excinfo:
+        NewObservable(property1='something', property3='something', allow_custom=True)
+
+    assert str(excinfo.value) == "Must be integer!"
+
+
+def test_custom_observable_object_no_init_1():
     @stix2.observables.CustomObservable('x-new-observable', [
         ('property1', stix2.properties.StringProperty()),
     ])
@@ -216,6 +242,8 @@ def test_custom_observable_object_no_init():
     no = NewObs(property1='something')
     assert no.property1 == 'something'
 
+
+def test_custom_observable_object_no_init_2():
     @stix2.observables.CustomObservable('x-new-obs2', [
         ('property1', stix2.properties.StringProperty()),
     ])
@@ -374,6 +402,15 @@ class NewExtension():
     def __init__(self, property2=None, **kwargs):
         if property2 and property2 < 10:
             raise ValueError("'property2' is too small.")
+        if "property3" in kwargs and not isinstance(kwargs.get("property3"), int):
+            raise TypeError("Must be integer!")
+
+
+def test_custom_extension_raises_exception():
+    with pytest.raises(TypeError) as excinfo:
+        NewExtension(property1='something', property3='something', allow_custom=True)
+
+    assert str(excinfo.value) == "Must be integer!"
 
 
 def test_custom_extension():
@@ -453,7 +490,7 @@ def test_custom_extension_empty_properties():
     assert "'properties' must be a dict!" in str(excinfo.value)
 
 
-def test_custom_extension_no_init():
+def test_custom_extension_no_init_1():
     @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-extension', {
         'property1': stix2.properties.StringProperty(required=True),
     })
@@ -463,6 +500,8 @@ def test_custom_extension_no_init():
     ne = NewExt(property1="foobar")
     assert ne.property1 == "foobar"
 
+
+def test_custom_extension_no_init_2():
     @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', {
         'property1': stix2.properties.StringProperty(required=True),
     })
diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py
index 456bf92..d2271f0 100644
--- a/stix2/test/test_markings.py
+++ b/stix2/test/test_markings.py
@@ -187,7 +187,8 @@ def test_parse_marking_definition(data):
 ])
 class NewMarking(object):
     def __init__(self, property2=None, **kwargs):
-        return
+        if "property3" in kwargs and not isinstance(kwargs.get("property3"), int):
+            raise TypeError("Must be integer!")
 
 
 def test_registered_custom_marking():
@@ -208,6 +209,13 @@ def test_registered_custom_marking():
     assert marking_def.definition_type == "x-new-marking-type"
 
 
+def test_registered_custom_marking_raises_exception():
+    with pytest.raises(TypeError) as excinfo:
+        NewMarking(property1='something', property3='something', allow_custom=True)
+
+    assert str(excinfo.value) == "Must be integer!"
+
+
 def test_not_registered_marking_raises_exception():
     with pytest.raises(ValueError) as excinfo:
         # Used custom object on purpose to demonstrate a not-registered marking

From 7e0f911972a0e4bce9a224f4ca96f699de46a5a4 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Fri, 17 Nov 2017 17:30:24 -0500
Subject: [PATCH 73/90] Add relationship_type note to docstrings

---
 stix2/sources/__init__.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index d2b85d0..64abc74 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -110,6 +110,7 @@ class DataStore(object):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 relationships will be looked up.
             relationship_type (str): Only retrieve Relationships of this type.
+                If None, all relationships will be returned, regardless of type.
             source_only (bool): Only retrieve Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only retrieve Relationships for which this
@@ -136,7 +137,8 @@ class DataStore(object):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 related objects will be looked up.
             relationship_type (str): Only retrieve objects related by this
-                Relationships type.
+                Relationships type. If None, all related objects will be
+                returned, regardless of type.
             source_only (bool): Only examine Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this
@@ -265,6 +267,7 @@ class DataSource(with_metaclass(ABCMeta)):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 relationships will be looked up.
             relationship_type (str): Only retrieve Relationships of this type.
+                If None, all relationships will be returned, regardless of type.
             source_only (bool): Only retrieve Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only retrieve Relationships for which this
@@ -305,7 +308,8 @@ class DataSource(with_metaclass(ABCMeta)):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 related objects will be looked up.
             relationship_type (str): Only retrieve objects related by this
-                Relationships type.
+                Relationships type. If None, all related objects will be
+                returned, regardless of type.
             source_only (bool): Only examine Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this
@@ -512,6 +516,7 @@ class CompositeDataSource(DataSource):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 relationships will be looked up.
             relationship_type (str): Only retrieve Relationships of this type.
+                If None, all relationships will be returned, regardless of type.
             source_only (bool): Only retrieve Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only retrieve Relationships for which this
@@ -559,7 +564,8 @@ class CompositeDataSource(DataSource):
             obj (STIX object OR dict OR str): The STIX object (or its ID) whose
                 related objects will be looked up.
             relationship_type (str): Only retrieve objects related by this
-                Relationships type.
+                Relationships type. If None, all related objects will be
+                returned, regardless of type.
             source_only (bool): Only examine Relationships for which this
                 object is the source_ref. Default: False.
             target_only (bool): Only examine Relationships for which this

From 8930e72cc390c9920b1a8dc4a06a1cec87948bbc Mon Sep 17 00:00:00 2001
From: Sam Cornwell <14048146+samcornwell@users.noreply.github.com>
Date: Sun, 19 Nov 2017 22:11:44 -0500
Subject: [PATCH 74/90] added get_type_from_id function to utils

---
 stix2/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/stix2/utils.py b/stix2/utils.py
index f23dbe2..960b89e 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -255,3 +255,6 @@ def get_class_hierarchy_names(obj):
     for cls in obj.__class__.__mro__:
         names.append(cls.__name__)
     return names
+
+def get_type_from_id(stix_id):
+    return stix_id.split('--', 1)[0]

From 78612530ffa06bc5a49e0c8f3201950de80cdeac Mon Sep 17 00:00:00 2001
From: Sam Cornwell <14048146+samcornwell@users.noreply.github.com>
Date: Sun, 19 Nov 2017 22:27:31 -0500
Subject: [PATCH 75/90] added tests for get_type_from_id

---
 stix2/test/test_utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py
index c73bcd2..cbe5b0f 100644
--- a/stix2/test/test_utils.py
+++ b/stix2/test/test_utils.py
@@ -74,3 +74,11 @@ def test_get_dict(data):
 def test_get_dict_invalid(data):
     with pytest.raises(ValueError):
         stix2.utils.get_dict(data)
+
+
+@pytest.mark.parametrize('stix_id, typ', [
+    ('malware--d69c8146-ab35-4d50-8382-6fc80e641d43', 'malware'),
+    ('intrusion-set--899ce53f-13a0-479b-a0e4-67d46e241542', 'intrusion-set')
+])
+def test_get_type_from_id(stix_id, typ):
+    assert stix2.utils.get_type_from_id(stix_id) == typ

From eff51a2bb8ad5f5dbe8c07ab314a70f809073c4b Mon Sep 17 00:00:00 2001
From: Sam Cornwell <14048146+samcornwell@users.noreply.github.com>
Date: Sun, 19 Nov 2017 23:00:42 -0500
Subject: [PATCH 76/90] fixed pycodestyle failure

---
 stix2/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/stix2/utils.py b/stix2/utils.py
index 960b89e..541e6d8 100644
--- a/stix2/utils.py
+++ b/stix2/utils.py
@@ -256,5 +256,6 @@ def get_class_hierarchy_names(obj):
         names.append(cls.__name__)
     return names
 
+
 def get_type_from_id(stix_id):
     return stix_id.split('--', 1)[0]

From 92f7e706bf6b275daeeddf2708d77b328d17bede Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 21 Nov 2017 10:29:57 -0500
Subject: [PATCH 77/90] Improve error handling in relationships/related_to

---
 stix2/sources/__init__.py      | 28 ++++++++++++++++++++--------
 stix2/test/test_environment.py | 22 ++++++++++++++++++++++
 2 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index 64abc74..aea42b1 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -281,8 +281,11 @@ class DataSource(with_metaclass(ABCMeta)):
         filters = [Filter('type', '=', 'relationship')]
 
         try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
+            obj_id = obj['id']
+        except KeyError:
+            raise ValueError("STIX object has no 'id' property")
+        except TypeError:
+            # Assume `obj` is an ID string
             obj_id = obj
 
         if relationship_type:
@@ -323,8 +326,11 @@ class DataSource(with_metaclass(ABCMeta)):
         rels = self.relationships(obj, relationship_type, source_only, target_only)
 
         try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
+            obj_id = obj['id']
+        except KeyError:
+            raise ValueError("STIX object has no 'id' property")
+        except TypeError:
+            # Assume `obj` is an ID string
             obj_id = obj
 
         for r in rels:
@@ -533,8 +539,11 @@ class CompositeDataSource(DataSource):
         filters = [Filter('type', '=', 'relationship')]
 
         try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
+            obj_id = obj['id']
+        except KeyError:
+            raise ValueError("STIX object has no 'id' property")
+        except TypeError:
+            # Assume `obj` is an ID string
             obj_id = obj
 
         if relationship_type:
@@ -583,8 +592,11 @@ class CompositeDataSource(DataSource):
             rels = ds.relationships(obj, relationship_type, source_only, target_only)
 
         try:
-            obj_id = obj.get('id', '')
-        except AttributeError:
+            obj_id = obj['id']
+        except KeyError:
+            raise ValueError("STIX object has no 'id' property")
+        except TypeError:
+            # Assume `obj` is an ID string
             obj_id = obj
 
         for ds in self.data_sources:
diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py
index d9e8fff..84ca803 100644
--- a/stix2/test/test_environment.py
+++ b/stix2/test/test_environment.py
@@ -258,6 +258,17 @@ def test_relationships(ds):
     assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp)
 
 
+def test_relationships_no_id(ds):
+    env = stix2.Environment(store=ds)
+    mal = {
+        "type": "malware",
+        "name": "some variant"
+    }
+    with pytest.raises(ValueError) as excinfo:
+        env.relationships(mal)
+    assert "object has no 'id' property" in str(excinfo.value)
+
+
 def test_relationships_by_type(ds):
     env = stix2.Environment(store=ds)
     mal = env.get(MALWARE_ID)
@@ -311,6 +322,17 @@ def test_related_to(ds):
     assert any(x['id'] == IDENTITY_ID for x in resp)
 
 
+def test_related_to_no_id(ds):
+    env = stix2.Environment(store=ds)
+    mal = {
+        "type": "malware",
+        "name": "some variant"
+    }
+    with pytest.raises(ValueError) as excinfo:
+        env.related_to(mal)
+    assert "object has no 'id' property" in str(excinfo.value)
+
+
 def test_related_to_by_source(ds):
     env = stix2.Environment(store=ds)
     resp = env.related_to(MALWARE_ID, source_only=True)

From 6446be310ce96f0512f92cc39feca607aeba6893 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 21 Nov 2017 15:57:35 -0500
Subject: [PATCH 78/90] Clean up relationships code

---
 stix2/sources/__init__.py     | 58 ++++++++---------------------------
 stix2/test/test_filesystem.py |  2 +-
 2 files changed, 14 insertions(+), 46 deletions(-)

diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index aea42b1..d42dcd1 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -327,8 +327,6 @@ class DataSource(with_metaclass(ABCMeta)):
 
         try:
             obj_id = obj['id']
-        except KeyError:
-            raise ValueError("STIX object has no 'id' property")
         except TypeError:
             # Assume `obj` is an ID string
             obj_id = obj
@@ -510,7 +508,7 @@ class CompositeDataSource(DataSource):
 
         return all_data
 
-    def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
+    def relationships(self, *args, **kwargs):
         """Retrieve Relationships involving the given STIX object.
 
         Only one of `source_only` and `target_only` may be `True`.
@@ -536,31 +534,17 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         results = []
-        filters = [Filter('type', '=', 'relationship')]
-
-        try:
-            obj_id = obj['id']
-        except KeyError:
-            raise ValueError("STIX object has no 'id' property")
-        except TypeError:
-            # Assume `obj` is an ID string
-            obj_id = obj
-
-        if relationship_type:
-            filters.append(Filter('relationship_type', '=', relationship_type))
-
-        if source_only and target_only:
-            raise ValueError("Search either source only or target only, but not both")
-
         for ds in self.data_sources:
-            if not target_only:
-                results.extend(ds.query(filters + [Filter('source_ref', '=', obj_id)]))
-            if not source_only:
-                results.extend(ds.query(filters + [Filter('target_ref', '=', obj_id)]))
+            results.extend(ds.relationships(*args, **kwargs))
+
+        # remove exact duplicates (where duplicates are STIX 2.0
+        # objects with the same 'id' and 'modified' values)
+        if len(results) > 0:
+            results = deduplicate(results)
 
         return results
 
-    def related_to(self, obj, relationship_type=None, source_only=False, target_only=False):
+    def related_to(self, *args, **kwargs):
         """Retrieve STIX Objects that have a Relationship involving the given
         STIX object.
 
@@ -589,28 +573,12 @@ class CompositeDataSource(DataSource):
 
         results = []
         for ds in self.data_sources:
-            rels = ds.relationships(obj, relationship_type, source_only, target_only)
+            results.extend(ds.related_to(*args, **kwargs))
 
-        try:
-            obj_id = obj['id']
-        except KeyError:
-            raise ValueError("STIX object has no 'id' property")
-        except TypeError:
-            # Assume `obj` is an ID string
-            obj_id = obj
-
-        for ds in self.data_sources:
-            for r in rels:
-                if not source_only:
-                    # relationships() found relationships where target_ref is obj_id
-                    source_id = r.source_ref
-                    if source_id != obj_id:  # needed if target_only is also false
-                        results.append(ds.get(source_id))
-                if not target_only:
-                    # relationships() found relationships where source_ref is obj_id
-                    target_id = r.target_ref
-                    if target_id != obj_id:  # needed if source_only is also false
-                        results.append(ds.get(target_id))
+        # remove exact duplicates (where duplicates are STIX 2.0
+        # objects with the same 'id' and 'modified' values)
+        if len(results) > 0:
+            results = deduplicate(results)
 
         return results
 
diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py
index 455132f..68fc185 100644
--- a/stix2/test/test_filesystem.py
+++ b/stix2/test/test_filesystem.py
@@ -45,7 +45,7 @@ def fs_sink():
     shutil.rmtree(os.path.join(FS_PATH, "campaign"), True)
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def rel_fs_store():
     cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
     idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)

From 078474259abe2d3a79aab554bf9eaf4d47e7a3a2 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 21 Nov 2017 16:19:15 -0500
Subject: [PATCH 79/90] Simplify `related_to`

---
 stix2/sources/__init__.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index d42dcd1..4e4cc24 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -331,17 +331,14 @@ class DataSource(with_metaclass(ABCMeta)):
             # Assume `obj` is an ID string
             obj_id = obj
 
+        # Get all unique ids from the relationships except that of the object
+        ids = set()
         for r in rels:
-            if not source_only:
-                # relationships() found relationships where target_ref is obj_id
-                source_id = r.source_ref
-                if source_id != obj_id:  # needed if target_only is also false
-                    results.append(self.get(source_id))
-            if not target_only:
-                # relationships() found relationships where source_ref is obj_id
-                target_id = r.target_ref
-                if target_id != obj_id:  # needed if source_only is also false
-                    results.append(self.get(target_id))
+            ids.update((r.source_ref, r.target_ref))
+        ids.remove(obj_id)
+
+        for i in ids:
+            results.append(self.get(i))
 
         return results
 

From d355d1827e8a085f1f66f6f7fadce209ecc76108 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 21 Nov 2017 16:29:06 -0500
Subject: [PATCH 80/90] Move `creator_of` to DataStore/DataSource

---
 stix2/environment.py      | 20 +-------------------
 stix2/sources/__init__.py | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 19 deletions(-)

diff --git a/stix2/environment.py b/stix2/environment.py
index 5bb876b..33fe6ea 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -108,6 +108,7 @@ class Environment(object):
     get = DataStore.__dict__['get']
     all_versions = DataStore.__dict__['all_versions']
     query = DataStore.__dict__['query']
+    creator_of = DataStore.__dict__['creator_of']
     relationships = DataStore.__dict__['relationships']
     related_to = DataStore.__dict__['related_to']
     add = DataStore.__dict__['add']
@@ -127,22 +128,3 @@ class Environment(object):
     def parse(self, *args, **kwargs):
         return _parse(*args, **kwargs)
     parse.__doc__ = _parse.__doc__
-
-    def creator_of(self, obj):
-        """Retrieve the Identity refered to by the object's `created_by_ref`.
-
-        Args:
-            obj: The STIX object whose `created_by_ref` property will be looked
-                up.
-
-        Returns:
-            The STIX object's creator, or
-            None, if the object contains no `created_by_ref` property or the
-                object's creator cannot be found.
-
-        """
-        creator_id = obj.get('created_by_ref', '')
-        if creator_id:
-            return self.get(creator_id)
-        else:
-            return None
diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py
index 4e4cc24..7afc08d 100644
--- a/stix2/sources/__init__.py
+++ b/stix2/sources/__init__.py
@@ -99,6 +99,26 @@ class DataStore(object):
         except AttributeError:
             raise AttributeError('%s has no data source to query' % self.__class__.__name__)
 
+    def creator_of(self, *args, **kwargs):
+        """Retrieve the Identity refered to by the object's `created_by_ref`.
+
+        Translate creator_of() call to the appropriate DataSource call.
+
+        Args:
+            obj: The STIX object whose `created_by_ref` property will be looked
+                up.
+
+        Returns:
+            The STIX object's creator, or
+            None, if the object contains no `created_by_ref` property or the
+                object's creator cannot be found.
+
+        """
+        try:
+            return self.source.creator_of(*args, **kwargs)
+        except AttributeError:
+            raise AttributeError('%s has no data source to query' % self.__class__.__name__)
+
     def relationships(self, *args, **kwargs):
         """Retrieve Relationships involving the given STIX object.
 
@@ -258,6 +278,25 @@ class DataSource(with_metaclass(ABCMeta)):
 
         """
 
+    def creator_of(self, obj):
+        """Retrieve the Identity refered to by the object's `created_by_ref`.
+
+        Args:
+            obj: The STIX object whose `created_by_ref` property will be looked
+                up.
+
+        Returns:
+            The STIX object's creator, or
+            None, if the object contains no `created_by_ref` property or the
+                object's creator cannot be found.
+
+        """
+        creator_id = obj.get('created_by_ref', '')
+        if creator_id:
+            return self.get(creator_id)
+        else:
+            return None
+
     def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
         """Retrieve Relationships involving the given STIX object.
 

From db9c93d9e11a718c871ce4378c78571f20709542 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Tue, 21 Nov 2017 17:32:17 -0500
Subject: [PATCH 81/90] Add documentation for related object functions

---
 docs/guide/datastore.ipynb   | 273 ++++++++++++++++++++++++++++++++++-
 docs/guide/environment.ipynb |   2 +-
 2 files changed, 273 insertions(+), 2 deletions(-)

diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index 7fc0997..24a2b4f 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 40,
    "metadata": {
     "collapsed": true,
     "nbsphinx": "hidden"
@@ -262,6 +262,277 @@
     "# attach multiple filters to a MemoryStore\n",
     "mem.source.filters.update([f1,f2])"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## De-Referencing Relationships\n",
+    "\n",
+    "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) and add some objects and relationships."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from stix2 import Campaign, Identity, Indicator, Malware, Relationship\n",
+    "\n",
+    "mem = MemoryStore()\n",
+    "cam = Campaign(name='Charge', description='Attack!')\n",
+    "idy = Identity(name='John Doe', identity_class=\"individual\")\n",
+    "ind = Indicator(labels=['malicious-activity'], pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n",
+    "mal = Malware(labels=['ransomware'], name=\"Cryptolocker\", created_by_ref=idy)\n",
+    "rel1 = Relationship(ind, 'indicates', mal,)\n",
+    "rel2 = Relationship(mal, 'targets', idy)\n",
+    "rel3 = Relationship(cam, 'uses', mal)\n",
+    "mem.add([cam, idy, ind, mal, rel1, rel2, rel3])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of) method to retrieve the [Identity](../api/stix2.v20.sdo.rst#stix2.v20.sdo.Identity) object that created it."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
{\n",
+       "    "type": "identity",\n",
+       "    "id": "identity--be3baac0-9aba-48a8-81e4-4408b1c379a8",\n",
+       "    "created": "2017-11-21T22:14:45.213Z",\n",
+       "    "modified": "2017-11-21T22:14:45.213Z",\n",
+       "    "name": "John Doe",\n",
+       "    "identity_class": "individual"\n",
+       "}\n",
+       "
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(mem.creator_of(mal))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use the [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) method to retrieve all the relationship objects that reference a STIX object."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rels = mem.relationships(mal)\n",
+    "len(rels)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can limit it to only specific relationship types:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mem.relationships(mal, relationship_type='indicates')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can limit it to only relationships where the given object is the source:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Relationship(type='relationship', id='relationship--7eb7f5cd-8bf2-4f7c-8756-84c0b5693b9a', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'targets', source_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4', target_ref='identity--be3baac0-9aba-48a8-81e4-4408b1c379a8')]"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mem.relationships(mal, source_only=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And you can limit it to only relationships where the given object is the target:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4'),\n",
+       " Relationship(type='relationship', id='relationship--3c759d40-c92a-430e-aab6-77d5c5763302', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'uses', source_ref='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mem.relationships(mal, target_only=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, you can retrieve all STIX objects related to a given STIX object using [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to). This calls [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) but then performs the extra step of getting the objects that these Relationships point to. [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to) takes all the same arguments that [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) does."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Campaign(type='campaign', id='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', created='2017-11-21T22:14:45.213Z', modified='2017-11-21T22:14:45.213Z', name=u'Charge', description=u'Attack!')]"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mem.related_to(mal, target_only=True, relationship_type='uses')"
+   ]
   }
  ],
  "metadata": {
diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb
index 2d85911..0cb5796 100644
--- a/docs/guide/environment.ipynb
+++ b/docs/guide/environment.ipynb
@@ -128,7 +128,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can retrieve STIX objects from the [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), and [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), just as you would for a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource)."
+    "You can retrieve STIX objects from the [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of), [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to), and [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) just as you would for a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource)."
    ]
   },
   {

From c0669d7a5f71e73bc040b1114a9649c811ea7ce4 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Wed, 29 Nov 2017 22:11:40 +0000
Subject: [PATCH 82/90] Drop support for Python 3.3

---
 .travis.yml | 3 +--
 setup.py    | 1 -
 tox.ini     | 3 +--
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index aba764d..0d5a046 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,6 @@ language: python
 cache: pip
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
   - "3.5-dev"
@@ -16,6 +15,6 @@ install:
   - pip install codecov
 script:
   - tox
-  - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]]; then pre-commit run --all-files; fi
+  - pre-commit run --all-files
 after_success:
   - codecov
diff --git a/setup.py b/setup.py
index 72bc5d7..fa68616 100644
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,6 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
diff --git a/tox.ini b/tox.ini
index fe4fb01..bfc8c1b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py33,py34,py35,py36,pycodestyle,isort-check
+envlist = py27,py34,py35,py36,pycodestyle,isort-check
 
 [testenv]
 deps =
@@ -36,7 +36,6 @@ commands =
 [travis]
 python =
   2.7: py27, pycodestyle
-  3.3: py33, pycodestyle
   3.4: py34, pycodestyle
   3.5: py35, pycodestyle
   3.6: py36, pycodestyle

From e2c9354d14b5424d5fbec0b6d115e74f3aeb7da3 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Tue, 16 Jan 2018 08:49:15 -0600
Subject: [PATCH 83/90] GH-121: Use correct property name for
 IntrusionSet.last_seen

Fix #121
---
 stix2/v20/sdo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/v20/sdo.py b/stix2/v20/sdo.py
index 1af0777..2d36aaf 100644
--- a/stix2/v20/sdo.py
+++ b/stix2/v20/sdo.py
@@ -144,7 +144,7 @@ class IntrusionSet(STIXDomainObject):
         ('description', StringProperty()),
         ('aliases', ListProperty(StringProperty)),
         ('first_seen', TimestampProperty()),
-        ('last_seen ', TimestampProperty()),
+        ('last_seen', TimestampProperty()),
         ('goals', ListProperty(StringProperty)),
         ('resource_level', StringProperty()),
         ('primary_motivation', StringProperty()),

From a758c7e1b2635908a0420e054cefcedd76c482bd Mon Sep 17 00:00:00 2001
From: Mo 
Date: Wed, 31 Jan 2018 17:24:57 +0000
Subject: [PATCH 84/90] Update datastore.ipynb

---
 docs/guide/datastore.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb
index 24a2b4f..ba8ad53 100644
--- a/docs/guide/datastore.ipynb
+++ b/docs/guide/datastore.ipynb
@@ -60,7 +60,7 @@
     "\n",
     "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) constructs: a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) for both pulling and pushing.\n",
     "\n",
-    "The DataStore, [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource), [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then sublcassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites."
+    "The DataStore, [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource), [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then subclassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites."
    ]
   },
   {

From 845934033eb5bf14ed1610941bb7de1f8a29fc84 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Feb 2018 14:44:28 -0500
Subject: [PATCH 85/90] Avoid checking valid refs when deepcopying

Using `deepcopy` on an Observable object that referenced another (e.g.
domain name -> resolves to -> IPv4 address) would fail because no
`_valid_refs` were available. Assuming that the references have
already been checked and are valid when using `deepcopy`, we can use a
special value ('*') to skip the valid_refs check.

This will also allow creating new versions of ObservedData objects that
contain related objects, since `new_version` calls `deepcopy`.
---
 stix2/base.py                    |  8 +++++++-
 stix2/test/test_observed_data.py | 22 ++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/stix2/base.py b/stix2/base.py
index 76b07b8..fc13094 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -161,9 +161,12 @@ class _STIXBase(collections.Mapping):
                                  ", ".join(["{0!s}={1!r}".format(k, v) for k, v in props]))
 
     def __deepcopy__(self, memo):
-        # Assumption: we can ignore the memo argument, because no object will ever contain the same sub-object multiple times.
+        # Assume: we can ignore the memo argument, because no object will ever contain the same sub-object multiple times.
         new_inner = copy.deepcopy(self._inner, memo)
         cls = type(self)
+        if isinstance(self, _Observable):
+            # Assume: valid references in the original object are still valid in the new version
+            new_inner['_valid_refs'] = {'*': '*'}
         return cls(**new_inner)
 
     def properties_populated(self):
@@ -221,6 +224,9 @@ class _Observable(_STIXBase):
         super(_Observable, self).__init__(**kwargs)
 
     def _check_ref(self, ref, prop, prop_name):
+        if '*' in self._STIXBase__valid_refs:
+            return  # don't check if refs are valid
+
         if ref not in self._STIXBase__valid_refs:
             raise InvalidObjRefError(self.__class__, prop_name, "'%s' is not a valid object in local scope" % ref)
 
diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py
index 3029b68..cc0c974 100644
--- a/stix2/test/test_observed_data.py
+++ b/stix2/test/test_observed_data.py
@@ -1162,3 +1162,25 @@ def test_x509_certificate_example():
     assert x509.type == "x509-certificate"
     assert x509.issuer == "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com"  # noqa
     assert x509.subject == "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, CN=www.freesoft.org/emailAddress=baccala@freesoft.org"  # noqa
+
+
+def test_new_version_with_related_objects():
+    data = stix2.ObservedData(
+        first_observed="2016-03-12T12:00:00Z",
+        last_observed="2016-03-12T12:00:00Z",
+        number_observed=1,
+        objects={
+            'src_ip': {
+                'type': 'ipv4-addr',
+                'value': '127.0.0.1/32'
+            },
+            'domain':{
+                'type': 'domain-name',
+                'value': 'example.com',
+                'resolves_to_refs': ['src_ip']
+            }
+        }
+    )
+    new_version = data.new_version(last_observed="2017-12-12T12:00:00Z")
+    assert new_version.last_observed.year == 2017
+    assert new_version.objects['domain'].resolves_to_refs[0] == 'src_ip'

From 5427e79d71aa0e6fb021b908694cb75414a253c0 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Feb 2018 15:03:20 -0500
Subject: [PATCH 86/90] Fix pycodestyle/isort issues

---
 stix2/test/test_attack_pattern.py     | 1 -
 stix2/test/test_campaign.py           | 1 -
 stix2/test/test_course_of_action.py   | 1 -
 stix2/test/test_external_reference.py | 1 -
 stix2/test/test_identity.py           | 1 -
 stix2/test/test_indicator.py          | 1 -
 stix2/test/test_intrusion_set.py      | 1 -
 stix2/test/test_kill_chain_phases.py  | 1 -
 stix2/test/test_malware.py            | 1 -
 stix2/test/test_markings.py           | 1 -
 stix2/test/test_object_markings.py    | 3 ++-
 stix2/test/test_observed_data.py      | 2 +-
 stix2/test/test_relationship.py       | 1 -
 stix2/test/test_report.py             | 1 -
 stix2/test/test_sighting.py           | 1 -
 stix2/test/test_threat_actor.py       | 1 -
 stix2/test/test_tool.py               | 1 -
 stix2/test/test_vulnerability.py      | 1 -
 18 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/stix2/test/test_attack_pattern.py b/stix2/test/test_attack_pattern.py
index 07d0898..0be118c 100644
--- a/stix2/test/test_attack_pattern.py
+++ b/stix2/test/test_attack_pattern.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import ATTACK_PATTERN_ID
 
-
 EXPECTED = """{
     "type": "attack-pattern",
     "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061",
diff --git a/stix2/test/test_campaign.py b/stix2/test/test_campaign.py
index 202534d..b226478 100644
--- a/stix2/test/test_campaign.py
+++ b/stix2/test/test_campaign.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import CAMPAIGN_ID
 
-
 EXPECTED = """{
     "type": "campaign",
     "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_course_of_action.py b/stix2/test/test_course_of_action.py
index 3dc379d..e376f0d 100644
--- a/stix2/test/test_course_of_action.py
+++ b/stix2/test/test_course_of_action.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import COURSE_OF_ACTION_ID
 
-
 EXPECTED = """{
     "type": "course-of-action",
     "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py
index c4c3755..2b79f01 100644
--- a/stix2/test/test_external_reference.py
+++ b/stix2/test/test_external_reference.py
@@ -6,7 +6,6 @@ import pytest
 
 import stix2
 
-
 VERIS = """{
     "source_name": "veris",
     "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json",
diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py
index 8e3dd42..ff3631f 100644
--- a/stix2/test/test_identity.py
+++ b/stix2/test/test_identity.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import IDENTITY_ID
 
-
 EXPECTED = """{
     "type": "identity",
     "id": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",
diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py
index c501da4..78d1bf2 100644
--- a/stix2/test/test_indicator.py
+++ b/stix2/test/test_indicator.py
@@ -8,7 +8,6 @@ import stix2
 
 from .constants import FAKE_TIME, INDICATOR_ID, INDICATOR_KWARGS
 
-
 EXPECTED_INDICATOR = """{
     "type": "indicator",
     "id": "indicator--01234567-89ab-cdef-0123-456789abcdef",
diff --git a/stix2/test/test_intrusion_set.py b/stix2/test/test_intrusion_set.py
index 481b3cb..53e18f5 100644
--- a/stix2/test/test_intrusion_set.py
+++ b/stix2/test/test_intrusion_set.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INTRUSION_SET_ID
 
-
 EXPECTED = """{
     "type": "intrusion-set",
     "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29",
diff --git a/stix2/test/test_kill_chain_phases.py b/stix2/test/test_kill_chain_phases.py
index 9cddcb9..220c714 100644
--- a/stix2/test/test_kill_chain_phases.py
+++ b/stix2/test/test_kill_chain_phases.py
@@ -4,7 +4,6 @@ import pytest
 
 import stix2
 
-
 LMCO_RECON = """{
     "kill_chain_name": "lockheed-martin-cyber-kill-chain",
     "phase_name": "reconnaissance"
diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py
index 7f665ea..8c565cd 100644
--- a/stix2/test/test_malware.py
+++ b/stix2/test/test_malware.py
@@ -8,7 +8,6 @@ import stix2
 
 from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS
 
-
 EXPECTED_MALWARE = """{
     "type": "malware",
     "id": "malware--fedcba98-7654-3210-fedc-ba9876543210",
diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py
index d2271f0..efd0476 100644
--- a/stix2/test/test_markings.py
+++ b/stix2/test/test_markings.py
@@ -8,7 +8,6 @@ from stix2 import TLP_WHITE
 
 from .constants import MARKING_DEFINITION_ID
 
-
 EXPECTED_TLP_MARKING_DEFINITION = """{
     "type": "marking-definition",
     "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py
index 10949ab..f216355 100644
--- a/stix2/test/test_object_markings.py
+++ b/stix2/test/test_object_markings.py
@@ -3,8 +3,9 @@ import pytest
 
 from stix2 import TLP_AMBER, Malware, exceptions, markings
 
-from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS
+from .constants import FAKE_TIME, MALWARE_ID
 from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST
+from .constants import MARKING_IDS
 
 """Tests for the Data Markings API."""
 
diff --git a/stix2/test/test_observed_data.py b/stix2/test/test_observed_data.py
index cc0c974..30c3cab 100644
--- a/stix2/test/test_observed_data.py
+++ b/stix2/test/test_observed_data.py
@@ -1174,7 +1174,7 @@ def test_new_version_with_related_objects():
                 'type': 'ipv4-addr',
                 'value': '127.0.0.1/32'
             },
-            'domain':{
+            'domain': {
                 'type': 'domain-name',
                 'value': 'example.com',
                 'resolves_to_refs': ['src_ip']
diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py
index 6d65544..c6e2d6f 100644
--- a/stix2/test/test_relationship.py
+++ b/stix2/test/test_relationship.py
@@ -8,7 +8,6 @@ import stix2
 from .constants import (FAKE_TIME, INDICATOR_ID, MALWARE_ID, RELATIONSHIP_ID,
                         RELATIONSHIP_KWARGS)
 
-
 EXPECTED_RELATIONSHIP = """{
     "type": "relationship",
     "id": "relationship--00000000-1111-2222-3333-444444444444",
diff --git a/stix2/test/test_report.py b/stix2/test/test_report.py
index a5775e3..b38ab40 100644
--- a/stix2/test/test_report.py
+++ b/stix2/test/test_report.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INDICATOR_KWARGS, REPORT_ID
 
-
 EXPECTED = """{
     "type": "report",
     "id": "report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3",
diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py
index af91413..ce1fab9 100644
--- a/stix2/test/test_sighting.py
+++ b/stix2/test/test_sighting.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS
 
-
 EXPECTED_SIGHTING = """{
     "type": "sighting",
     "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb",
diff --git a/stix2/test/test_threat_actor.py b/stix2/test/test_threat_actor.py
index c095c3b..8079a21 100644
--- a/stix2/test/test_threat_actor.py
+++ b/stix2/test/test_threat_actor.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import THREAT_ACTOR_ID
 
-
 EXPECTED = """{
     "type": "threat-actor",
     "id": "threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py
index be52f6d..21ece24 100644
--- a/stix2/test/test_tool.py
+++ b/stix2/test/test_tool.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import TOOL_ID
 
-
 EXPECTED = """{
     "type": "tool",
     "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_vulnerability.py b/stix2/test/test_vulnerability.py
index a6426d8..e7358df 100644
--- a/stix2/test/test_vulnerability.py
+++ b/stix2/test/test_vulnerability.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import VULNERABILITY_ID
 
-
 EXPECTED = """{
     "type": "vulnerability",
     "id": "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061",

From 99addbaadac43b17600f5191b9fae85bf3043429 Mon Sep 17 00:00:00 2001
From: Chris Lenk 
Date: Mon, 19 Feb 2018 15:43:25 -0500
Subject: [PATCH 87/90] Allow Python nightly to fail in Travis

...without causing the entire build to fail, but still recording the result.
---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0d5a046..7c3a4bb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,9 @@ python:
   - "3.6"
   - "3.6-dev"
   - "nightly"
+matrix:
+  allow_failures:
+    - python: "nightly"
 install:
   - pip install -U pip setuptools
   - pip install tox-travis pre-commit

From b1a020bb38d74c407ac125b1e032850acfe7a880 Mon Sep 17 00:00:00 2001
From: Greg Back 
Date: Wed, 21 Feb 2018 11:02:08 -0600
Subject: [PATCH 88/90] Update imports to appease isort.

---
 stix2/test/test_attack_pattern.py     | 1 -
 stix2/test/test_campaign.py           | 1 -
 stix2/test/test_course_of_action.py   | 1 -
 stix2/test/test_external_reference.py | 1 -
 stix2/test/test_identity.py           | 1 -
 stix2/test/test_indicator.py          | 1 -
 stix2/test/test_intrusion_set.py      | 1 -
 stix2/test/test_kill_chain_phases.py  | 1 -
 stix2/test/test_malware.py            | 1 -
 stix2/test/test_markings.py           | 1 -
 stix2/test/test_object_markings.py    | 3 ++-
 stix2/test/test_relationship.py       | 1 -
 stix2/test/test_report.py             | 1 -
 stix2/test/test_sighting.py           | 1 -
 stix2/test/test_threat_actor.py       | 1 -
 stix2/test/test_tool.py               | 1 -
 stix2/test/test_vulnerability.py      | 1 -
 17 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/stix2/test/test_attack_pattern.py b/stix2/test/test_attack_pattern.py
index 07d0898..0be118c 100644
--- a/stix2/test/test_attack_pattern.py
+++ b/stix2/test/test_attack_pattern.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import ATTACK_PATTERN_ID
 
-
 EXPECTED = """{
     "type": "attack-pattern",
     "id": "attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061",
diff --git a/stix2/test/test_campaign.py b/stix2/test/test_campaign.py
index 202534d..b226478 100644
--- a/stix2/test/test_campaign.py
+++ b/stix2/test/test_campaign.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import CAMPAIGN_ID
 
-
 EXPECTED = """{
     "type": "campaign",
     "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_course_of_action.py b/stix2/test/test_course_of_action.py
index 3dc379d..e376f0d 100644
--- a/stix2/test/test_course_of_action.py
+++ b/stix2/test/test_course_of_action.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import COURSE_OF_ACTION_ID
 
-
 EXPECTED = """{
     "type": "course-of-action",
     "id": "course-of-action--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_external_reference.py b/stix2/test/test_external_reference.py
index c4c3755..2b79f01 100644
--- a/stix2/test/test_external_reference.py
+++ b/stix2/test/test_external_reference.py
@@ -6,7 +6,6 @@ import pytest
 
 import stix2
 
-
 VERIS = """{
     "source_name": "veris",
     "url": "https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json",
diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py
index 8e3dd42..ff3631f 100644
--- a/stix2/test/test_identity.py
+++ b/stix2/test/test_identity.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import IDENTITY_ID
 
-
 EXPECTED = """{
     "type": "identity",
     "id": "identity--311b2d2d-f010-5473-83ec-1edf84858f4c",
diff --git a/stix2/test/test_indicator.py b/stix2/test/test_indicator.py
index c501da4..78d1bf2 100644
--- a/stix2/test/test_indicator.py
+++ b/stix2/test/test_indicator.py
@@ -8,7 +8,6 @@ import stix2
 
 from .constants import FAKE_TIME, INDICATOR_ID, INDICATOR_KWARGS
 
-
 EXPECTED_INDICATOR = """{
     "type": "indicator",
     "id": "indicator--01234567-89ab-cdef-0123-456789abcdef",
diff --git a/stix2/test/test_intrusion_set.py b/stix2/test/test_intrusion_set.py
index 481b3cb..53e18f5 100644
--- a/stix2/test/test_intrusion_set.py
+++ b/stix2/test/test_intrusion_set.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INTRUSION_SET_ID
 
-
 EXPECTED = """{
     "type": "intrusion-set",
     "id": "intrusion-set--4e78f46f-a023-4e5f-bc24-71b3ca22ec29",
diff --git a/stix2/test/test_kill_chain_phases.py b/stix2/test/test_kill_chain_phases.py
index 9cddcb9..220c714 100644
--- a/stix2/test/test_kill_chain_phases.py
+++ b/stix2/test/test_kill_chain_phases.py
@@ -4,7 +4,6 @@ import pytest
 
 import stix2
 
-
 LMCO_RECON = """{
     "kill_chain_name": "lockheed-martin-cyber-kill-chain",
     "phase_name": "reconnaissance"
diff --git a/stix2/test/test_malware.py b/stix2/test/test_malware.py
index 7f665ea..8c565cd 100644
--- a/stix2/test/test_malware.py
+++ b/stix2/test/test_malware.py
@@ -8,7 +8,6 @@ import stix2
 
 from .constants import FAKE_TIME, MALWARE_ID, MALWARE_KWARGS
 
-
 EXPECTED_MALWARE = """{
     "type": "malware",
     "id": "malware--fedcba98-7654-3210-fedc-ba9876543210",
diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py
index d2271f0..efd0476 100644
--- a/stix2/test/test_markings.py
+++ b/stix2/test/test_markings.py
@@ -8,7 +8,6 @@ from stix2 import TLP_WHITE
 
 from .constants import MARKING_DEFINITION_ID
 
-
 EXPECTED_TLP_MARKING_DEFINITION = """{
     "type": "marking-definition",
     "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9",
diff --git a/stix2/test/test_object_markings.py b/stix2/test/test_object_markings.py
index 10949ab..f216355 100644
--- a/stix2/test/test_object_markings.py
+++ b/stix2/test/test_object_markings.py
@@ -3,8 +3,9 @@ import pytest
 
 from stix2 import TLP_AMBER, Malware, exceptions, markings
 
-from .constants import FAKE_TIME, MALWARE_ID, MARKING_IDS
+from .constants import FAKE_TIME, MALWARE_ID
 from .constants import MALWARE_KWARGS as MALWARE_KWARGS_CONST
+from .constants import MARKING_IDS
 
 """Tests for the Data Markings API."""
 
diff --git a/stix2/test/test_relationship.py b/stix2/test/test_relationship.py
index 6d65544..c6e2d6f 100644
--- a/stix2/test/test_relationship.py
+++ b/stix2/test/test_relationship.py
@@ -8,7 +8,6 @@ import stix2
 from .constants import (FAKE_TIME, INDICATOR_ID, MALWARE_ID, RELATIONSHIP_ID,
                         RELATIONSHIP_KWARGS)
 
-
 EXPECTED_RELATIONSHIP = """{
     "type": "relationship",
     "id": "relationship--00000000-1111-2222-3333-444444444444",
diff --git a/stix2/test/test_report.py b/stix2/test/test_report.py
index a5775e3..b38ab40 100644
--- a/stix2/test/test_report.py
+++ b/stix2/test/test_report.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INDICATOR_KWARGS, REPORT_ID
 
-
 EXPECTED = """{
     "type": "report",
     "id": "report--84e4d88f-44ea-4bcd-bbf3-b2c1c320bcb3",
diff --git a/stix2/test/test_sighting.py b/stix2/test/test_sighting.py
index af91413..ce1fab9 100644
--- a/stix2/test/test_sighting.py
+++ b/stix2/test/test_sighting.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import INDICATOR_ID, SIGHTING_ID, SIGHTING_KWARGS
 
-
 EXPECTED_SIGHTING = """{
     "type": "sighting",
     "id": "sighting--bfbc19db-ec35-4e45-beed-f8bde2a772fb",
diff --git a/stix2/test/test_threat_actor.py b/stix2/test/test_threat_actor.py
index c095c3b..8079a21 100644
--- a/stix2/test/test_threat_actor.py
+++ b/stix2/test/test_threat_actor.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import THREAT_ACTOR_ID
 
-
 EXPECTED = """{
     "type": "threat-actor",
     "id": "threat-actor--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_tool.py b/stix2/test/test_tool.py
index be52f6d..21ece24 100644
--- a/stix2/test/test_tool.py
+++ b/stix2/test/test_tool.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import TOOL_ID
 
-
 EXPECTED = """{
     "type": "tool",
     "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
diff --git a/stix2/test/test_vulnerability.py b/stix2/test/test_vulnerability.py
index a6426d8..e7358df 100644
--- a/stix2/test/test_vulnerability.py
+++ b/stix2/test/test_vulnerability.py
@@ -7,7 +7,6 @@ import stix2
 
 from .constants import VULNERABILITY_ID
 
-
 EXPECTED = """{
     "type": "vulnerability",
     "id": "vulnerability--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061",

From d51f1014c76307e9e16506459217a31fd7a3e0fc Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 23 Feb 2018 08:24:26 -0500
Subject: [PATCH 89/90] Fix InstrusionSet 'last_seen' in 2.1

---
 stix2/v21/sdo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/v21/sdo.py b/stix2/v21/sdo.py
index 3c46fa1..223410e 100644
--- a/stix2/v21/sdo.py
+++ b/stix2/v21/sdo.py
@@ -155,7 +155,7 @@ class IntrusionSet(STIXDomainObject):
         ('description', StringProperty()),
         ('aliases', ListProperty(StringProperty)),
         ('first_seen', TimestampProperty()),
-        ('last_seen ', TimestampProperty()),
+        ('last_seen', TimestampProperty()),
         ('goals', ListProperty(StringProperty)),
         ('resource_level', StringProperty()),
         ('primary_motivation', StringProperty()),

From b6c22010bb6922129a34093fb909bce57c093174 Mon Sep 17 00:00:00 2001
From: Emmanuelle Vargas-Gonzalez 
Date: Fri, 23 Feb 2018 08:52:34 -0500
Subject: [PATCH 90/90] Fix imports for test_location.py

---
 stix2/test/test_location.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stix2/test/test_location.py b/stix2/test/test_location.py
index 8ce44d7..71d7877 100644
--- a/stix2/test/test_location.py
+++ b/stix2/test/test_location.py
@@ -8,7 +8,6 @@ import stix2
 
 from .constants import LOCATION_ID
 
-
 EXPECTED_LOCATION_1 = """{
     "type": "location",
     "id": "location--a6e9345f-5a15-4c29-8bb3-7dcc5d168d64",