From 598f38921fa195af007d7f28e3cc40de1904575f Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 6 Sep 2017 16:20:16 -0400 Subject: [PATCH 1/5] Start Environment layer --- stix2/environment.py | 64 ++++++++++++++++++++++++++++++++++++--- stix2/sources/__init__.py | 16 +++++----- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/stix2/environment.py b/stix2/environment.py index f855755..91f0676 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -1,11 +1,10 @@ import copy +from .sources import CompositeDataSource + class ObjectFactory(object): - """Object Factory - - Used to easily create STIX objects with default values for certain - properties. + """Easily create STIX objects with default values for certain properties. Args: created_by_ref: Default created_by_ref value to apply to all @@ -66,3 +65,60 @@ class ObjectFactory(object): properties.update(**kwargs) return cls(**properties) + + +class Environment(object): + """ + + Args: + factory (ObjectFactory): Factory for creating objects with common + defaults for certain properties. + store (DataStore): Data store providing the source and sink for the + environment. + source (DataSource): Source for retrieving STIX objects. + sink (DataSink): Destination for saving STIX objects. + Invalid if `store` is also provided. + """ + + def __init__(self, factory=None, store=None, source=None, sink=None): + self.factory = factory + self.source = CompositeDataSource() + if store: + self.source.add_data_source(store.source) + self.sink = store.sink + if source: + self.source.add_data_source(source) + if sink: + if store: + raise ValueError("Data store already provided! Environment may only have one data sink.") + self.sink = sink + + def create(self, *args, **kwargs): + """Use the object factory to create a STIX object with default property values. + """ + return self.factory.create(*args, **kwargs) + + def get(self, *args, **kwargs): + """Retrieve the most recent version of a single STIX object by ID. + """ + return self.source.get(*args, **kwargs) + + def all_versions(self, *args, **kwargs): + """Retrieve all versions of a single STIX object by ID. + """ + return self.source.all_versions(*args, **kwargs) + + def query(self, *args, **kwargs): + """Retrieve STIX objects matching a set of filters. + """ + return self.source.query(*args, **kwargs) + + def add_filter(self, *args, **kwargs): + """Add a filter to be applied to all queries for STIX objects from this environment. + """ + return self.source.add_filter(*args, **kwargs) + + def add(self, *args, **kwargs): + """Store a STIX object. + """ + return self.sink.add(*args, **kwargs) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index b50fd1d..08063f5 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -5,7 +5,7 @@ Classes: DataStore DataSink DataSource - STIXCommonPropertyFilters + CompositeDataSource TODO:Test everything @@ -226,7 +226,7 @@ class DataSource(object): self.filters.add(filter) def apply_common_filters(self, stix_objs, query): - """Evaluates filters against a set of STIX 2.0 objects + """Evaluate filters against a set of STIX 2.0 objects. Supports only STIX 2.0 common property fields @@ -300,11 +300,10 @@ class DataSource(object): class CompositeDataSource(DataSource): - """Composite Data Source + """Controller for all the defined/configured STIX Data Sources. - Acts as a controller for all the defined/configured STIX Data Sources - e.g. a user can define n Data Sources - creating Data Source (objects) - for each. There is only one instance of this for any python STIX 2.0 + E.g. a user can define n Data Sources - creating Data Source (objects) + for each. There is only one instance of this for any Python STIX 2.0 application. Attributes: @@ -314,8 +313,7 @@ class CompositeDataSource(DataSource): """ def __init__(self): - """ - Creates a new STIX Data Source. + """Create a new STIX Data Source. Args: name (str): A string containing the name to attach in the @@ -448,6 +446,8 @@ class CompositeDataSource(DataSource): to the Composite Data Source """ + if not isinstance(data_sources, list): + data_sources = [data_sources] for ds in data_sources: if issubclass(ds.__class__, DataSource): if ds.id in self.data_sources: From 4dfb5d2365e851aa7bee0cfd60bbbef7fbb6528a Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 8 Sep 2017 09:01:12 -0400 Subject: [PATCH 2/5] Test Environment layer --- stix2/__init__.py | 9 +++- stix2/environment.py | 63 +++++++++++++++++-------- stix2/sources/__init__.py | 9 ++++ stix2/test/test_environment.py | 85 +++++++++++++++++++++++++++++++++- 4 files changed, 144 insertions(+), 22 deletions(-) diff --git a/stix2/__init__.py b/stix2/__init__.py index c2aae2e..35b65b0 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -7,7 +7,7 @@ from .common import (TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE, CustomMarking, ExternalReference, GranularMarking, KillChainPhase, MarkingDefinition, StatementMarking, TLPMarking) from .core import Bundle, _register_type, parse -from .environment import ObjectFactory +from .environment import Environment, ObjectFactory from .observables import (URL, AlternateDataStream, ArchiveExt, Artifact, AutonomousSystem, CustomObservable, Directory, DomainName, EmailAddress, EmailMessage, @@ -41,6 +41,13 @@ from .patterns import (AndBooleanExpression, AndObservationExpression, from .sdo import (AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, Vulnerability) +from .sources import CompositeDataSource +from .sources.filesystem import (FileSystemSink, FileSystemSource, + FileSystemStore) +from .sources.filters import Filter +from .sources.memory import MemorySink, MemorySource, MemoryStore +from .sources.taxii import (TAXIICollectionSink, TAXIICollectionSource, + TAXIICollectionStore) from .sro import Relationship, Sighting from .utils import get_dict, new_version, revoke from .version import __version__ diff --git a/stix2/environment.py b/stix2/environment.py index 91f0676..93f8ba3 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -80,7 +80,7 @@ class Environment(object): Invalid if `store` is also provided. """ - def __init__(self, factory=None, store=None, source=None, sink=None): + def __init__(self, factory=ObjectFactory(), store=None, source=None, sink=None): self.factory = factory self.source = CompositeDataSource() if store: @@ -93,32 +93,55 @@ class Environment(object): raise ValueError("Data store already provided! Environment may only have one data sink.") self.sink = sink - def create(self, *args, **kwargs): - """Use the object factory to create a STIX object with default property values. - """ - return self.factory.create(*args, **kwargs) + def create(self, *args, **kwargs): + """Use the object factory to create a STIX object with default property values. + """ + return self.factory.create(*args, **kwargs) - def get(self, *args, **kwargs): - """Retrieve the most recent version of a single STIX object by ID. - """ + def get(self, *args, **kwargs): + """Retrieve the most recent version of a single STIX object by ID. + """ + try: return self.source.get(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data source to query') - def all_versions(self, *args, **kwargs): - """Retrieve all versions of a single STIX object by ID. - """ + def all_versions(self, *args, **kwargs): + """Retrieve all versions of a single STIX object by ID. + """ + try: return self.source.all_versions(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data source to query') - def query(self, *args, **kwargs): - """Retrieve STIX objects matching a set of filters. - """ + def query(self, *args, **kwargs): + """Retrieve STIX objects matching a set of filters. + """ + try: return self.source.query(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data source to query') - def add_filter(self, *args, **kwargs): - """Add a filter to be applied to all queries for STIX objects from this environment. - """ + def add_filters(self, *args, **kwargs): + """Add multiple filters to be applied to all queries for STIX objects from this environment. + """ + try: + return self.source.add_filters(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data source') + + def add_filter(self, *args, **kwargs): + """Add a filter to be applied to all queries for STIX objects from this environment. + """ + try: return self.source.add_filter(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data source') - def add(self, *args, **kwargs): - """Store a STIX object. - """ + def add(self, *args, **kwargs): + """Store a STIX object. + """ + try: return self.sink.add(*args, **kwargs) + except AttributeError: + raise AttributeError('Environment has no data sink to put objects in') diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 08063f5..e76495e 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -346,6 +346,9 @@ class CompositeDataSource(DataSource): stix_obj (dict): the STIX object to be returned. """ + if not self.get_all_data_sources(): + raise AttributeError('CompositeDataSource has no data sources') + all_data = [] # for every configured Data Source, call its retrieve handler @@ -382,6 +385,9 @@ class CompositeDataSource(DataSource): all_data (list): list of STIX objects that have the specified id """ + if not self.get_all_data_sources(): + raise AttributeError('CompositeDataSource has no data sources') + all_data = [] all_filters = self.filters @@ -416,6 +422,9 @@ class CompositeDataSource(DataSource): all_data (list): list of STIX objects to be returned """ + if not self.get_all_data_sources(): + raise AttributeError('CompositeDataSource has no data sources') + if not query: query = [] diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index 9be8101..26fd1b4 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -1,6 +1,8 @@ +import pytest + import stix2 -from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, +from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS) @@ -81,3 +83,84 @@ def test_object_factory_list_replace(): ind = factory.create(stix2.Indicator, external_references=ext_ref2, **INDICATOR_KWARGS) assert len(ind.external_references) == 1 assert ind.external_references[0].source_name == "Yet Another Threat Report" + + +def test_environment_functions(): + env = stix2.Environment(stix2.ObjectFactory(created_by_ref=IDENTITY_ID), + stix2.MemoryStore()) + + # Create a STIX object + ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS) + assert ind.created_by_ref == IDENTITY_ID + + # Add objects to datastore + ind2 = ind.new_version(labels=['benign']) + env.add([ind, ind2]) + + # Get both versions of the object + resp = env.all_versions(INDICATOR_ID) + assert len(resp) == 1 # should be 2, but MemoryStore only keeps 1 version of objects + + # Get just the most recent version of the object + resp = env.get(INDICATOR_ID) + assert resp['labels'][0] == 'benign' + + # Search on something other than id + query = [stix2.Filter('type', '=', 'vulnerability')] + resp = env.query(query) + assert len(resp) == 0 + + # See different results after adding filters to the environment + env.add_filters([stix2.Filter('type', '=', 'indicator'), + stix2.Filter('created_by_ref', '=', IDENTITY_ID)]) + env.add_filter(stix2.Filter('labels', '=', 'benign')) # should be 'malicious-activity' + resp = env.get(INDICATOR_ID) + assert resp['labels'][0] == 'benign' # should be 'malicious-activity' + + +def test_environment_source_and_sink(): + ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) + env = stix2.Environment(source=stix2.MemorySource([ind]), sink=stix2.MemorySink([ind])) + assert env.get(INDICATOR_ID).labels[0] == 'malicious-activity' + + +def test_environment_datastore_and_sink(): + with pytest.raises(ValueError) as excinfo: + stix2.Environment(factory=stix2.ObjectFactory(), + store=stix2.MemoryStore(), sink=stix2.MemorySink) + assert 'Data store already provided' in str(excinfo.value) + + +def test_environment_no_datastore(): + env = stix2.Environment(factory=stix2.ObjectFactory()) + + with pytest.raises(AttributeError) as excinfo: + env.add(stix2.Indicator(**INDICATOR_KWARGS)) + assert 'Environment has no data sink to put objects in' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.get(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.all_versions(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.query(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.add_filters(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.add_filter(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + +def test_environment_datastore_and_no_object_factory(): + # Uses a default object factory + env = stix2.Environment(store=stix2.MemoryStore()) + ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS) + assert ind.id == INDICATOR_ID From f60331fb7764735df19653a1938bdce199a2ed7b Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 8 Sep 2017 11:15:10 -0400 Subject: [PATCH 3/5] Improve docstrings for Environment layer --- stix2/environment.py | 44 +++++++++++++++++++---------------- stix2/sources/__init__.py | 48 +++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/stix2/environment.py b/stix2/environment.py index 93f8ba3..96726a4 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -1,21 +1,21 @@ import copy -from .sources import CompositeDataSource +from .sources import CompositeDataSource, DataSource, DataStore class ObjectFactory(object): """Easily create STIX objects with default values for certain properties. Args: - created_by_ref: Default created_by_ref value to apply to all + created_by_ref (optional): Default created_by_ref value to apply to all objects created by this factory. - created: Default created value to apply to all + created (optional): Default created value to apply to all objects created by this factory. - external_references: Default `external_references` value to apply + external_references (optional): Default `external_references` value to apply to all objects created by this factory. - object_marking_refs: Default `object_marking_refs` value to apply + object_marking_refs (optional): Default `object_marking_refs` value to apply to all objects created by this factory. - list_append: When a default is set for a list property like + list_append (bool, optional): When a default is set for a list property like `external_references` or `object_marking_refs` and a value for that property is passed into `create()`, if this is set to True, that value will be added to the list alongside the default. If @@ -43,6 +43,13 @@ class ObjectFactory(object): self._list_properties = ['external_references', 'object_marking_refs'] def create(self, cls, **kwargs): + """Create a STIX object using object factory defaults. + + Args: + cls: the python-stix2 class of the object to be created (eg. Indicator) + **kwargs: The property/value pairs of the STIX object to be created + """ + # Use self.defaults as the base, but update with any explicit args # provided by the user. properties = copy.deepcopy(self._defaults) @@ -71,12 +78,12 @@ class Environment(object): """ Args: - factory (ObjectFactory): Factory for creating objects with common + factory (ObjectFactory, optional): Factory for creating objects with common defaults for certain properties. - store (DataStore): Data store providing the source and sink for the + store (DataStore, optional): Data store providing the source and sink for the environment. - source (DataSource): Source for retrieving STIX objects. - sink (DataSink): Destination for saving STIX objects. + source (DataSource, optional): Source for retrieving STIX objects. + sink (DataSink, optional): Destination for saving STIX objects. Invalid if `store` is also provided. """ @@ -94,17 +101,15 @@ class Environment(object): self.sink = sink def create(self, *args, **kwargs): - """Use the object factory to create a STIX object with default property values. - """ return self.factory.create(*args, **kwargs) + create.__doc__ = ObjectFactory.create.__doc__ def get(self, *args, **kwargs): - """Retrieve the most recent version of a single STIX object by ID. - """ try: return self.source.get(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data source to query') + get.__doc__ = DataStore.get.__doc__ def all_versions(self, *args, **kwargs): """Retrieve all versions of a single STIX object by ID. @@ -113,6 +118,7 @@ class Environment(object): return self.source.all_versions(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data source to query') + all_versions.__doc__ = DataStore.all_versions.__doc__ def query(self, *args, **kwargs): """Retrieve STIX objects matching a set of filters. @@ -121,27 +127,25 @@ class Environment(object): return self.source.query(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data source to query') + query.__doc__ = DataStore.query.__doc__ def add_filters(self, *args, **kwargs): - """Add multiple filters to be applied to all queries for STIX objects from this environment. - """ try: return self.source.add_filters(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data source') + add_filters.__doc__ = DataSource.add_filters.__doc__ def add_filter(self, *args, **kwargs): - """Add a filter to be applied to all queries for STIX objects from this environment. - """ try: return self.source.add_filter(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data source') + add_filter.__doc__ = DataSource.add_filter.__doc__ def add(self, *args, **kwargs): - """Store a STIX object. - """ try: return self.sink.add(*args, **kwargs) except AttributeError: raise AttributeError('Environment has no data sink to put objects in') + add.__doc__ = DataStore.add.__doc__ diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index e76495e..cb6e5b5 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -45,30 +45,29 @@ class DataStore(object): self.sink = sink def get(self, stix_id): - """ + """Retrieve the most recent version of a single STIX object by ID. + Notes: Translate API get() call to the appropriate DataSource call. Args: - stix_id (str): the id of the STIX 2.0 object to retrieve. Should - return a single object, the most recent version of the object - specified by the "id". + stix_id (str): the id of the STIX 2.0 object to retrieve. Returns: - stix_obj (dictionary): the STIX object to be returned + stix_obj (dictionary): the single most recent version of the STIX + object specified by the "id". """ return self.source.get(stix_id) def all_versions(self, stix_id): - """ + """Retrieve all versions of a single STIX object by ID. + Implement: Translate all_versions() call to the appropriate DataSource call Args: - stix_id (str): the id of the STIX 2.0 object to retrieve. Should - return a single object, the most recent version of the object - specified by the "id". + stix_id (str): the id of the STIX 2.0 object to retrieve. Returns: stix_objs (list): a list of STIX objects (where each object is a @@ -78,7 +77,8 @@ class DataStore(object): return self.source.all_versions(stix_id) def query(self, query): - """ + """Retrieve STIX objects matching a set of filters. + Notes: Implement the specific data source API calls, processing, functionality required for retrieving query from the data source. @@ -95,10 +95,15 @@ class DataStore(object): return self.source.query(query=query) def add(self, stix_objs): - """ + """Store STIX objects. + Notes: Translate add() to the appropriate DataSink call(). + Args: + stix_objs (list): a list of STIX objects (where each object is a + STIX object) + """ return self.sink.add(stix_objs) @@ -116,11 +121,16 @@ class DataSink(object): self.id = make_id() def add(self, stix_objs): - """ + """Store STIX objects. + Notes: Implement the specific data sink API calls, processing, functionality required for adding data to the sink + Args: + stix_objs (list): a list of STIX objects (where each object is a + STIX object) + """ raise NotImplementedError() @@ -201,16 +211,22 @@ class DataSource(object): raise NotImplementedError() def add_filters(self, filters): - """Add multiple filters to the DataSource. + """Add multiple filters to be applied to all queries for STIX objects. Args: filters (list): list of filters (dict) to add to the Data Source. + """ for filter in filters: self.add_filter(filter) def add_filter(self, filter): - """Add a filter.""" + """Add a filter to be applied to all queries for STIX objects. + + Args: + filter: filter to add to the Data Source. + + """ # check filter field is a supported STIX 2.0 common field if filter.field not in STIX_COMMON_FIELDS: raise ValueError("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported") @@ -407,9 +423,7 @@ class CompositeDataSource(DataSource): return all_data def query(self, query=None, _composite_filters=None): - """Composite data source query - - Federate the query to all Data Sources attached to the + """Federate the query to all Data Sources attached to the Composite Data Source. Args: From be07c3250014db2d11e03ab660711a9bf3464656 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 8 Sep 2017 12:39:36 -0400 Subject: [PATCH 4/5] Add parse() to Environment layer The Environment layer does not keep its own mapping of custom object types. If we think it likely that users will want to maintain separate lists of custom object types between two or more Environments, we can add this later. --- stix2/core.py | 6 +++--- stix2/environment.py | 5 +++++ stix2/test/test_environment.py | 24 +++++++++++++++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/stix2/core.py b/stix2/core.py index be2a53d..0271e34 100644 --- a/stix2/core.py +++ b/stix2/core.py @@ -75,13 +75,13 @@ def parse(data, allow_custom=False): """Deserialize a string or file-like object into a STIX object. Args: - data: The STIX 2 string to be parsed. + data (str, dict, file-like object): The STIX 2 content to be parsed. allow_custom (bool): Whether to allow custom properties or not. Default: False. Returns: An instantiated Python STIX object. - """ + """ obj = get_dict(data) if 'type' not in obj: @@ -96,6 +96,6 @@ def parse(data, allow_custom=False): def _register_type(new_type): """Register a custom STIX Object type. - """ + """ OBJ_MAP[new_type._type] = new_type diff --git a/stix2/environment.py b/stix2/environment.py index 96726a4..8e24c9b 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -1,5 +1,6 @@ import copy +from .core import parse as _parse from .sources import CompositeDataSource, DataSource, DataStore @@ -149,3 +150,7 @@ class Environment(object): except AttributeError: raise AttributeError('Environment has no data sink to put objects in') add.__doc__ = DataStore.add.__doc__ + + def parse(self, *args, **kwargs): + return _parse(*args, **kwargs) + parse.__doc__ = _parse.__doc__ diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index 26fd1b4..0871bb5 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -3,7 +3,7 @@ import pytest import stix2 from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, - INDICATOR_KWARGS) + INDICATOR_KWARGS, MALWARE_ID) def test_object_factory_created_by_ref_str(): @@ -164,3 +164,25 @@ def test_environment_datastore_and_no_object_factory(): env = stix2.Environment(store=stix2.MemoryStore()) ind = env.create(stix2.Indicator, id=INDICATOR_ID, **INDICATOR_KWARGS) assert ind.id == INDICATOR_ID + + +def test_parse_malware(): + env = stix2.Environment() + data = """{ + "type": "malware", + "id": "malware--fedcba98-7654-3210-fedc-ba9876543210", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Cryptolocker", + "labels": [ + "ransomware" + ] + }""" + mal = env.parse(data) + + assert mal.type == 'malware' + assert mal.id == MALWARE_ID + assert mal.created == FAKE_TIME + assert mal.modified == FAKE_TIME + assert mal.labels == ['ransomware'] + assert mal.name == "Cryptolocker" From 190b639126bd18dc58cf9af0e2199e51ad28b637 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 8 Sep 2017 12:49:08 -0400 Subject: [PATCH 5/5] Fix memory datastore to pass tests 1) 'stix_data' is now optional; you can set up a MemorySink without needing a starting set of data. 2) Removed stix2-validator calls. The validator fails when called on our _STIXBase-derived classes because we store timestamps as datetime objects, while the validator expects strings. Also, this was the only datastore that used the validator, and we should be consistent across all of them. The validator can be added back in later. --- stix2/sources/memory.py | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 9eca969..95d053c 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -22,33 +22,22 @@ import collections import json import os -from stix2validator import validate_instance - from stix2 import Bundle from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter -def _add(store, stix_data): +def _add(store, stix_data=None): """Adds stix objects to MemoryStore/Source/Sink.""" if isinstance(stix_data, collections.Mapping): # stix objects are in a bundle - # verify STIX json data - r = validate_instance(stix_data) # make dictionary of the objects for easy lookup - if r.is_valid: - for stix_obj in stix_data["objects"]: - store.data[stix_obj["id"]] = stix_obj - else: - raise ValueError("Error: data passed was found to not be valid by the STIX 2 Validator: \n%s", r.as_dict()) + for stix_obj in stix_data["objects"]: + store.data[stix_obj["id"]] = stix_obj elif isinstance(stix_data, list): # stix objects are in a list for stix_obj in stix_data: - r = validate_instance(stix_obj) - if r.is_valid: - store.data[stix_obj["id"]] = stix_obj - else: - raise ValueError("Error: STIX object %s is not valid under STIX 2 validator.\n%s", stix_obj["id"], r) + store.data[stix_obj["id"]] = stix_obj else: raise ValueError("stix_data must be in bundle format or raw list") @@ -56,7 +45,7 @@ def _add(store, stix_data): class MemoryStore(DataStore): """ """ - def __init__(self, stix_data): + def __init__(self, stix_data=None): """ Notes: It doesn't make sense to create a MemoryStore by passing @@ -83,7 +72,7 @@ class MemoryStore(DataStore): class MemorySink(DataSink): """ """ - def __init__(self, stix_data, _store=False): + def __init__(self, stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -114,7 +103,7 @@ class MemorySink(DataSink): class MemorySource(DataSource): - def __init__(self, stix_data, _store=False): + def __init__(self, stix_data=None, _store=False): """ Args: stix_data (dictionary OR list): valid STIX 2.0 content in @@ -193,10 +182,5 @@ class MemorySource(DataSource): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - r = validate_instance(stix_data) - - if r.is_valid: - for stix_obj in stix_data["objects"]: - self.data[stix_obj["id"]] = stix_obj - - raise ValueError("Error: STIX data loaded from file (%s) was found to not be validated by STIX 2 Validator.\n%s", file_path, r) + for stix_obj in stix_data["objects"]: + self.data[stix_obj["id"]] = stix_obj