MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

ABC for DataSink, DataStore and DataSource. Fixes across the concrete objects

stix2.0
Emmanuelle Vargas-Gonzalez 2017-11-02 21:29:25 -04:00
parent 37e9049536
commit 1e591a827d
4 changed files with 336 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
stix2/sources/__init__.py
View File

@ -11,8 +11,11 @@
| |
""" """
from abc import ABCMeta, abstractmethod
import uuid import uuid
from six import with_metaclass
from stix2.utils import deduplicate from stix2.utils import deduplicate
@ -20,95 +23,94 @@ def make_id():
return str(uuid.uuid4()) return str(uuid.uuid4())
class DataStore(object): class DataStore(with_metaclass(ABCMeta)):
"""An implementer will create a concrete subclass from """An implementer will create a concrete subclass from
this class for the specific DataStore. this class for the specific DataStore.
Args: Args:
source (DataSource): An existing DataSource to use source (DataSource): An existing DataSource to use
as this DataStore's DataSource component as this DataStore's DataSource component
sink (DataSink): An existing DataSink to use sink (DataSink): An existing DataSink to use
as this DataStore's DataSink component as this DataStore's DataSink component
Attributes: Attributes:
id (str): A unique UUIDv4 to identify this DataStore. id (str): A unique UUIDv4 to identify this DataStore.
source (DataSource): An object that implements DataSource class. source (DataSource): An object that implements DataSource class.
sink (DataSink): An object that implements DataSink class. sink (DataSink): An object that implements DataSink class.
""" """
def __init__(self, source=None, sink=None): def __init__(self, source=None, sink=None):
super(DataStore, self).__init__()
self.id = make_id() self.id = make_id()
self.source = source self.source = source
self.sink = sink self.sink = sink
def get(self, stix_id, allow_custom=False): @abstractmethod
def get(self, stix_id): # pragma: no cover
"""Retrieve the most recent version of a single STIX object by ID. """Retrieve the most recent version of a single STIX object by ID.
Translate get() call to the appropriate DataSource call. Translate get() call to the appropriate DataSource call.
Args: Args:
stix_id (str): the id of the STIX object to retrieve. stix_id (str): the id of the STIX object to retrieve.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_obj: the single most recent version of the STIX stix_obj: the single most recent version of the STIX
object specified by the "id". object specified by the "id".
""" """
return self.source.get(stix_id, allow_custom=allow_custom) return NotImplementedError()
def all_versions(self, stix_id, allow_custom=False): @abstractmethod
def all_versions(self, stix_id): # pragma: no cover
"""Retrieve all versions of a single STIX object by ID. """Retrieve all versions of a single STIX object by ID.
Implement: Translate all_versions() call to the appropriate DataSource call Implement: Define a function that performs any custom behavior before
calling the associated DataSource all_versions() method.
Args: Args:
stix_id (str): the id of the STIX object to retrieve. stix_id (str): the id of the STIX object to retrieve.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_objs (list): a list of STIX objects stix_objs (list): a list of STIX objects
""" """
return self.source.all_versions(stix_id, allow_custom=allow_custom) return NotImplementedError()
def query(self, query=None, allow_custom=False): @abstractmethod
def query(self, query=None): # pragma: no cover
"""Retrieve STIX objects matching a set of filters. """Retrieve STIX objects matching a set of filters.
Implement: Specific data source API calls, processing, Implement: Specific data source API calls, processing,
functionality required for retrieving query from the data source. functionality required for retrieving query from the data source.
Define custom behavior before calling the associated DataSource query()
Args: Args:
query (list): a list of filters (which collectively are the query) query (list): a list of filters (which collectively are the query)
to conduct search on. to conduct search on.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_objs (list): a list of STIX objects stix_objs (list): a list of STIX objects
""" """
return self.source.query(query=query) return NotImplementedError()
def add(self, stix_objs, allow_custom=False): @abstractmethod
"""Store STIX objects. def add(self, stix_objs): # pragma: no cover
"""Method for storing STIX objects.
Translates add() to the appropriate DataSink call. Define custom behavior before storing STIX objects using the associated
DataSink. Translates add() to the appropriate DataSink call.
Args: Args:
stix_objs (list): a list of STIX objects stix_objs (list): a list of STIX objects
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
""" """
return self.sink.add(stix_objs, allow_custom=allow_custom) return NotImplementedError()
class DataSink(object): class DataSink(with_metaclass(ABCMeta)):
"""An implementer will create a concrete subclass from """An implementer will create a concrete subclass from
this class for the specific DataSink. this class for the specific DataSink.
@ -117,10 +119,12 @@ class DataSink(object):
""" """
def __init__(self): def __init__(self):
super(DataSink, self).__init__()
self.id = make_id() self.id = make_id()
def add(self, stix_objs, allow_custom=False): @abstractmethod
"""Store STIX objects. def add(self, stix_objs): # pragma: no cover
"""Method for storing STIX objects.
Implement: Specific data sink API calls, processing, Implement: Specific data sink API calls, processing,
functionality required for adding data to the sink functionality required for adding data to the sink
@ -128,28 +132,27 @@ class DataSink(object):
Args: Args:
stix_objs (list): a list of STIX objects (where each object is a stix_objs (list): a list of STIX objects (where each object is a
STIX object) STIX object)
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
""" """
raise NotImplementedError() raise NotImplementedError()
class DataSource(object): class DataSource(with_metaclass(ABCMeta)):
"""An implementer will create a concrete subclass from """An implementer will create a concrete subclass from
this class for the specific DataSource. this class for the specific DataSource.
Attributes: Attributes:
id (str): A unique UUIDv4 to identify this DataSource. id (str): A unique UUIDv4 to identify this DataSource.
filters (set): A collection of filters attached to this DataSource.
_filters (set): A collection of filters attached to this DataSource.
""" """
def __init__(self): def __init__(self):
super(DataSource, self).__init__()
self.id = make_id() self.id = make_id()
self.filters = set() self.filters = set()
def get(self, stix_id, _composite_filters=None, allow_custom=False): @abstractmethod
def get(self, stix_id): # pragma: no cover
""" """
Implement: Specific data source API calls, processing, Implement: Specific data source API calls, processing,
functionality required for retrieving data from the data source functionality required for retrieving data from the data source
@ -158,10 +161,6 @@ class DataSource(object):
stix_id (str): the id of the STIX 2.0 object to retrieve. Should stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object return a single object, the most recent version of the object
specified by the "id". specified by the "id".
_composite_filters (set): set of filters passed from the parent
the CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_obj: the STIX object stix_obj: the STIX object
@ -169,10 +168,11 @@ class DataSource(object):
""" """
raise NotImplementedError() raise NotImplementedError()
def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): @abstractmethod
def all_versions(self, stix_id): # pragma: no cover
""" """
Implement: Similar to get() except returns list of all object versions of Implement: Similar to get() except returns list of all object versions
the specified "id". In addition, implement the specific data of the specified "id". In addition, implement the specific data
source API calls, processing, functionality required for retrieving source API calls, processing, functionality required for retrieving
data from the data source. data from the data source.
@ -180,10 +180,6 @@ class DataSource(object):
stix_id (str): The id of the STIX 2.0 object to retrieve. Should stix_id (str): The id of the STIX 2.0 object to retrieve. Should
return a list of objects, all the versions of the object return a list of objects, all the versions of the object
specified by the "id". specified by the "id".
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_objs (list): a list of STIX objects stix_objs (list): a list of STIX objects
@ -191,18 +187,15 @@ class DataSource(object):
""" """
raise NotImplementedError() raise NotImplementedError()
def query(self, query=None, _composite_filters=None, allow_custom=False): @abstractmethod
def query(self, query=None): # pragma: no cover
""" """
Implement:Implement the specific data source API calls, processing, Implement: The specific data source API calls, processing,
functionality required for retrieving query from the data source functionality required for retrieving query from the data source
Args: Args:
query (list): a list of filters (which collectively are the query) query (list): a list of filters (which collectively are the query)
to conduct search on to conduct search on.
_composite_filters (set): a set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_objs (list): a list of STIX objects stix_objs (list): a list of STIX objects
@ -224,7 +217,7 @@ class CompositeDataSource(DataSource):
Attributes: Attributes:
data_sources (dict): A dictionary of DataSource objects; to be data_sources (list): A dictionary of DataSource objects; to be
controlled and used by the Data Source Controller object. controlled and used by the Data Source Controller object.
""" """
@ -237,7 +230,7 @@ class CompositeDataSource(DataSource):
super(CompositeDataSource, self).__init__() super(CompositeDataSource, self).__init__()
self.data_sources = [] self.data_sources = []
def get(self, stix_id, _composite_filters=None, allow_custom=False): def get(self, stix_id, _composite_filters=None):
"""Retrieve STIX object by STIX ID """Retrieve STIX object by STIX ID
Federated retrieve method, iterates through all DataSources Federated retrieve method, iterates through all DataSources
@ -253,9 +246,7 @@ class CompositeDataSource(DataSource):
stix_id (str): the id of the STIX object to retrieve. stix_id (str): the id of the STIX object to retrieve.
_composite_filters (list): a list of filters passed from a _composite_filters (list): a list of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is attached CompositeDataSource (i.e. if this CompositeDataSource is attached
to another parent CompositeDataSource), not user supplied to another parent CompositeDataSource), not user supplied.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
stix_obj: the STIX object to be returned. stix_obj: the STIX object to be returned.
@ -273,7 +264,7 @@ class CompositeDataSource(DataSource):
# for every configured Data Source, call its retrieve handler # for every configured Data Source, call its retrieve handler
for ds in self.data_sources: for ds in self.data_sources:
data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) data = ds.get(stix_id=stix_id, _composite_filters=all_filters)
if data: if data:
all_data.append(data) all_data.append(data)
@ -288,22 +279,20 @@ class CompositeDataSource(DataSource):
return stix_obj return stix_obj
def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): def all_versions(self, stix_id, _composite_filters=None):
"""Retrieve STIX objects by STIX ID """Retrieve all versions of a STIX object by STIX ID.
Federated all_versions retrieve method - iterates through all DataSources Federated all_versions retrieve method - iterates through all
defined in "data_sources" DataSources defined in "data_sources".
A composite data source will pass its attached filters to A composite data source will pass its attached filters to
each configured data source, pushing filtering to them to handle each configured data source, pushing filtering to them to handle.
Args: Args:
stix_id (str): id of the STIX objects to retrieve stix_id (str): id of the STIX objects to retrieve.
_composite_filters (list): a list of filters passed from a _composite_filters (list): a list of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is attached CompositeDataSource (i.e. if this CompositeDataSource is
to a parent CompositeDataSource), not user supplied attached to a parent CompositeDataSource), not user supplied.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
all_data (list): list of STIX objects that have the specified id all_data (list): list of STIX objects that have the specified id
@ -322,7 +311,7 @@ class CompositeDataSource(DataSource):
# retrieve STIX objects from all configured data sources # retrieve STIX objects from all configured data sources
for ds in self.data_sources: for ds in self.data_sources:
data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters)
all_data.extend(data) all_data.extend(data)
# remove exact duplicates (where duplicates are STIX 2.0 objects # remove exact duplicates (where duplicates are STIX 2.0 objects
@ -332,19 +321,17 @@ class CompositeDataSource(DataSource):
return all_data return all_data
def query(self, query=None, _composite_filters=None, allow_custom=False): def query(self, query=None, _composite_filters=None):
"""Retrieve STIX objects that match query """Retrieve STIX objects that match a query.
Federate the query to all DataSources attached to the Federate the query to all DataSources attached to the
Composite Data Source. Composite Data Source.
Args: Args:
query (list): list of filters to search on query (list): list of filters to search on.
_composite_filters (list): a list of filters passed from a _composite_filters (list): a list of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is attached CompositeDataSource (i.e. if this CompositeDataSource is
to a parent CompositeDataSource), not user supplied attached to a parent CompositeDataSource), not user supplied.
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
all_data (list): list of STIX objects to be returned all_data (list): list of STIX objects to be returned
@ -354,7 +341,7 @@ class CompositeDataSource(DataSource):
raise AttributeError('CompositeDataSource has no data sources') raise AttributeError('CompositeDataSource has no data sources')
if not query: if not query:
# dont mess with the query (i.e. convert to a set, as thats done # don't mess with the query (i.e. convert to a set, as that's done
# within the specific DataSources that are called) # within the specific DataSources that are called)
query = [] query = []
@ -369,7 +356,7 @@ class CompositeDataSource(DataSource):
# federate query to all attached data sources, # federate query to all attached data sources,
# pass composite filters to id # pass composite filters to id
for ds in self.data_sources: for ds in self.data_sources:
data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) data = ds.query(query=query, _composite_filters=all_filters)
all_data.extend(data) all_data.extend(data)
# remove exact duplicates (where duplicates are STIX 2.0 # remove exact duplicates (where duplicates are STIX 2.0

103
stix2/sources/filesystem.py
View File

@ -26,7 +26,7 @@ class FileSystemStore(DataStore):
Default: False. Default: False.
Attributes: Attributes:
source (FileSystemSource): FuleSystemSource source (FileSystemSource): FileSystemSource
sink (FileSystemSink): FileSystemSink sink (FileSystemSink): FileSystemSink
""" """
@ -35,6 +35,85 @@ class FileSystemStore(DataStore):
self.source = FileSystemSource(stix_dir=stix_dir) self.source = FileSystemSource(stix_dir=stix_dir)
self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify)
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve the most recent version of a single STIX object by ID.
Translate get() call to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns:
stix_obj: the single most recent version of the STIX
object specified by the "id".
"""
return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve all versions of a single STIX object by ID.
Implement: Translate all_versions() call to the appropriate DataSource
call.
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve STIX objects matching a set of filters.
Implement: Specific data source API calls, processing,
functionality required for retrieving query from the data source.
Args:
query (list): a list of filters (which collectively are the query)
to conduct search on.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
def add(self, stix_objs, allow_custom=False, version=None):
"""Store STIX objects.
Translates add() to the appropriate DataSink call.
Args:
stix_objs (list): a list of STIX objects
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
"""
return self.sink.add(stix_objs, allow_custom=allow_custom, version=version)
class FileSystemSink(DataSink): class FileSystemSink(DataSink):
"""Interface for adding/pushing STIX objects to file directory of STIX """Interface for adding/pushing STIX objects to file directory of STIX
@ -99,11 +178,11 @@ class FileSystemSink(DataSink):
self._check_path_and_write(stix_data) self._check_path_and_write(stix_data)
elif isinstance(stix_data, (str, dict)): elif isinstance(stix_data, (str, dict)):
stix_data = parse(stix_data, allow_custom, version) stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
if stix_data["type"] == "bundle": if stix_data["type"] == "bundle":
# extract STIX objects # extract STIX objects
for stix_obj in stix_data.get("objects", []): for stix_obj in stix_data.get("objects", []):
self.add(stix_obj) self.add(stix_obj, allow_custom=allow_custom, version=version)
else: else:
# adding json-formatted STIX # adding json-formatted STIX
self._check_path_and_write(stix_data) self._check_path_and_write(stix_data)
@ -111,12 +190,12 @@ class FileSystemSink(DataSink):
elif isinstance(stix_data, Bundle): elif isinstance(stix_data, Bundle):
# recursively add individual STIX objects # recursively add individual STIX objects
for stix_obj in stix_data.get("objects", []): for stix_obj in stix_data.get("objects", []):
self.add(stix_obj) self.add(stix_obj, allow_custom=allow_custom, version=version)
elif isinstance(stix_data, list): elif isinstance(stix_data, list):
# recursively add individual STIX objects # recursively add individual STIX objects
for stix_obj in stix_data: for stix_obj in stix_data:
self.add(stix_obj) self.add(stix_obj, allow_custom=allow_custom, version=version)
else: else:
raise TypeError("stix_data must be a STIX object (or list of), " raise TypeError("stix_data must be a STIX object (or list of), "
@ -146,7 +225,7 @@ class FileSystemSource(DataSource):
def stix_dir(self): def stix_dir(self):
return self._stix_dir return self._stix_dir
def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve STIX object from file directory via STIX ID. """Retrieve STIX object from file directory via STIX ID.
Args: Args:
@ -166,8 +245,7 @@ class FileSystemSource(DataSource):
""" """
query = [Filter("id", "=", stix_id)] query = [Filter("id", "=", stix_id)]
all_data = self.query(query=query, _composite_filters=_composite_filters, all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
allow_custom=allow_custom, version=version)
if all_data: if all_data:
stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
@ -176,7 +254,7 @@ class FileSystemSource(DataSource):
return stix_obj return stix_obj
def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, version=None): def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve STIX object from file directory via STIX ID, all versions. """Retrieve STIX object from file directory via STIX ID, all versions.
Note: Since FileSystem sources/sinks don't handle multiple versions Note: Since FileSystem sources/sinks don't handle multiple versions
@ -197,10 +275,9 @@ class FileSystemSource(DataSource):
a python STIX objects and then returned a python STIX objects and then returned
""" """
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)]
allow_custom=allow_custom, version=version)]
def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
"""Search and retrieve STIX objects based on the complete query. """Search and retrieve STIX objects based on the complete query.
A "complete query" includes the filters from the query, the filters A "complete query" includes the filters from the query, the filters
@ -305,7 +382,7 @@ class FileSystemSource(DataSource):
all_data = deduplicate(all_data) all_data = deduplicate(all_data)
# parse python STIX objects from the STIX object dicts # parse python STIX objects from the STIX object dicts
stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data]
return stix_objs return stix_objs

131
stix2/sources/memory.py
View File

@ -24,7 +24,7 @@ from stix2.sources import DataSink, DataSource, DataStore
from stix2.sources.filters import Filter, apply_common_filters from stix2.sources.filters import Filter, apply_common_filters
def _add(store, stix_data=None, allow_custom=False): def _add(store, stix_data=None, allow_custom=False, version=None):
"""Add STIX objects to MemoryStore/Sink. """Add STIX objects to MemoryStore/Sink.
Adds STIX objects to an in-memory dictionary for fast lookup. Adds STIX objects to an in-memory dictionary for fast lookup.
@ -34,6 +34,8 @@ def _add(store, stix_data=None, allow_custom=False):
stix_data (list OR dict OR STIX object): STIX objects to be added stix_data (list OR dict OR STIX object): STIX objects to be added
allow_custom (bool): whether to allow custom objects/properties or allow_custom (bool): whether to allow custom objects/properties or
not. Default: False. not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
""" """
if isinstance(stix_data, _STIXBase): if isinstance(stix_data, _STIXBase):
@ -44,25 +46,25 @@ def _add(store, stix_data=None, allow_custom=False):
if stix_data["type"] == "bundle": if stix_data["type"] == "bundle":
# adding a json bundle - so just grab STIX objects # adding a json bundle - so just grab STIX objects
for stix_obj in stix_data.get("objects", []): for stix_obj in stix_data.get("objects", []):
_add(store, stix_obj, allow_custom=allow_custom) _add(store, stix_obj, allow_custom=allow_custom, version=version)
else: else:
# adding a json STIX object # adding a json STIX object
store._data[stix_data["id"]] = stix_data store._data[stix_data["id"]] = stix_data
elif isinstance(stix_data, str): elif isinstance(stix_data, str):
# adding json encoded string of STIX content # adding json encoded string of STIX content
stix_data = parse(stix_data, allow_custom=allow_custom) stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
if stix_data["type"] == "bundle": if stix_data["type"] == "bundle":
# recurse on each STIX object in bundle # recurse on each STIX object in bundle
for stix_obj in stix_data.get("objects", []): for stix_obj in stix_data.get("objects", []):
_add(store, stix_obj, allow_custom=allow_custom) _add(store, stix_obj, allow_custom=allow_custom, version=version)
else: else:
_add(store, stix_data) _add(store, stix_data, allow_custom=allow_custom, version=version)
elif isinstance(stix_data, list): elif isinstance(stix_data, list):
# STIX objects are in a list- recurse on each object # STIX objects are in a list- recurse on each object
for stix_obj in stix_data: for stix_obj in stix_data:
_add(store, stix_obj, allow_custom=allow_custom) _add(store, stix_obj, allow_custom=allow_custom, version=version)
else: else:
raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle")
@ -81,6 +83,8 @@ class MemoryStore(DataStore):
stix_data (list OR dict OR STIX object): STIX content to be added stix_data (list OR dict OR STIX object): STIX content to be added
allow_custom (bool): whether to allow custom objects/properties or allow_custom (bool): whether to allow custom objects/properties or
not. Default: False. not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Attributes: Attributes:
_data (dict): the in-memory dict that holds STIX objects _data (dict): the in-memory dict that holds STIX objects
@ -88,15 +92,15 @@ class MemoryStore(DataStore):
sink (MemorySink): MemorySink sink (MemorySink): MemorySink
""" """
def __init__(self, stix_data=None, allow_custom=False): def __init__(self, stix_data=None, allow_custom=False, version=None):
super(MemoryStore, self).__init__() super(MemoryStore, self).__init__()
self._data = {} self._data = {}
if stix_data: if stix_data:
_add(self, stix_data, allow_custom=allow_custom) _add(self, stix_data, allow_custom=allow_custom, version=version)
self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) self.source = MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True)
self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) self.sink = MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True)
def save_to_file(self, file_path, allow_custom=False): def save_to_file(self, file_path, allow_custom=False):
"""Write SITX objects from in-memory dictionary to JSON file, as a STIX """Write SITX objects from in-memory dictionary to JSON file, as a STIX
@ -110,7 +114,7 @@ class MemoryStore(DataStore):
""" """
return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom)
def load_from_file(self, file_path, allow_custom=False): def load_from_file(self, file_path, allow_custom=False, version=None):
"""Load STIX data from JSON file. """Load STIX data from JSON file.
File format is expected to be a single JSON File format is expected to be a single JSON
@ -120,9 +124,72 @@ class MemoryStore(DataStore):
file_path (str): file path to load STIX data from file_path (str): file path to load STIX data from
allow_custom (bool): whether to allow custom objects/properties or allow_custom (bool): whether to allow custom objects/properties or
not. Default: False. not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
""" """
return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version)
def get(self, stix_id, _composite_filters=None):
"""Retrieve the most recent version of a single STIX object by ID.
Translate get() call to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
stix_obj: the single most recent version of the STIX
object specified by the "id".
"""
return self.source.get(stix_id, _composite_filters=_composite_filters)
def all_versions(self, stix_id, _composite_filters=None):
"""Retrieve all versions of a single STIX object by ID.
Translate all_versions() call to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.all_versions(stix_id, _composite_filters=_composite_filters)
def query(self, query=None, _composite_filters=None):
"""Retrieve STIX objects matching a set of filters.
Translates query() to appropriate DataStore call.
Args:
query (list): a list of filters (which collectively are the query)
to conduct search on.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.query(query=query, _composite_filters=_composite_filters)
def add(self, stix_objs, allow_custom=False, version=None):
"""Store STIX objects.
Translates add() to the appropriate DataSink call.
Args:
stix_objs (list): a list of STIX objects
"""
return self.sink.add(stix_objs, allow_custom=allow_custom, version=version)
class MemorySink(DataSink): class MemorySink(DataSink):
@ -146,17 +213,17 @@ class MemorySink(DataSink):
a MemorySource a MemorySource
""" """
def __init__(self, stix_data=None, _store=False, allow_custom=False): def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False):
super(MemorySink, self).__init__() super(MemorySink, self).__init__()
self._data = {} self._data = {}
if _store: if _store:
self._data = stix_data self._data = stix_data
elif stix_data: elif stix_data:
_add(self, stix_data, allow_custom=allow_custom) _add(self, stix_data, allow_custom=allow_custom, version=version)
def add(self, stix_data, allow_custom=False): def add(self, stix_data, allow_custom=False, version=None):
_add(self, stix_data, allow_custom=allow_custom) _add(self, stix_data, allow_custom=allow_custom, version=version)
add.__doc__ = _add.__doc__ add.__doc__ = _add.__doc__
def save_to_file(self, file_path, allow_custom=False): def save_to_file(self, file_path, allow_custom=False):
@ -190,24 +257,22 @@ class MemorySource(DataSource):
a MemorySink a MemorySink
""" """
def __init__(self, stix_data=None, _store=False, allow_custom=False): def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False):
super(MemorySource, self).__init__() super(MemorySource, self).__init__()
self._data = {} self._data = {}
if _store: if _store:
self._data = stix_data self._data = stix_data
elif stix_data: elif stix_data:
_add(self, stix_data, allow_custom=allow_custom) _add(self, stix_data, allow_custom=allow_custom, version=version)
def get(self, stix_id, _composite_filters=None, allow_custom=False): def get(self, stix_id, _composite_filters=None):
"""Retrieve STIX object from in-memory dict via STIX ID. """Retrieve STIX object from in-memory dict via STIX ID.
Args: Args:
stix_id (str): The STIX ID of the STIX object to be retrieved. stix_id (str): The STIX ID of the STIX object to be retrieved.
composite_filters (set): set of filters passed from the parent _composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
(dict OR STIX object): STIX object that has the supplied (dict OR STIX object): STIX object that has the supplied
@ -227,7 +292,7 @@ class MemorySource(DataSource):
# if there are filters from the composite level, process full query # if there are filters from the composite level, process full query
query = [Filter("id", "=", stix_id)] query = [Filter("id", "=", stix_id)]
all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) all_data = self.query(query=query, _composite_filters=_composite_filters)
if all_data: if all_data:
# reduce to most recent version # reduce to most recent version
@ -237,7 +302,7 @@ class MemorySource(DataSource):
else: else:
return None return None
def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): def all_versions(self, stix_id, _composite_filters=None):
"""Retrieve STIX objects from in-memory dict via STIX ID, all versions of it """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it
Note: Since Memory sources/sinks don't handle multiple versions of a Note: Since Memory sources/sinks don't handle multiple versions of a
@ -245,10 +310,8 @@ class MemorySource(DataSource):
Args: Args:
stix_id (str): The STIX ID of the STIX 2 object to retrieve. stix_id (str): The STIX ID of the STIX 2 object to retrieve.
composite_filters (set): set of filters passed from the parent _composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
(list): list of STIX objects that has the supplied ID. As the (list): list of STIX objects that has the supplied ID. As the
@ -257,9 +320,9 @@ class MemorySource(DataSource):
is returned in the same form as it as added is returned in the same form as it as added
""" """
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
def query(self, query=None, _composite_filters=None, allow_custom=False): def query(self, query=None, _composite_filters=None):
"""Search and retrieve STIX objects based on the complete query. """Search and retrieve STIX objects based on the complete query.
A "complete query" includes the filters from the query, the filters A "complete query" includes the filters from the query, the filters
@ -268,10 +331,8 @@ class MemorySource(DataSource):
Args: Args:
query (list): list of filters to search on query (list): list of filters to search on
composite_filters (set): set of filters passed from the _composite_filters (set): set of filters passed from the
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
Returns: Returns:
(list): list of STIX objects that matches the supplied (list): list of STIX objects that matches the supplied
@ -284,7 +345,7 @@ class MemorySource(DataSource):
query = set() query = set()
else: else:
if not isinstance(query, list): if not isinstance(query, list):
# make sure dont make set from a Filter object, # make sure don't make set from a Filter object,
# need to make a set from a list of Filter objects (even if just one Filter) # need to make a set from a list of Filter objects (even if just one Filter)
query = [query] query = [query]
query = set(query) query = set(query)
@ -300,8 +361,8 @@ class MemorySource(DataSource):
return all_data return all_data
def load_from_file(self, file_path, allow_custom=False): def load_from_file(self, file_path, allow_custom=False, version=None):
file_path = os.path.abspath(file_path) file_path = os.path.abspath(file_path)
stix_data = json.load(open(file_path, "r")) stix_data = json.load(open(file_path, "r"))
_add(self, stix_data, allow_custom=allow_custom) _add(self, stix_data, allow_custom=allow_custom, version=version)
load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ load_from_file.__doc__ = MemoryStore.load_from_file.__doc__

101
stix2/sources/taxii.py
View File

@ -1,5 +1,5 @@
""" """
Python STIX 2.x TaxiiCollectionStore Python STIX 2.x TAXIICollectionStore
""" """
from stix2.base import _STIXBase from stix2.base import _STIXBase
@ -24,6 +24,71 @@ class TAXIICollectionStore(DataStore):
self.source = TAXIICollectionSource(collection) self.source = TAXIICollectionSource(collection)
self.sink = TAXIICollectionSink(collection) self.sink = TAXIICollectionSink(collection)
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve the most recent version of a single STIX object by ID.
Translate get() call to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns:
stix_obj: the single most recent version of the STIX
object specified by the "id".
"""
return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
def all_versions(self, stix_id):
"""Retrieve all versions of a single STIX object by ID.
Translate all_versions() to the appropriate DataSource call.
Args:
stix_id (str): the id of the STIX object to retrieve.
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.all_versions(stix_id)
def query(self, query=None):
"""Retrieve STIX objects matching a set of filters.
Translate query() to the appropriate DataSource call.
Args:
query (list): a list of filters (which collectively are the query)
to conduct search on.
Returns:
stix_objs (list): a list of STIX objects
"""
return self.source.query(query=query)
def add(self, stix_objs, allow_custom=False, version=None):
"""Store STIX objects.
Translate add() to the appropriate DataSink call.
Args:
stix_objs (list): a list of STIX objects
allow_custom (bool): whether to allow custom objects/properties or
not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
"""
return self.sink.add(stix_objs, allow_custom=allow_custom, version=version)
class TAXIICollectionSink(DataSink): class TAXIICollectionSink(DataSink):
"""Provides an interface for pushing STIX objects to a local/remote """Provides an interface for pushing STIX objects to a local/remote
@ -37,7 +102,7 @@ class TAXIICollectionSink(DataSink):
super(TAXIICollectionSink, self).__init__() super(TAXIICollectionSink, self).__init__()
self.collection = collection self.collection = collection
def add(self, stix_data, allow_custom=False): def add(self, stix_data, allow_custom=False, version=None):
"""Add/push STIX content to TAXII Collection endpoint """Add/push STIX content to TAXII Collection endpoint
Args: Args:
@ -46,6 +111,8 @@ class TAXIICollectionSink(DataSink):
json encoded string, or list of any of the following json encoded string, or list of any of the following
allow_custom (bool): whether to allow custom objects/properties or allow_custom (bool): whether to allow custom objects/properties or
not. Default: False. not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
""" """
if isinstance(stix_data, _STIXBase): if isinstance(stix_data, _STIXBase):
@ -62,11 +129,11 @@ class TAXIICollectionSink(DataSink):
elif isinstance(stix_data, list): elif isinstance(stix_data, list):
# adding list of something - recurse on each # adding list of something - recurse on each
for obj in stix_data: for obj in stix_data:
self.add(obj, allow_custom=allow_custom) self.add(obj, allow_custom=allow_custom, version=version)
elif isinstance(stix_data, str): elif isinstance(stix_data, str):
# adding json encoded string of STIX content # adding json encoded string of STIX content
stix_data = parse(stix_data, allow_custom=allow_custom) stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
if stix_data["type"] == "bundle": if stix_data["type"] == "bundle":
bundle = dict(stix_data) bundle = dict(stix_data)
else: else:
@ -90,16 +157,18 @@ class TAXIICollectionSource(DataSource):
super(TAXIICollectionSource, self).__init__() super(TAXIICollectionSource, self).__init__()
self.collection = collection self.collection = collection
def get(self, stix_id, _composite_filters=None, allow_custom=False): def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None):
"""Retrieve STIX object from local/remote STIX Collection """Retrieve STIX object from local/remote STIX Collection
endpoint. endpoint.
Args: Args:
stix_id (str): The STIX ID of the STIX object to be retrieved. stix_id (str): The STIX ID of the STIX object to be retrieved.
composite_filters (set): set of filters passed from the parent _composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False. or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns: Returns:
(STIX object): STIX object that has the supplied STIX ID. (STIX object): STIX object that has the supplied STIX ID.
@ -121,7 +190,7 @@ class TAXIICollectionSource(DataSource):
stix_obj = list(apply_common_filters(stix_objs, query)) stix_obj = list(apply_common_filters(stix_objs, query))
if len(stix_obj): if len(stix_obj):
stix_obj = parse(stix_obj[0], allow_custom=allow_custom) stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version)
if stix_obj.id != stix_id: if stix_obj.id != stix_id:
# check - was added to handle erroneous TAXII servers # check - was added to handle erroneous TAXII servers
stix_obj = None stix_obj = None
@ -130,16 +199,18 @@ class TAXIICollectionSource(DataSource):
return stix_obj return stix_obj
def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
"""Retrieve STIX object from local/remote TAXII Collection """Retrieve STIX object from local/remote TAXII Collection
endpoint, all versions of it endpoint, all versions of it
Args: Args:
stix_id (str): The STIX ID of the STIX objects to be retrieved. stix_id (str): The STIX ID of the STIX objects to be retrieved.
composite_filters (set): set of filters passed from the parent _composite_filters (set): set of filters passed from the parent
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False. or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns: Returns:
(see query() as all_versions() is just a wrapper) (see query() as all_versions() is just a wrapper)
@ -154,14 +225,14 @@ class TAXIICollectionSource(DataSource):
all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom)
# parse STIX objects from TAXII returned json # parse STIX objects from TAXII returned json
all_data = [parse(stix_obj) for stix_obj in all_data] all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data]
# check - was added to handle erroneous TAXII servers # check - was added to handle erroneous TAXII servers
all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id]
return all_data_clean return all_data_clean
def query(self, query=None, _composite_filters=None, allow_custom=False): def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
"""Search and retreive STIX objects based on the complete query """Search and retreive STIX objects based on the complete query
A "complete query" includes the filters from the query, the filters A "complete query" includes the filters from the query, the filters
@ -170,10 +241,12 @@ class TAXIICollectionSource(DataSource):
Args: Args:
query (list): list of filters to search on query (list): list of filters to search on
composite_filters (set): set of filters passed from the _composite_filters (set): set of filters passed from the
CompositeDataSource, not user supplied CompositeDataSource, not user supplied
allow_custom (bool): whether to retrieve custom objects/properties allow_custom (bool): whether to retrieve custom objects/properties
or not. Default: False. or not. Default: False.
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
Returns: Returns:
(list): list of STIX objects that matches the supplied (list): list of STIX objects that matches the supplied
@ -200,7 +273,7 @@ class TAXIICollectionSource(DataSource):
taxii_filters = self._parse_taxii_filters(query) taxii_filters = self._parse_taxii_filters(query)
# query TAXII collection # query TAXII collection
all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
# deduplicate data (before filtering as reduces wasted filtering) # deduplicate data (before filtering as reduces wasted filtering)
all_data = deduplicate(all_data) all_data = deduplicate(all_data)
@ -209,7 +282,7 @@ class TAXIICollectionSource(DataSource):
all_data = list(apply_common_filters(all_data, query)) all_data = list(apply_common_filters(all_data, query))
# parse python STIX objects from the STIX object dicts # parse python STIX objects from the STIX object dicts
stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data]
return stix_objs return stix_objs