716 lines
24 KiB
Python
716 lines
24 KiB
Python
"""
|
|
Python STIX2 DataStore API.
|
|
|
|
.. autosummary::
|
|
:toctree: datastore
|
|
|
|
filesystem
|
|
filters
|
|
memory
|
|
taxii
|
|
|
|
|
|
|
"""
|
|
|
|
from abc import ABCMeta, abstractmethod
|
|
import uuid
|
|
|
|
from six import with_metaclass
|
|
|
|
from stix2.datastore.filters import Filter, FilterSet
|
|
from stix2.utils import deduplicate
|
|
|
|
|
|
def make_id():
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
class DataSourceError(Exception):
|
|
"""General DataSource error instance, used primarily for wrapping
|
|
lower level errors
|
|
|
|
Args:
|
|
message (str): error message
|
|
root_exception (Exception): Exception instance of root exception
|
|
in the case that DataSourceError is wrapping a lower level or
|
|
other exception
|
|
"""
|
|
def __init__(self, message, root_exception=None):
|
|
self.message = message
|
|
self.root_exception = root_exception
|
|
|
|
def __str__(self):
|
|
if self.root_exception:
|
|
return "{} \"{}\"".format(self.message, self.root_exception)
|
|
else:
|
|
return self.message
|
|
|
|
|
|
class DataStoreMixin(object):
|
|
"""Provides mechanisms for storing and retrieving STIX data. The specific
|
|
behavior can be customized by subclasses.
|
|
|
|
Args:
|
|
source (DataSource): An existing DataSource to use
|
|
as this DataStore's DataSource component
|
|
sink (DataSink): An existing DataSink to use
|
|
as this DataStore's DataSink component
|
|
|
|
Attributes:
|
|
id (str): A unique UUIDv4 to identify this DataStore.
|
|
source (DataSource): An object that implements DataSource class.
|
|
sink (DataSink): An object that implements DataSink class.
|
|
|
|
"""
|
|
def __init__(self, source=None, sink=None):
|
|
super(DataStoreMixin, self).__init__()
|
|
self.id = make_id()
|
|
self.source = source
|
|
self.sink = sink
|
|
|
|
def get(self, *args, **kwargs):
|
|
"""Retrieve the most recent version of a single STIX object by ID.
|
|
|
|
Translate get() call to the appropriate DataSource call.
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX object to retrieve.
|
|
|
|
Returns:
|
|
stix_obj: the single most recent version of the STIX
|
|
object specified by the "id".
|
|
|
|
"""
|
|
try:
|
|
return self.source.get(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def all_versions(self, *args, **kwargs):
|
|
"""Retrieve all versions of a single STIX object by ID.
|
|
|
|
Translate all_versions() call to the appropriate DataSource call.
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX object to retrieve.
|
|
|
|
Returns:
|
|
list: All versions of the specified STIX object.
|
|
|
|
"""
|
|
try:
|
|
return self.source.all_versions(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def query(self, *args, **kwargs):
|
|
"""Retrieve STIX objects matching a set of filters.
|
|
|
|
Translate query() call to the appropriate DataSource call.
|
|
|
|
Args:
|
|
query (list): a list of filters (which collectively are the query)
|
|
to conduct search on.
|
|
|
|
Returns:
|
|
list: The STIX objects matching the query.
|
|
|
|
"""
|
|
try:
|
|
return self.source.query(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def creator_of(self, *args, **kwargs):
|
|
"""Retrieve the Identity refered to by the object's `created_by_ref`.
|
|
|
|
Translate creator_of() call to the appropriate DataSource call.
|
|
|
|
Args:
|
|
obj: The STIX object whose `created_by_ref` property will be looked
|
|
up.
|
|
|
|
Returns:
|
|
The STIX object's creator, or None, if the object contains no
|
|
`created_by_ref` property or the object's creator cannot be found.
|
|
|
|
"""
|
|
try:
|
|
return self.source.creator_of(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def relationships(self, *args, **kwargs):
|
|
"""Retrieve Relationships involving the given STIX object.
|
|
|
|
Translate relationships() call to the appropriate DataSource call.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
relationships will be looked up.
|
|
relationship_type (str): Only retrieve Relationships of this type.
|
|
If None, all relationships will be returned, regardless of type.
|
|
source_only (bool): Only retrieve Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only retrieve Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
|
|
Returns:
|
|
list: The Relationship objects involving the given STIX object.
|
|
|
|
"""
|
|
try:
|
|
return self.source.relationships(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def related_to(self, *args, **kwargs):
|
|
"""Retrieve STIX Objects that have a Relationship involving the given
|
|
STIX object.
|
|
|
|
Translate related_to() call to the appropriate DataSource call.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
related objects will be looked up.
|
|
relationship_type (str): Only retrieve objects related by this
|
|
Relationships type. If None, all related objects will be
|
|
returned, regardless of type.
|
|
source_only (bool): Only examine Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only examine Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
filters (list): list of additional filters the related objects must
|
|
match.
|
|
|
|
Returns:
|
|
list: The STIX objects related to the given STIX object.
|
|
|
|
"""
|
|
try:
|
|
return self.source.related_to(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data source to query"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
def add(self, *args, **kwargs):
|
|
"""Method for storing STIX objects.
|
|
|
|
Defines custom behavior before storing STIX objects using the
|
|
appropriate method call on the associated DataSink.
|
|
|
|
Args:
|
|
stix_objs (list): a list of STIX objects
|
|
|
|
"""
|
|
try:
|
|
return self.sink.add(*args, **kwargs)
|
|
except AttributeError:
|
|
msg = "%s has no data sink to put objects in"
|
|
raise AttributeError(msg % self.__class__.__name__)
|
|
|
|
|
|
class DataSink(with_metaclass(ABCMeta)):
|
|
"""An implementer will create a concrete subclass from
|
|
this class for the specific DataSink.
|
|
|
|
Attributes:
|
|
id (str): A unique UUIDv4 to identify this DataSink.
|
|
|
|
"""
|
|
def __init__(self):
|
|
super(DataSink, self).__init__()
|
|
self.id = make_id()
|
|
|
|
@abstractmethod
|
|
def add(self, stix_objs):
|
|
"""Method for storing STIX objects.
|
|
|
|
Implement: Specific data sink API calls, processing,
|
|
functionality required for adding data to the sink
|
|
|
|
Args:
|
|
stix_objs (list): a list of STIX objects (where each object is a
|
|
STIX object)
|
|
|
|
"""
|
|
|
|
|
|
class DataSource(with_metaclass(ABCMeta)):
|
|
"""An implementer will create a concrete subclass from
|
|
this class for the specific DataSource.
|
|
|
|
Attributes:
|
|
id (str): A unique UUIDv4 to identify this DataSource.
|
|
filters (FilterSet): A collection of filters attached to this DataSource.
|
|
|
|
"""
|
|
def __init__(self):
|
|
super(DataSource, self).__init__()
|
|
self.id = make_id()
|
|
self.filters = FilterSet()
|
|
|
|
@abstractmethod
|
|
def get(self, stix_id):
|
|
"""
|
|
Implement: Specific data source API calls, processing,
|
|
functionality required for retrieving data from the data source
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX 2.0 object to retrieve. Should
|
|
return a single object, the most recent version of the object
|
|
specified by the "id".
|
|
|
|
Returns:
|
|
stix_obj: The STIX object.
|
|
|
|
"""
|
|
|
|
@abstractmethod
|
|
def all_versions(self, stix_id):
|
|
"""
|
|
Implement: Similar to get() except returns list of all object versions
|
|
of the specified "id". In addition, implement the specific data
|
|
source API calls, processing, functionality required for retrieving
|
|
data from the data source.
|
|
|
|
Args:
|
|
stix_id (str): The id of the STIX 2.0 object to retrieve. Should
|
|
return a list of objects, all the versions of the object
|
|
specified by the "id".
|
|
|
|
Returns:
|
|
list: All versions of the specified STIX object.
|
|
|
|
"""
|
|
|
|
@abstractmethod
|
|
def query(self, query=None):
|
|
"""
|
|
Implement: The specific data source API calls, processing,
|
|
functionality required for retrieving query from the data source
|
|
|
|
Args:
|
|
query (list): a list of filters (which collectively are the query)
|
|
to conduct search on.
|
|
|
|
Returns:
|
|
list: The STIX objects that matched the query.
|
|
|
|
"""
|
|
|
|
def creator_of(self, obj):
|
|
"""Retrieve the Identity referred to by the object's `created_by_ref`.
|
|
|
|
Args:
|
|
obj: The STIX object whose `created_by_ref` property will be looked
|
|
up.
|
|
|
|
Returns:
|
|
The STIX object's creator, or None, if the object contains no
|
|
`created_by_ref` property or the object's creator cannot be found.
|
|
|
|
"""
|
|
creator_id = obj.get('created_by_ref', '')
|
|
if creator_id:
|
|
return self.get(creator_id)
|
|
else:
|
|
return None
|
|
|
|
def relationships(self, obj, relationship_type=None, source_only=False, target_only=False):
|
|
"""Retrieve Relationships involving the given STIX object.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
relationships will be looked up.
|
|
relationship_type (str): Only retrieve Relationships of this type.
|
|
If None, all relationships will be returned, regardless of type.
|
|
source_only (bool): Only retrieve Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only retrieve Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
|
|
Returns:
|
|
list: The Relationship objects involving the given STIX object.
|
|
|
|
"""
|
|
results = []
|
|
filters = [Filter('type', '=', 'relationship')]
|
|
|
|
try:
|
|
obj_id = obj['id']
|
|
except KeyError:
|
|
raise ValueError("STIX object has no 'id' property")
|
|
except TypeError:
|
|
# Assume `obj` is an ID string
|
|
obj_id = obj
|
|
|
|
if relationship_type:
|
|
filters.append(Filter('relationship_type', '=', relationship_type))
|
|
|
|
if source_only and target_only:
|
|
raise ValueError("Search either source only or target only, but not both")
|
|
|
|
if not target_only:
|
|
results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)]))
|
|
if not source_only:
|
|
results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)]))
|
|
|
|
return results
|
|
|
|
def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=None):
|
|
"""Retrieve STIX Objects that have a Relationship involving the given
|
|
STIX object.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
related objects will be looked up.
|
|
relationship_type (str): Only retrieve objects related by this
|
|
Relationships type. If None, all related objects will be
|
|
returned, regardless of type.
|
|
source_only (bool): Only examine Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only examine Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
filters (list): list of additional filters the related objects must
|
|
match.
|
|
|
|
Returns:
|
|
list: The STIX objects related to the given STIX object.
|
|
|
|
"""
|
|
results = []
|
|
rels = self.relationships(obj, relationship_type, source_only, target_only)
|
|
|
|
try:
|
|
obj_id = obj['id']
|
|
except TypeError:
|
|
# Assume `obj` is an ID string
|
|
obj_id = obj
|
|
|
|
# Get all unique ids from the relationships except that of the object
|
|
ids = set()
|
|
for r in rels:
|
|
ids.update((r.source_ref, r.target_ref))
|
|
ids.discard(obj_id)
|
|
|
|
# Assemble filters
|
|
filter_list = FilterSet(filters)
|
|
|
|
for i in ids:
|
|
results.extend(self.query([f for f in filter_list] + [Filter('id', '=', i)]))
|
|
|
|
return results
|
|
|
|
|
|
class CompositeDataSource(DataSource):
|
|
"""Controller for all the attached DataSources.
|
|
|
|
A user can have a single CompositeDataSource as an interface
|
|
the a set of DataSources. When an API call is made to the
|
|
CompositeDataSource, it is delegated to each of the (real)
|
|
DataSources that are attached to it.
|
|
|
|
DataSources can be attached to CompositeDataSource for a variety
|
|
of reasons, e.g. common filters, organization, less API calls.
|
|
|
|
Attributes:
|
|
|
|
data_sources (list): A dictionary of DataSource objects; to be
|
|
controlled and used by the Data Source Controller object.
|
|
|
|
"""
|
|
def __init__(self):
|
|
"""Create a new STIX Data Source.
|
|
|
|
Args:
|
|
|
|
"""
|
|
super(CompositeDataSource, self).__init__()
|
|
self.data_sources = []
|
|
|
|
def get(self, stix_id, _composite_filters=None):
|
|
"""Retrieve STIX object by STIX ID
|
|
|
|
Federated retrieve method, iterates through all DataSources
|
|
defined in the "data_sources" parameter. Each data source has a
|
|
specific API retrieve-like function and associated parameters. This
|
|
function does a federated retrieval and consolidation of the data
|
|
returned from all the STIX data sources.
|
|
|
|
A composite data source will pass its attached filters to
|
|
each configured data source, pushing filtering to them to handle.
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX object to retrieve.
|
|
_composite_filters (FilterSet): a collection of filters passed from a
|
|
CompositeDataSource (i.e. if this CompositeDataSource is attached
|
|
to another parent CompositeDataSource), not user supplied.
|
|
|
|
Returns:
|
|
stix_obj: The STIX object to be returned.
|
|
|
|
"""
|
|
if not self.has_data_sources():
|
|
raise AttributeError("CompositeDataSource has no data sources")
|
|
|
|
all_data = []
|
|
all_filters = FilterSet()
|
|
|
|
all_filters.add(self.filters)
|
|
|
|
if _composite_filters:
|
|
all_filters.add(_composite_filters)
|
|
|
|
# for every configured Data Source, call its retrieve handler
|
|
for ds in self.data_sources:
|
|
data = ds.get(stix_id=stix_id, _composite_filters=all_filters)
|
|
if data:
|
|
all_data.append(data)
|
|
|
|
# remove duplicate versions
|
|
if len(all_data) > 0:
|
|
all_data = deduplicate(all_data)
|
|
else:
|
|
return None
|
|
|
|
# reduce to most recent version
|
|
stix_obj = sorted(all_data, key=lambda k: k['modified'], reverse=True)[0]
|
|
|
|
return stix_obj
|
|
|
|
def all_versions(self, stix_id, _composite_filters=None):
|
|
"""Retrieve all versions of a STIX object by STIX ID.
|
|
|
|
Federated all_versions retrieve method - iterates through all
|
|
DataSources defined in "data_sources".
|
|
|
|
A composite data source will pass its attached filters to
|
|
each configured data source, pushing filtering to them to handle.
|
|
|
|
Args:
|
|
stix_id (str): id of the STIX objects to retrieve.
|
|
_composite_filters (FilterSet): a collection of filters passed from a
|
|
CompositeDataSource (i.e. if this CompositeDataSource is
|
|
attached to a parent CompositeDataSource), not user supplied.
|
|
|
|
Returns:
|
|
list: The STIX objects that have the specified id.
|
|
|
|
"""
|
|
if not self.has_data_sources():
|
|
raise AttributeError("CompositeDataSource has no data sources")
|
|
|
|
all_data = []
|
|
all_filters = FilterSet()
|
|
|
|
all_filters.add(self.filters)
|
|
|
|
if _composite_filters:
|
|
all_filters.add(_composite_filters)
|
|
|
|
# retrieve STIX objects from all configured data sources
|
|
for ds in self.data_sources:
|
|
data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters)
|
|
all_data.extend(data)
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0 objects
|
|
# with the same 'id' and 'modified' values)
|
|
if len(all_data) > 0:
|
|
all_data = deduplicate(all_data)
|
|
|
|
return all_data
|
|
|
|
def query(self, query=None, _composite_filters=None):
|
|
"""Retrieve STIX objects that match a query.
|
|
|
|
Federate the query to all DataSources attached to the
|
|
Composite Data Source.
|
|
|
|
Args:
|
|
query (list): list of filters to search on.
|
|
_composite_filters (FilterSet): a collection of filters passed from a
|
|
CompositeDataSource (i.e. if this CompositeDataSource is
|
|
attached to a parent CompositeDataSource), not user supplied.
|
|
|
|
Returns:
|
|
list: The STIX objects to be returned.
|
|
|
|
"""
|
|
if not self.has_data_sources():
|
|
raise AttributeError("CompositeDataSource has no data sources")
|
|
|
|
if not query:
|
|
# don't mess with the query (i.e. deduplicate, as that's done
|
|
# within the specific DataSources that are called)
|
|
query = []
|
|
|
|
all_data = []
|
|
all_filters = FilterSet()
|
|
|
|
all_filters.add(self.filters)
|
|
|
|
if _composite_filters:
|
|
all_filters.add(_composite_filters)
|
|
|
|
# federate query to all attached data sources,
|
|
# pass composite filters to id
|
|
for ds in self.data_sources:
|
|
data = ds.query(query=query, _composite_filters=all_filters)
|
|
all_data.extend(data)
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0
|
|
# objects with the same 'id' and 'modified' values)
|
|
if len(all_data) > 0:
|
|
all_data = deduplicate(all_data)
|
|
|
|
return all_data
|
|
|
|
def relationships(self, *args, **kwargs):
|
|
"""Retrieve Relationships involving the given STIX object.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Federated relationships retrieve method - iterates through all
|
|
DataSources defined in "data_sources".
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
relationships will be looked up.
|
|
relationship_type (str): Only retrieve Relationships of this type.
|
|
If None, all relationships will be returned, regardless of type.
|
|
source_only (bool): Only retrieve Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only retrieve Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
|
|
Returns:
|
|
list: The Relationship objects involving the given STIX object.
|
|
|
|
"""
|
|
if not self.has_data_sources():
|
|
raise AttributeError("CompositeDataSource has no data sources")
|
|
|
|
results = []
|
|
for ds in self.data_sources:
|
|
results.extend(ds.relationships(*args, **kwargs))
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0
|
|
# objects with the same 'id' and 'modified' values)
|
|
if len(results) > 0:
|
|
results = deduplicate(results)
|
|
|
|
return results
|
|
|
|
def related_to(self, *args, **kwargs):
|
|
"""Retrieve STIX Objects that have a Relationship involving the given
|
|
STIX object.
|
|
|
|
Only one of `source_only` and `target_only` may be `True`.
|
|
|
|
Federated related objects method - iterates through all
|
|
DataSources defined in "data_sources".
|
|
|
|
Args:
|
|
obj (STIX object OR dict OR str): The STIX object (or its ID) whose
|
|
related objects will be looked up.
|
|
relationship_type (str): Only retrieve objects related by this
|
|
Relationships type. If None, all related objects will be
|
|
returned, regardless of type.
|
|
source_only (bool): Only examine Relationships for which this
|
|
object is the source_ref. Default: False.
|
|
target_only (bool): Only examine Relationships for which this
|
|
object is the target_ref. Default: False.
|
|
filters (list): list of additional filters the related objects must
|
|
match.
|
|
|
|
Returns:
|
|
list: The STIX objects related to the given STIX object.
|
|
|
|
"""
|
|
if not self.has_data_sources():
|
|
raise AttributeError("CompositeDataSource has no data sources")
|
|
|
|
results = []
|
|
for ds in self.data_sources:
|
|
results.extend(ds.related_to(*args, **kwargs))
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0
|
|
# objects with the same 'id' and 'modified' values)
|
|
if len(results) > 0:
|
|
results = deduplicate(results)
|
|
|
|
return results
|
|
|
|
def add_data_source(self, data_source):
|
|
"""Attach a DataSource to CompositeDataSource instance
|
|
|
|
Args:
|
|
data_source (DataSource): a stix2.DataSource to attach
|
|
to the CompositeDataSource
|
|
|
|
"""
|
|
if issubclass(data_source.__class__, DataSource):
|
|
if data_source.id not in [ds_.id for ds_ in self.data_sources]:
|
|
# check DataSource not already attached CompositeDataSource
|
|
self.data_sources.append(data_source)
|
|
else:
|
|
raise TypeError("DataSource (to be added) is not of type stix2.DataSource. DataSource type is '%s'" % type(data_source))
|
|
|
|
return
|
|
|
|
def add_data_sources(self, data_sources):
|
|
"""Attach list of DataSources to CompositeDataSource instance
|
|
|
|
Args:
|
|
data_sources (list): stix2.DataSources to attach to
|
|
CompositeDataSource
|
|
"""
|
|
for ds in data_sources:
|
|
self.add_data_source(ds)
|
|
return
|
|
|
|
def remove_data_source(self, data_source_id):
|
|
"""Remove DataSource from the CompositeDataSource instance
|
|
|
|
Args:
|
|
data_source_id (str): DataSource IDs.
|
|
|
|
"""
|
|
def _match(ds_id, candidate_ds_id):
|
|
return ds_id == candidate_ds_id
|
|
|
|
self.data_sources[:] = [ds for ds in self.data_sources if not _match(ds.id, data_source_id)]
|
|
|
|
return
|
|
|
|
def remove_data_sources(self, data_source_ids):
|
|
"""Remove DataSources from the CompositeDataSource instance
|
|
|
|
Args:
|
|
data_source_ids (list): DataSource IDs
|
|
|
|
"""
|
|
for ds_id in data_source_ids:
|
|
self.remove_data_source(ds_id)
|
|
return
|
|
|
|
def has_data_sources(self):
|
|
return len(self.data_sources)
|
|
|
|
def get_all_data_sources(self):
|
|
return self.data_sources
|