329 lines
13 KiB
Python
329 lines
13 KiB
Python
"""
|
|
Python STIX 2.x TAXIICollectionStore
|
|
"""
|
|
|
|
from stix2.base import _STIXBase
|
|
from stix2.core import Bundle, parse
|
|
from stix2.sources import DataSink, DataSource, DataStore
|
|
from stix2.sources.filters import Filter, apply_common_filters
|
|
from stix2.utils import deduplicate
|
|
|
|
TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
|
|
|
|
|
|
class TAXIICollectionStore(DataStore):
|
|
"""Provides an interface to a local/remote TAXII Collection
|
|
of STIX data. TAXIICollectionStore is a wrapper
|
|
around a paired TAXIICollectionSink and TAXIICollectionSource.
|
|
|
|
Args:
|
|
collection (taxii2.Collection): TAXII Collection instance
|
|
"""
|
|
def __init__(self, collection):
|
|
super(TAXIICollectionStore, self).__init__()
|
|
self.source = TAXIICollectionSource(collection)
|
|
self.sink = TAXIICollectionSink(collection)
|
|
|
|
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Retrieve the most recent version of a single STIX object by ID.
|
|
|
|
Translate get() call to the appropriate DataSource call.
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX object to retrieve.
|
|
_composite_filters (set): set of filters passed from the parent
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
stix_obj: the single most recent version of the STIX
|
|
object specified by the "id".
|
|
|
|
"""
|
|
return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
|
|
|
|
def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Retrieve all versions of a single STIX object by ID.
|
|
|
|
Translate all_versions() to the appropriate DataSource call.
|
|
|
|
Args:
|
|
stix_id (str): the id of the STIX object to retrieve.
|
|
_composite_filters (set): set of filters passed from the parent
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
stix_objs (list): a list of STIX objects
|
|
|
|
"""
|
|
return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
|
|
|
|
def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Retrieve STIX objects matching a set of filters.
|
|
|
|
Translate query() to the appropriate DataSource call.
|
|
|
|
Args:
|
|
query (list): a list of filters (which collectively are the query)
|
|
to conduct search on.
|
|
_composite_filters (set): set of filters passed from the parent
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
stix_objs (list): a list of STIX objects
|
|
|
|
"""
|
|
return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)
|
|
|
|
def add(self, stix_objs, allow_custom=False, version=None):
|
|
"""Store STIX objects.
|
|
|
|
Translate add() to the appropriate DataSink call.
|
|
|
|
Args:
|
|
stix_objs (list): a list of STIX objects
|
|
allow_custom (bool): whether to allow custom objects/properties or
|
|
not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
"""
|
|
return self.sink.add(stix_objs, allow_custom=allow_custom, version=version)
|
|
|
|
|
|
class TAXIICollectionSink(DataSink):
|
|
"""Provides an interface for pushing STIX objects to a local/remote
|
|
TAXII Collection endpoint.
|
|
|
|
Args:
|
|
collection (taxii2.Collection): TAXII2 Collection instance
|
|
|
|
"""
|
|
def __init__(self, collection):
|
|
super(TAXIICollectionSink, self).__init__()
|
|
self.collection = collection
|
|
|
|
def add(self, stix_data, allow_custom=False, version=None):
|
|
"""Add/push STIX content to TAXII Collection endpoint
|
|
|
|
Args:
|
|
stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content
|
|
in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0
|
|
json encoded string, or list of any of the following
|
|
allow_custom (bool): whether to allow custom objects/properties or
|
|
not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
"""
|
|
if isinstance(stix_data, _STIXBase):
|
|
# adding python STIX object
|
|
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
|
|
|
|
elif isinstance(stix_data, dict):
|
|
# adding python dict (of either Bundle or STIX obj)
|
|
if stix_data["type"] == "bundle":
|
|
bundle = stix_data
|
|
else:
|
|
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
|
|
|
|
elif isinstance(stix_data, list):
|
|
# adding list of something - recurse on each
|
|
for obj in stix_data:
|
|
self.add(obj, allow_custom=allow_custom, version=version)
|
|
|
|
elif isinstance(stix_data, str):
|
|
# adding json encoded string of STIX content
|
|
stix_data = parse(stix_data, allow_custom=allow_custom, version=version)
|
|
if stix_data["type"] == "bundle":
|
|
bundle = dict(stix_data)
|
|
else:
|
|
bundle = dict(Bundle(stix_data, allow_custom=allow_custom))
|
|
|
|
else:
|
|
raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle")
|
|
|
|
self.collection.add_objects(bundle)
|
|
|
|
|
|
class TAXIICollectionSource(DataSource):
|
|
"""Provides an interface for searching/retrieving STIX objects
|
|
from a local/remote TAXII Collection endpoint.
|
|
|
|
Args:
|
|
collection (taxii2.Collection): TAXII Collection instance
|
|
|
|
"""
|
|
def __init__(self, collection):
|
|
super(TAXIICollectionSource, self).__init__()
|
|
self.collection = collection
|
|
|
|
def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Retrieve STIX object from local/remote STIX Collection
|
|
endpoint.
|
|
|
|
Args:
|
|
stix_id (str): The STIX ID of the STIX object to be retrieved.
|
|
_composite_filters (set): set of filters passed from the parent
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
(STIX object): STIX object that has the supplied STIX ID.
|
|
The STIX object is received from TAXII has dict, parsed into
|
|
a python STIX object and then returned
|
|
|
|
"""
|
|
# combine all query filters
|
|
query = set()
|
|
if self.filters:
|
|
query.update(self.filters)
|
|
if _composite_filters:
|
|
query.update(_composite_filters)
|
|
|
|
# dont extract TAXII filters from query (to send to TAXII endpoint)
|
|
# as directly retrieveing a STIX object by ID
|
|
stix_objs = self.collection.get_object(stix_id)["objects"]
|
|
|
|
stix_obj = list(apply_common_filters(stix_objs, query))
|
|
|
|
if len(stix_obj):
|
|
stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version)
|
|
if stix_obj.id != stix_id:
|
|
# check - was added to handle erroneous TAXII servers
|
|
stix_obj = None
|
|
else:
|
|
stix_obj = None
|
|
|
|
return stix_obj
|
|
|
|
def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Retrieve STIX object from local/remote TAXII Collection
|
|
endpoint, all versions of it
|
|
|
|
Args:
|
|
stix_id (str): The STIX ID of the STIX objects to be retrieved.
|
|
_composite_filters (set): set of filters passed from the parent
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
(see query() as all_versions() is just a wrapper)
|
|
|
|
"""
|
|
# make query in TAXII query format since 'id' is TAXII field
|
|
query = [
|
|
Filter("match[id]", "=", stix_id),
|
|
Filter("match[version]", "=", "all")
|
|
]
|
|
|
|
all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters)
|
|
|
|
# parse STIX objects from TAXII returned json
|
|
all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data]
|
|
|
|
# check - was added to handle erroneous TAXII servers
|
|
all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id]
|
|
|
|
return all_data_clean
|
|
|
|
def query(self, query=None, allow_custom=False, version=None, _composite_filters=None):
|
|
"""Search and retreive STIX objects based on the complete query
|
|
|
|
A "complete query" includes the filters from the query, the filters
|
|
attached to MemorySource, and any filters passed from a
|
|
CompositeDataSource (i.e. _composite_filters)
|
|
|
|
Args:
|
|
query (list): list of filters to search on
|
|
_composite_filters (set): set of filters passed from the
|
|
CompositeDataSource, not user supplied
|
|
allow_custom (bool): whether to retrieve custom objects/properties
|
|
or not. Default: False.
|
|
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
|
|
None, use latest version.
|
|
|
|
Returns:
|
|
(list): list of STIX objects that matches the supplied
|
|
query. The STIX objects are received from TAXII as dicts,
|
|
parsed into python STIX objects and then returned.
|
|
|
|
"""
|
|
if query is None:
|
|
query = set()
|
|
else:
|
|
if not isinstance(query, list):
|
|
# make sure dont make set from a Filter object,
|
|
# need to make a set from a list of Filter objects (even if just one Filter)
|
|
query = [query]
|
|
query = set(query)
|
|
|
|
# combine all query filters
|
|
if self.filters:
|
|
query.update(self.filters)
|
|
if _composite_filters:
|
|
query.update(_composite_filters)
|
|
|
|
# separate taxii query terms (can be done remotely)
|
|
taxii_filters = self._parse_taxii_filters(query)
|
|
|
|
# query TAXII collection
|
|
all_data = self.collection.get_objects(filters=taxii_filters)["objects"]
|
|
|
|
# deduplicate data (before filtering as reduces wasted filtering)
|
|
all_data = deduplicate(all_data)
|
|
|
|
# apply local (CompositeDataSource, TAXIICollectionSource and query filters)
|
|
all_data = list(apply_common_filters(all_data, query))
|
|
|
|
# parse python STIX objects from the STIX object dicts
|
|
stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data]
|
|
|
|
return stix_objs
|
|
|
|
def _parse_taxii_filters(self, query):
|
|
"""Parse out TAXII filters that the TAXII server can filter on.
|
|
|
|
Note:
|
|
For instance - "?match[type]=indicator,sighting" should be in a
|
|
query dict as follows:
|
|
|
|
Filter("type", "=", "indicator,sighting")
|
|
|
|
Args:
|
|
query (list): list of filters to extract which ones are TAXII
|
|
specific.
|
|
|
|
Returns:
|
|
params (dict): dict of the TAXII filters but in format required
|
|
for 'requests.get()'.
|
|
|
|
"""
|
|
params = {}
|
|
|
|
for filter_ in query:
|
|
if filter_.property in TAXII_FILTERS:
|
|
if filter_.property == "added_after":
|
|
params[filter_.property] = filter_.value
|
|
else:
|
|
taxii_field = "match[%s]" % filter_.property
|
|
params[taxii_field] = filter_.value
|
|
return params
|