707 lines
22 KiB
Python
707 lines
22 KiB
Python
"""
|
|
Python STIX 2.0 Composite Data Source and Data Source (classes)
|
|
|
|
|
|
---TODO/Questions---
|
|
|
|
-Test everything
|
|
|
|
-add_filter(), remove_filter(), deduplicate() - if these functions remain
|
|
the exact same for both CompositeDataSource and DataSource, they just
|
|
inherit/have module access to
|
|
|
|
"""
|
|
|
|
import abc
|
|
import copy
|
|
import uuid
|
|
|
|
from six import iteritems
|
|
|
|
|
|
def make_id():
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
# STIX 2.0 fields used to denote object version
|
|
STIX_VERSION_FIELDS = ['id', 'modified']
|
|
|
|
# Currently, only STIX 2.0 common SDO fields (that are not compex objects)
|
|
# are supported for filtering on
|
|
STIX_COMMON_FIELDS = [
|
|
'type',
|
|
'id',
|
|
'created_by_ref',
|
|
'created',
|
|
'modified',
|
|
'revoked',
|
|
'labels',
|
|
# 'external_references', # list of external references object type - not supported for filtering
|
|
'object_references',
|
|
'object_marking_refs',
|
|
'granular_marking_refs',
|
|
# 'granular_markings' # list of granular-marking type - not supported for filtering
|
|
]
|
|
|
|
|
|
# Required fields in filter(dict)
|
|
FILTER_FIELDS = ['field', 'op', 'value']
|
|
|
|
# Supported filter operations
|
|
FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
|
|
|
|
# Supported filter value types
|
|
FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
|
|
|
|
|
|
class CompositeDataSource(object):
|
|
"""Composite Data Source
|
|
|
|
Acts as a controller for all the defined/configured STIX Data Sources
|
|
e.g. a user can defined n Data Sources - creating Data Source (objects)
|
|
for each. There is only one instance of this for any python STIX 2.0
|
|
application
|
|
|
|
"""
|
|
|
|
def __init__(self, name="CompositeDataSource"):
|
|
"""
|
|
Creates a new STIX Data Source.
|
|
|
|
Args:
|
|
'data_sources' (dict): a dict of DataSource objects; to be
|
|
controlled and used by the Data Source Controller object
|
|
|
|
filters :
|
|
name :
|
|
"""
|
|
self.id_ = make_id()
|
|
self.name = name
|
|
self.data_sources = {}
|
|
self.filters = {}
|
|
self.filter_allowed = {}
|
|
|
|
def get(self, id_):
|
|
"""Retrieve STIX object by 'id'
|
|
|
|
federated retrieve method-iterates through all STIX data sources
|
|
defined in the "data_sources" parameter. Each data source has a
|
|
specific API retrieve-like function and associated parameters. This
|
|
function does a federated retrieval and consolidation of the data
|
|
returned from all the STIX data sources.
|
|
|
|
note: a composite data source will pass its attached filters to
|
|
each configured data source, pushing filtering to them to handle
|
|
|
|
Args:
|
|
id_ (str): the id of the STIX object to retrieve
|
|
|
|
Returns:
|
|
stix_obj (dict): the STIX object to be returned
|
|
|
|
"""
|
|
|
|
all_data = []
|
|
|
|
# for every configured Data Source, call its retrieve handler
|
|
for ds_id, ds in iteritems(self.data_sources):
|
|
data = ds.get(id_=id_, _composite_filters=self.filters.values())
|
|
all_data += data
|
|
|
|
# remove duplicate versions
|
|
if len(all_data) > 0:
|
|
all_data = self.deduplicate(all_data)
|
|
|
|
# reduce to most recent version
|
|
stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
|
|
|
|
return stix_obj
|
|
|
|
def all_versions(self, id_):
|
|
"""Retrieve STIX objects by 'id'
|
|
|
|
Federated all_versions retrieve method - iterates through all STIX data
|
|
sources defined in "data_sources"
|
|
|
|
note: a composite data source will pass its attached filters to
|
|
each configured data source, pushing filtering to them to handle
|
|
|
|
Args:
|
|
id_ (str): id of the STIX objects to retrieve
|
|
|
|
Returns:
|
|
all_data (list): list of STIX objects that have the specified id
|
|
"""
|
|
all_data = []
|
|
|
|
# retrieve STIX objects from all configured data sources
|
|
for ds_id, ds in iteritems(self.data_sources):
|
|
data = ds.all_versions(id_=id_, _composite_filters=self.filters.values())
|
|
all_data += data
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0 objects
|
|
# with the same 'id' and 'modified' values)
|
|
if len(all_data) > 0:
|
|
all_data = self.deduplicate(all_data)
|
|
|
|
return all_data
|
|
|
|
def query(self, query=None):
|
|
"""composite data source query
|
|
|
|
Federate the query to all Data Sources attached
|
|
to the Composite Data Source
|
|
|
|
Args:
|
|
query (list): list of filters to search on
|
|
|
|
Returns:
|
|
all_data (list): list of STIX objects to be returned
|
|
|
|
"""
|
|
if not query:
|
|
query = []
|
|
|
|
all_data = []
|
|
|
|
# federate query to all attached data sources,
|
|
# pass composite filters to them
|
|
for ds_id, ds in iteritems(self.data_sources):
|
|
data = ds.query(query=query, _composite_filters=self.filters.values())
|
|
all_data += data
|
|
|
|
# remove exact duplicates (where duplicates are STIX 2.0
|
|
# objects with the same 'id' and 'modified' values)
|
|
if len(all_data) > 0:
|
|
all_data = self.deduplicate(all_data)
|
|
|
|
return all_data
|
|
|
|
def add_data_source(self, data_sources):
|
|
"""add/attach Data Source to the Composite Data Source instance
|
|
|
|
Args:
|
|
data_sources (list): a list of Data Source objects to attach
|
|
to the Composite Data Source
|
|
|
|
Returns:
|
|
|
|
"""
|
|
|
|
for ds in data_sources:
|
|
if issubclass(ds, DataSource):
|
|
if self.data_sources[ds['id']] in self.data_sources.keys():
|
|
# data source already attached to Composite Data Source
|
|
continue
|
|
|
|
# add data source to Composite Data Source
|
|
# (its id will be its key identifier)
|
|
self.data_sources[ds['id']] = ds
|
|
else:
|
|
# the Data Source object is not a proper subclass
|
|
# of DataSource Abstract Class
|
|
# TODO: maybe log error?
|
|
continue
|
|
|
|
return
|
|
|
|
def remove_data_source(self, data_source_ids):
|
|
"""remove/detach Data Source from the Composite Data Source instance
|
|
|
|
Args:
|
|
data_source_ids (list): a list of Data Source
|
|
id's(which are strings)
|
|
|
|
Returns:
|
|
|
|
"""
|
|
|
|
for id_ in data_source_ids:
|
|
try:
|
|
if self.data_sources[id_]:
|
|
del self.data_sources[id_]
|
|
except KeyError:
|
|
# Data Source 'id' was not found in CompositeDataSource's
|
|
# list of data sources
|
|
pass
|
|
return
|
|
|
|
def get_data_sources(self):
|
|
"""return all attached Data Sources
|
|
|
|
TODO: Make this a property?
|
|
|
|
Args:
|
|
|
|
Returns:
|
|
|
|
"""
|
|
return copy.deepcopy(self.data_sources.values())
|
|
|
|
def add_filter(self, filters):
|
|
"""add/attach a filter to the Composite Data Source instance
|
|
|
|
Args:
|
|
filters (list): list of filters (dict) to add to the Data Source
|
|
|
|
Returns:
|
|
status (list): list of status/error messages
|
|
|
|
"""
|
|
|
|
status = []
|
|
errors = []
|
|
ids = []
|
|
allowed = True
|
|
|
|
for filter_ in filters:
|
|
# check required filter components ("field", "op", "value") exist
|
|
for field in FILTER_FIELDS:
|
|
if field not in filter_.keys():
|
|
allowed = False
|
|
errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.")
|
|
break
|
|
|
|
if allowed:
|
|
# no need for further checks if filter is missing parameters
|
|
|
|
# check filter field is a supported STIX 2.0 common field
|
|
if filter_['field'] not in STIX_COMMON_FIELDS:
|
|
allowed = False
|
|
errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported")
|
|
|
|
# check filter operator is supported
|
|
if filter_['op'] not in FILTER_OPS:
|
|
allowed = False
|
|
errors.append("Filter operation(from 'op' field) not supported")
|
|
|
|
# check filter value type is supported
|
|
if type(filter_['value']) not in FILTER_VALUE_TYPES:
|
|
allowed = False
|
|
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
|
|
|
|
# Filter is added regardless of whether it fits requirements
|
|
# to be a common filter. This is done because some filters
|
|
# may be added and used by third party Data Sources, where the
|
|
# filtering may be conducted within those plugins, just not here
|
|
|
|
id_ = make_id()
|
|
filter_['id'] = id_
|
|
self.filters['id_'] = filter_
|
|
ids.append(id_)
|
|
|
|
if allowed:
|
|
self.filter_allowed[id_] = True
|
|
status.append({
|
|
"status": "added as a common filter",
|
|
"filter": filter_,
|
|
"data_source_name": self.name,
|
|
"data_source_id": self.id_
|
|
})
|
|
else:
|
|
self.filter_allowed[id_] = False
|
|
status.append({
|
|
"status": "added but is not a common filter",
|
|
"filter": filter_,
|
|
"errors": errors,
|
|
"data_source_name": self.name,
|
|
"data_source_id": self.id_
|
|
})
|
|
del errors[:]
|
|
|
|
allowed = True
|
|
|
|
return ids, status
|
|
|
|
def remove_filter(self, filter_ids):
|
|
"""Remove/detach a filter from the Data Source instance
|
|
|
|
Args:
|
|
filter_ids (list): list of filter id's (which are strings)
|
|
dettach from the Composite Data Source
|
|
|
|
Returns:
|
|
|
|
"""
|
|
|
|
for filter_id in filter_ids:
|
|
try:
|
|
if filter_id in self.filters:
|
|
del self.filters[filter_id]
|
|
del self.filter_allowed[filter_id]
|
|
except KeyError:
|
|
# filter id not found in list of filters
|
|
# attached to the Composite Data Source
|
|
pass
|
|
|
|
return
|
|
|
|
def get_filters(self):
|
|
"""return filters attached to Composite Data Source
|
|
|
|
Args:
|
|
|
|
Returns:
|
|
(list): the list of filters currently attached to the Data Source
|
|
|
|
"""
|
|
return copy.deepcopy(list(self.filters.values()))
|
|
|
|
def deduplicate(self, stix_obj_list):
|
|
"""deduplicate a list fo STIX objects to a unique set
|
|
|
|
Reduces a set of STIX objects to unique set by looking
|
|
at 'id' and 'modified' fields - as a unique object version
|
|
is determined by the combination of those fields
|
|
|
|
Args:
|
|
stix_obj_list (list): list of STIX objects (dicts)
|
|
|
|
Returns:
|
|
(list): unique set of the passed list of STIX objects
|
|
"""
|
|
|
|
unique = []
|
|
dont_have = False
|
|
for i in stix_obj_list:
|
|
dont_have = False
|
|
for j in unique:
|
|
for field in STIX_VERSION_FIELDS:
|
|
if not i[field] == j[field]:
|
|
dont_have = True
|
|
break
|
|
if dont_have:
|
|
unique.append(i)
|
|
return unique
|
|
|
|
|
|
class DataSource(object):
|
|
"""
|
|
Abstract Data Source class for STIX 2.0
|
|
|
|
An implementer will create a concrete subclass from
|
|
this abstract class for the specific data source.
|
|
|
|
The purpose of the concrete subclasses is to then
|
|
supply them to a Composite Data Source which calls
|
|
the subclass methods when conducting STIX 2.0
|
|
data retrievals.
|
|
"""
|
|
|
|
__metaclass__ = abc.ABCMeta
|
|
|
|
def __init__(self, name="DataSource"):
|
|
self.name = name
|
|
self.id_ = make_id()
|
|
self.filters = {}
|
|
self.filter_allowed = {}
|
|
|
|
@abc.abstractmethod
|
|
def get(self, id_, _composite_filters=None):
|
|
"""
|
|
Fill:
|
|
-implement the specific data source API calls, processing,
|
|
functionality required for retrieving data from the data source
|
|
|
|
Args:
|
|
|
|
id_ (str): the id of the STIX 2.0 object to retrieve. Should
|
|
return a single object, the most recent version of the object
|
|
specified by the "id".
|
|
|
|
_composite_filters (list): list of filters passed along from
|
|
the Composite Data Filter.
|
|
|
|
Returns:
|
|
stix_obj (dictionary): the STIX object to be returned
|
|
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
@abc.abstractmethod
|
|
def all_versions(self, id_, _composite_filters=None):
|
|
"""
|
|
Fill:
|
|
-Similar to get() except returns list of all object versions of
|
|
the specified "id".
|
|
|
|
-implement the specific data source API calls, processing,
|
|
functionality required for retrieving data from the data source
|
|
|
|
Args:
|
|
id_ (str): The id of the STIX 2.0 object to retrieve. Should
|
|
return a list of objects, all the versions of the object
|
|
specified by the "id".
|
|
|
|
_composite_filters (list): list of filters passed from the
|
|
Composite Data Source
|
|
|
|
Returns:
|
|
stix_objs (list): a list of STIX objects (where each object is a
|
|
STIX object)
|
|
"""
|
|
stix_objs = []
|
|
|
|
return stix_objs
|
|
|
|
@abc.abstractmethod
|
|
def query(self, query, _composite_filters=None):
|
|
"""
|
|
Fill:
|
|
-implement the specific data source API calls, processing,
|
|
functionality required for retrieving query from the data source
|
|
|
|
Args:
|
|
query (list): a list of filters (which collectively are the query)
|
|
to conduct search on
|
|
|
|
_composite_filters (list): a list of filters passed from the
|
|
Composite Data Source
|
|
|
|
Returns:
|
|
|
|
|
|
"""
|
|
stix_objs = []
|
|
|
|
return stix_objs
|
|
|
|
@abc.abstractmethod
|
|
def close(self):
|
|
"""
|
|
Fill:
|
|
Close, release, shutdown any objects, contexts, variables
|
|
Args:
|
|
|
|
Returns:
|
|
(list): list of status/error messages
|
|
"""
|
|
|
|
status = []
|
|
|
|
return status
|
|
|
|
def add_filter(self, filters):
|
|
"""add/attach a filter to the Data Source instance
|
|
|
|
Args:
|
|
filters (list): list of filters (dict) to add to the Data Source
|
|
|
|
Returns:
|
|
status (list): list of status/error messages
|
|
|
|
"""
|
|
|
|
status = []
|
|
errors = []
|
|
ids = []
|
|
allowed = True
|
|
|
|
for filter_ in filters:
|
|
# check required filter components ("field", "op", "value") exist
|
|
for field in FILTER_FIELDS:
|
|
if field not in filter_.keys():
|
|
allowed = False
|
|
errors.append("Filter was missing a required field(key). Each filter requires 'field', 'op', 'value' keys.")
|
|
break
|
|
|
|
if allowed:
|
|
# no reason for further checks if missing filter parameters
|
|
|
|
# check filter field is a supported STIX 2.0 common field
|
|
if filter_['field'] not in STIX_COMMON_FIELDS:
|
|
allowed = False
|
|
errors.append("Filter 'field' is not a STIX 2.0 common property. Currently only STIX object common properties supported")
|
|
|
|
# check filter operator is supported
|
|
if filter_['op'] not in FILTER_OPS:
|
|
allowed = False
|
|
errors.append("Filter operation(from 'op' field) not supported")
|
|
|
|
# check filter value type is supported
|
|
if type(filter_['value']) not in FILTER_VALUE_TYPES:
|
|
allowed = False
|
|
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
|
|
|
|
# Filter is added regardless of whether it fits requirements
|
|
# to be a common filter. This is done because some filters
|
|
# may be added and used by third party Data Sources, where the
|
|
# filtering may be conducted within those plugins, just not here
|
|
|
|
id_ = make_id()
|
|
filter_['id'] = id_
|
|
self.filters[id_] = filter_
|
|
ids.append(id_)
|
|
|
|
if allowed:
|
|
self.filter_allowed[id_] = True
|
|
status.append({
|
|
"status": "added as a common filter",
|
|
"filter": filter_,
|
|
"data_source_name": self.name,
|
|
"data_source_id": self.id_
|
|
})
|
|
else:
|
|
self.filter_allowed[id_] = False
|
|
status.append({
|
|
"status": "added but is not a common filter",
|
|
"filter": filter_,
|
|
"errors": copy.deepcopy(errors),
|
|
"data_source_name": self.name,
|
|
"data_source_id": self.id_
|
|
})
|
|
del errors[:]
|
|
|
|
allowed = True
|
|
|
|
return ids, status
|
|
|
|
def remove_filter(self, filter_ids):
|
|
"""remove/detach a filter from the Data Source instance
|
|
|
|
Args:
|
|
filter_ids (list): list of filter ids to dettach/remove
|
|
from Data Source
|
|
|
|
Returns:
|
|
|
|
|
|
"""
|
|
for filter_id in filter_ids:
|
|
try:
|
|
if filter_id in self.filters:
|
|
del self.filters[filter_id]
|
|
del self.filter_allowed[filter_id]
|
|
except KeyError:
|
|
# filter 'id' not found list of filters attached to Data Source
|
|
pass
|
|
|
|
return
|
|
|
|
def get_filters(self):
|
|
"""return copy of all filters currently attached to Data Source
|
|
|
|
TODO: make this a property?
|
|
|
|
Returns:
|
|
(list): a copy of all the filters(dict) which are attached
|
|
to Data Source
|
|
|
|
"""
|
|
return copy.deepcopy(list(self.filters.values()))
|
|
|
|
def apply_common_filters(self, stix_objs, query):
|
|
"""evaluates filters against a set of STIX 2.0 objects
|
|
|
|
Supports only STIX 2.0 common property fields
|
|
|
|
Args:
|
|
stix_objs (list): list of STIX objects to apply the query to
|
|
query (list): list of filters (combined form complete query)
|
|
|
|
Returns:
|
|
(list): list of STIX objects that successfully evaluate against
|
|
the query
|
|
|
|
"""
|
|
|
|
filtered_stix_objs = []
|
|
|
|
# evaluate objects against filter
|
|
for stix_obj in stix_objs:
|
|
clean = True
|
|
for filter_ in query:
|
|
|
|
# skip filter as filter was identified (when added) as
|
|
# not a common filter
|
|
if 'id' in filter_ and self.filter_allowed[filter_['id']] is False:
|
|
continue
|
|
|
|
# check filter "field" is in STIX object - if cant be applied
|
|
# due to STIX object, STIX object is discarded (i.e. did not
|
|
# make it through the filter)
|
|
if filter_['field'] not in stix_obj.keys():
|
|
break
|
|
|
|
try:
|
|
|
|
if filter_['op'] == '=':
|
|
if not stix_obj[filter_['field']] == filter_['value']:
|
|
clean = False
|
|
break
|
|
elif filter_['op'] == "!=":
|
|
if not stix_obj[filter_['field']] != filter_['value']:
|
|
clean = False
|
|
break
|
|
elif filter_['op'] == "in":
|
|
if not stix_obj[filter_['field']] in filter_['value']:
|
|
clean = False
|
|
break
|
|
else:
|
|
# filter operation not supported
|
|
continue
|
|
|
|
# TODO: I think the rest of the operations only
|
|
# apply to timestamps, in which case I don't think
|
|
# simple operator usage (like below) works
|
|
|
|
# elif filter_['op'] == ">":
|
|
# if not stix_obj[filter_['field']] > filter_['value']:
|
|
# clean = False
|
|
# break
|
|
#
|
|
# elif filter_['op'] == "<":
|
|
# if not stix_obj[filter_['field']] < filter_['value']:
|
|
# clean = False
|
|
# break
|
|
#
|
|
# elif filter_['op'] == ">=":
|
|
# if not stix_obj[filter_['field']] >= filter_['value']:
|
|
# clean = False
|
|
# break
|
|
#
|
|
# elif filter_['op'] == "<=":
|
|
# if not stix_obj[filter_['field']] <= filter_['value']:
|
|
# clean = False
|
|
# break
|
|
|
|
except TypeError:
|
|
# type mismatch of comparison operands - ignore filter,
|
|
# no error raised for now
|
|
pass
|
|
|
|
# if object unmarked after all filter, add it
|
|
if clean:
|
|
filtered_stix_objs.append(stix_obj)
|
|
|
|
clean = True
|
|
|
|
return filtered_stix_objs
|
|
|
|
def deduplicate(self, stix_obj_list):
|
|
"""deduplicate a list of STIX objects into a unique set
|
|
|
|
reduces a set of STIX objects to unique set by looking
|
|
at 'id' and 'modified' fields - as a unique object version
|
|
is determined by the combination of those fields
|
|
|
|
Args:
|
|
stix_obj_list (list): list of STIX objects (dicts)
|
|
|
|
Returns:
|
|
(list): a unique set of the passed STIX object list
|
|
|
|
|
|
"""
|
|
unique = []
|
|
have = False
|
|
for i in stix_obj_list:
|
|
for j in unique:
|
|
if i['id'] == j['id'] and i['modified'] == j['modified']:
|
|
have = True
|
|
break
|
|
if not have:
|
|
unique.append(i)
|
|
have = False
|
|
return unique
|