MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Code style changes.

stix2.1
Emmanuelle Vargas-Gonzalez 2017-05-26 15:24:33 -04:00
parent 97d8d732fc
commit 2a8af45ec2
2 changed files with 208 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
stix2/sources/__init__.py
View File

@ -1,4 +1,4 @@
''' """
Python STIX 2.0 Composite Data Source and Data Source (classes) Python STIX 2.0 Composite Data Source and Data Source (classes)
@ -6,24 +6,28 @@ Python STIX 2.0 Composite Data Source and Data Source (classes)
-Test everything -Test everything
-add_filter(), remove_filter(), deduplicate() - if these functions remain the exact same for -add_filter(), remove_filter(), deduplicate() - if these functions remain
both CompositeDataSource and DataSource, they just inherit/have module access to the exact same for both CompositeDataSource and DataSource, they just
inherit/have module access to
''' """
import abc import abc
import copy import copy
import uuid import uuid
from six import iteritems
def make_id(): def make_id():
str(uuid.uuid4()) return str(uuid.uuid4())
# STIX 2.0 fields used to denote object version # STIX 2.0 fields used to denote object version
STIX_VERSION_FIELDS = ['id', 'modified'] STIX_VERSION_FIELDS = ['id', 'modified']
# currently, only STIX 2.0 common SDO fields (that are not compex objects) are supported for filtering on # Currently, only STIX 2.0 common SDO fields (that are not compex objects)
# are supported for filtering on
STIX_COMMON_FIELDS = [ STIX_COMMON_FIELDS = [
'type', 'type',
'id', 'id',
@ -32,11 +36,11 @@ STIX_COMMON_FIELDS = [
'modified', 'modified',
'revoked', 'revoked',
'labels', 'labels',
# 'external_references', #list of external references object type - not supported for filtering # 'external_references', # list of external references object type - not supported for filtering
'object_references', 'object_references',
'object_marking_refs', 'object_marking_refs',
'granular_marking_refs', 'granular_marking_refs',
# 'granular_markings' #list of granular-marking type - not supported for filtering # 'granular_markings' # list of granular-marking type - not supported for filtering
] ]
@ -51,28 +55,26 @@ FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
class CompositeDataSource(object): class CompositeDataSource(object):
'''Composite Data Source """Composite Data Source
Acts as a controller for all the defined/configured STIX Data Sources Acts as a controller for all the defined/configured STIX Data Sources
e.g. a user can defined n Data Sources - creating Data Source (objects) e.g. a user can defined n Data Sources - creating Data Source (objects)
for each. There is only one instance of this for any python STIX 2.0 application for each. There is only one instance of this for any python STIX 2.0
application
''' """
def __init__(self, name="CompositeDataSource"): def __init__(self, name="CompositeDataSource"):
''' """
Creates a new STIX Data Source. Creates a new STIX Data Source.
Args: Args:
'data_sources' (dict): a dict of DataSource objects; to be controlled and used by 'data_sources' (dict): a dict of DataSource objects; to be
the Data Source Controller object controlled and used by the Data Source Controller object
filters : filters :
name : name :
"""
Returns:
'''
self.id_ = make_id() self.id_ = make_id()
self.name = name self.name = name
self.data_sources = {} self.data_sources = {}
@ -80,7 +82,7 @@ class CompositeDataSource(object):
self.filter_allowed = {} self.filter_allowed = {}
def get(self, id_): def get(self, id_):
'''retrieve STIX object by 'id' """Retrieve STIX object by 'id'
federated retrieve method-iterates through all STIX data sources federated retrieve method-iterates through all STIX data sources
defined in the "data_sources" parameter. Each data source has a defined in the "data_sources" parameter. Each data source has a
@ -97,12 +99,12 @@ class CompositeDataSource(object):
Returns: Returns:
stix_obj (dict): the STIX object to be returned stix_obj (dict): the STIX object to be returned
''' """
all_data = [] all_data = []
# for every configured Data Source, call its retrieve handler # for every configured Data Source, call its retrieve handler
for ds_id, ds in self.data_sources.iteritems(): for ds_id, ds in iteritems(self.data_sources):
data = ds.get(id_=id_, _composite_filters=self.filters.values()) data = ds.get(id_=id_, _composite_filters=self.filters.values())
all_data += data all_data += data
@ -116,7 +118,7 @@ class CompositeDataSource(object):
return stix_obj return stix_obj
def all_versions(self, id_): def all_versions(self, id_):
'''retrieve STIX objects by 'id' """Retrieve STIX objects by 'id'
Federated all_versions retrieve method - iterates through all STIX data Federated all_versions retrieve method - iterates through all STIX data
sources defined in "data_sources" sources defined in "data_sources"
@ -129,22 +131,23 @@ class CompositeDataSource(object):
Returns: Returns:
all_data (list): list of STIX objects that have the specified id all_data (list): list of STIX objects that have the specified id
''' """
all_data = [] all_data = []
# retrieve STIX objects from all configured data sources # retrieve STIX objects from all configured data sources
for ds_id, ds in self.data_sources.iteritems(): for ds_id, ds in iteritems(self.data_sources):
data = ds.all_versions(id_=id_, _composite_filters=self.filters.values()) data = ds.all_versions(id_=id_, _composite_filters=self.filters.values())
all_data += data all_data += data
# remove exact duplicates (where duplicates are STIX 2.0 objects with the same 'id' and 'modified' values) # remove exact duplicates (where duplicates are STIX 2.0 objects
# with the same 'id' and 'modified' values)
if len(all_data) > 0: if len(all_data) > 0:
all_data = self.deduplicate(all_data) all_data = self.deduplicate(all_data)
return all_data return all_data
def query(self, query=None): def query(self, query=None):
'''composite data source query """composite data source query
Federate the query to all Data Sources attached Federate the query to all Data Sources attached
to the Composite Data Source to the Composite Data Source
@ -155,32 +158,35 @@ class CompositeDataSource(object):
Returns: Returns:
all_data (list): list of STIX objects to be returned all_data (list): list of STIX objects to be returned
''' """
if not query: if not query:
query = [] query = []
all_data = [] all_data = []
# federate query to all attached data sources, pass composite filters to them # federate query to all attached data sources,
for ds_id, ds in self.data_sources.iteritems(): # pass composite filters to them
for ds_id, ds in iteritems(self.data_sources):
data = ds.query(query=query, _composite_filters=self.filters.values()) data = ds.query(query=query, _composite_filters=self.filters.values())
all_data += data all_data += data
# remove exact duplicates (where duplicates are STIX 2.0 objects with the same 'id' and 'modified' values) # remove exact duplicates (where duplicates are STIX 2.0
# objects with the same 'id' and 'modified' values)
if len(all_data) > 0: if len(all_data) > 0:
all_data = self.deduplicate(all_data) all_data = self.deduplicate(all_data)
return all_data return all_data
def add_data_source(self, data_sources): def add_data_source(self, data_sources):
'''add/attach Data Source to the Composite Data Source instance """add/attach Data Source to the Composite Data Source instance
Args: Args:
data_sources (list): a list of Data Source objects to attach to the Composite Data Source data_sources (list): a list of Data Source objects to attach
to the Composite Data Source
Returns: Returns:
''' """
for ds in data_sources: for ds in data_sources:
if issubclass(ds, DataSource): if issubclass(ds, DataSource):
@ -188,37 +194,40 @@ class CompositeDataSource(object):
# data source already attached to Composite Data Source # data source already attached to Composite Data Source
continue continue
# add data source to Composite Data Source (its id will be its key identifier) # add data source to Composite Data Source
# (its id will be its key identifier)
self.data_sources[ds['id']] = ds self.data_sources[ds['id']] = ds
else: else:
# the Data Source object is not a proper subclass of DataSource Abstract Class # the Data Source object is not a proper subclass
# of DataSource Abstract Class
# TODO: maybe log error? # TODO: maybe log error?
continue continue
return return
def remove_data_source(self, data_source_ids): def remove_data_source(self, data_source_ids):
'''remove/detach Data Source from the Composite Data Source instance """remove/detach Data Source from the Composite Data Source instance
Args: Args:
data_source_ids (list): a list of Data Source id's( which are strings ) data_source_ids (list): a list of Data Source
id's(which are strings)
Returns: Returns:
"""
'''
for id_ in data_source_ids: for id_ in data_source_ids:
try: try:
if self.data_sources[id_]: if self.data_sources[id_]:
del self.data_sources[id_] del self.data_sources[id_]
except KeyError: except KeyError:
# Data Source 'id' was not found in CompositeDataSource's list of data sources # Data Source 'id' was not found in CompositeDataSource's
# list of data sources
pass pass
return return
def get_data_sources(self): def get_data_sources(self):
'''return all attached Data Sources """return all attached Data Sources
TODO: Make this a property? TODO: Make this a property?
@ -226,11 +235,11 @@ class CompositeDataSource(object):
Returns: Returns:
''' """
return copy.deepcopy(self.data_sources.values()) return copy.deepcopy(self.data_sources.values())
def add_filter(self, filters): def add_filter(self, filters):
'''add/attach a filter to the Composite Data Source instance """add/attach a filter to the Composite Data Source instance
Args: Args:
filters (list): list of filters (dict) to add to the Data Source filters (list): list of filters (dict) to add to the Data Source
@ -238,7 +247,7 @@ class CompositeDataSource(object):
Returns: Returns:
status (list): list of status/error messages status (list): list of status/error messages
''' """
status = [] status = []
errors = [] errors = []
@ -268,12 +277,11 @@ class CompositeDataSource(object):
allowed = False allowed = False
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
''' # Filter is added regardless of whether it fits requirements
Filter is added regardless of whether it fits requirements # to be a common filter. This is done because some filters
to be a common filter. This is done because some filters # may be added and used by third party Data Sources, where the
may be added and used by third party Data Sources, where # filtering may be conducted within those plugins, just not here
the filtering may be conducted within those plugins, just not here
'''
id_ = make_id() id_ = make_id()
filter_['id'] = id_ filter_['id'] = id_
self.filters['id_'] = filter_ self.filters['id_'] = filter_
@ -302,7 +310,7 @@ class CompositeDataSource(object):
return ids, status return ids, status
def remove_filter(self, filter_ids): def remove_filter(self, filter_ids):
'''remove/detach a filter from the Data Source instance """Remove/detach a filter from the Data Source instance
Args: Args:
filter_ids (list): list of filter id's (which are strings) filter_ids (list): list of filter id's (which are strings)
@ -310,7 +318,7 @@ class CompositeDataSource(object):
Returns: Returns:
''' """
for filter_id in filter_ids: for filter_id in filter_ids:
try: try:
@ -318,35 +326,36 @@ class CompositeDataSource(object):
del self.filters[filter_id] del self.filters[filter_id]
del self.filter_allowed[filter_id] del self.filter_allowed[filter_id]
except KeyError: except KeyError:
# filter id not found in list of filters attached to the Composite Data Source # filter id not found in list of filters
# attached to the Composite Data Source
pass pass
return return
def get_filters(self): def get_filters(self):
'''return filters attached to Composite Data Source """return filters attached to Composite Data Source
Args: Args:
Returns: Returns:
(list): the list of filters currently attached to the Data Source (list): the list of filters currently attached to the Data Source
''' """
return copy.deepcopy(list(self.filters.values())) return copy.deepcopy(list(self.filters.values()))
def deduplicate(self, stix_obj_list): def deduplicate(self, stix_obj_list):
'''deduplicate a list fo STIX objects to a unique set """deduplicate a list fo STIX objects to a unique set
Reduces a set of STIX objects to unique set by looking Reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version is determined at 'id' and 'modified' fields - as a unique object version
by the combination of those fields is determined by the combination of those fields
Args: Args:
stix_obj_list (list): list of STIX objects (dicts) stix_obj_list (list): list of STIX objects (dicts)
Returns: Returns:
(list): unique set of the passed list of STIX objects (list): unique set of the passed list of STIX objects
''' """
unique = [] unique = []
dont_have = False dont_have = False
@ -363,7 +372,7 @@ class CompositeDataSource(object):
class DataSource(object): class DataSource(object):
''' """
Abstract Data Source class for STIX 2.0 Abstract Data Source class for STIX 2.0
An implementer will create a concrete subclass from An implementer will create a concrete subclass from
@ -373,7 +382,7 @@ class DataSource(object):
supply them to a Composite Data Source which calls supply them to a Composite Data Source which calls
the subclass methods when conducting STIX 2.0 the subclass methods when conducting STIX 2.0
data retrievals. data retrievals.
''' """
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
@ -385,87 +394,93 @@ class DataSource(object):
@abc.abstractmethod @abc.abstractmethod
def get(self, id_, _composite_filters=None): def get(self, id_, _composite_filters=None):
''' """
Fill: Fill:
-implement the specific data source API calls, processing, functionality -implement the specific data source API calls, processing,
requried for retrieving data from the data source functionality required for retrieving data from the data source
Args: Args:
id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, id_ (str): the id of the STIX 2.0 object to retrieve. Should
the most recent version of the object specified by the "id". return a single object, the most recent version of the object
specified by the "id".
_composite_filters (list): list of filters passed along from the Composite Data Filter _composite_filters (list): list of filters passed along from
the Composite Data Filter.
Returns: Returns:
stix_obj (dictionary): the STIX object to be returned stix_obj (dictionary): the STIX object to be returned
''' """
stix_obj = None stix_obj = None
return stix_obj return stix_obj
@abc.abstractmethod @abc.abstractmethod
def all_versions(self, id_, _composite_filters=None): def all_versions(self, id_, _composite_filters=None):
''' """
Fill: Fill:
-Similar to get() except returns list of all object versions of the specified "id". -Similar to get() except returns list of all object versions of
the specified "id".
-implement the specific data source API calls, processing, functionality
requried for retrieving data from the data source
-implement the specific data source API calls, processing,
functionality required for retrieving data from the data source
Args: Args:
id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, id_ (str): The id of the STIX 2.0 object to retrieve. Should
all the versions of the object specified by the "id". return a list of objects, all the versions of the object
specified by the "id".
_composite_filters (list): list of filters passed from the Composite Data Source _composite_filters (list): list of filters passed from the
Composite Data Source
Returns: Returns:
stix_objs (list): a list of STIX objects(where each object is a STIX object) stix_objs (list): a list of STIX objects (where each object is a
STIX object)
''' """
stix_objs = [] stix_objs = []
return stix_objs return stix_objs
@abc.abstractmethod @abc.abstractmethod
def query(self, query, _composite_filters=None): def query(self, query, _composite_filters=None):
''' """
Fill: Fill:
-implement the specific data source API calls, processing, functionality -implement the specific data source API calls, processing,
requried for retrieving query from the data source functionality required for retrieving query from the data source
Args: Args:
query (list): a list of filters (which collectively are the query) to conduct search on query (list): a list of filters (which collectively are the query)
to conduct search on
_composite_filters (list): a list of filters passed from the Composite Data Source _composite_filters (list): a list of filters passed from the
Composite Data Source
Returns: Returns:
''' """
stix_objs = [] stix_objs = []
return stix_objs return stix_objs
@abc.abstractmethod @abc.abstractmethod
def close(self): def close(self):
''' """
Fill: Fill:
Close, release, shutdown any objects, contexts, variables Close, release, shutdown any objects, contexts, variables
Args: Args:
Returns: Returns:
(list): list of status/error messages (list): list of status/error messages
''' """
status = [] status = []
return status return status
def add_filter(self, filters): def add_filter(self, filters):
'''add/attach a filter to the Data Source instance """add/attach a filter to the Data Source instance
Args: Args:
filters (list): list of filters (dict) to add to the Data Source filters (list): list of filters (dict) to add to the Data Source
@ -473,7 +488,7 @@ class DataSource(object):
Returns: Returns:
status (list): list of status/error messages status (list): list of status/error messages
''' """
status = [] status = []
errors = [] errors = []
@ -503,12 +518,11 @@ class DataSource(object):
allowed = False allowed = False
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary") errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
''' # Filter is added regardless of whether it fits requirements
Filter is added regardless of whether it fits requirements # to be a common filter. This is done because some filters
to be a common filter. This is done because some filters # may be added and used by third party Data Sources, where the
may be added and used by third party Data Sources, where # filtering may be conducted within those plugins, just not here
the filtering may be conducted within those plugins, just not here
'''
id_ = make_id() id_ = make_id()
filter_['id'] = id_ filter_['id'] = id_
self.filters[id_] = filter_ self.filters[id_] = filter_
@ -538,15 +552,16 @@ class DataSource(object):
return ids, status return ids, status
def remove_filter(self, filter_ids): def remove_filter(self, filter_ids):
'''remove/detach a filter from the Data Source instance """remove/detach a filter from the Data Source instance
Args: Args:
filter_ids (list): list of filter ids to dettach/remove from Data Source filter_ids (list): list of filter ids to dettach/remove
from Data Source
Returns: Returns:
''' """
for filter_id in filter_ids: for filter_id in filter_ids:
try: try:
if filter_id in self.filters: if filter_id in self.filters:
@ -559,18 +574,19 @@ class DataSource(object):
return return
def get_filters(self): def get_filters(self):
'''return copy of all filters currently attached to Data Source """return copy of all filters currently attached to Data Source
TODO: make this a property? TODO: make this a property?
Returns: Returns:
(list): a copy of all the filters(dict) which are attached to Data Source (list): a copy of all the filters(dict) which are attached
to Data Source
''' """
return copy.deepcopy(list(self.filters.values())) return copy.deepcopy(list(self.filters.values()))
def apply_common_filters(self, stix_objs, query): def apply_common_filters(self, stix_objs, query):
'''evaluates filters against a set of STIX 2.0 objects """evaluates filters against a set of STIX 2.0 objects
Supports only STIX 2.0 common property fields Supports only STIX 2.0 common property fields
@ -579,9 +595,10 @@ class DataSource(object):
query (list): list of filters (combined form complete query) query (list): list of filters (combined form complete query)
Returns: Returns:
(list): list of STIX objects that successfully evaluate against the query (list): list of STIX objects that successfully evaluate against
the query
''' """
filtered_stix_objs = [] filtered_stix_objs = []
@ -590,12 +607,14 @@ class DataSource(object):
clean = True clean = True
for filter_ in query: for filter_ in query:
# skip filter as filter was identified (when added) as not a common filter # skip filter as filter was identified (when added) as
# not a common filter
if 'id' in filter_ and self.filter_allowed[filter_['id']] is False: if 'id' in filter_ and self.filter_allowed[filter_['id']] is False:
continue continue
# check filter "field" is in STIX object - if cant be applied due to STIX object, # check filter "field" is in STIX object - if cant be applied
# STIX object is discarded (i.e. did not make it through the filter) # due to STIX object, STIX object is discarded (i.e. did not
# make it through the filter)
if filter_['field'] not in stix_obj.keys(): if filter_['field'] not in stix_obj.keys():
break break
@ -616,34 +635,34 @@ class DataSource(object):
else: else:
# filter operation not supported # filter operation not supported
continue continue
'''
#TODO: I think the rest of the operations only
#apply to timestamps, in which case I dont think
#simple operator usage (like below) works
elif filter_['op'] == ">": # TODO: I think the rest of the operations only
if not stix_obj[filter_['field']] > filter_['value']: # apply to timestamps, in which case I don't think
clean = False # simple operator usage (like below) works
break
elif filter_['op'] == "<": # elif filter_['op'] == ">":
if not stix_obj[filter_['field']] < filter_['value']: # if not stix_obj[filter_['field']] > filter_['value']:
clean = False # clean = False
break # break
#
elif filter_['op'] == ">=": # elif filter_['op'] == "<":
if not stix_obj[filter_['field']] >= filter_['value']: # if not stix_obj[filter_['field']] < filter_['value']:
clean = False # clean = False
break # break
#
elif filter_['op'] == "<=": # elif filter_['op'] == ">=":
if not stix_obj[filter_['field']] <= filter_['value']: # if not stix_obj[filter_['field']] >= filter_['value']:
clean = False # clean = False
break # break
''' #
# elif filter_['op'] == "<=":
# if not stix_obj[filter_['field']] <= filter_['value']:
# clean = False
# break
except TypeError: except TypeError:
# type mismatch of comparison operands - ignore filter, no error raised for now # type mismatch of comparison operands - ignore filter,
# no error raised for now
pass pass
# if object unmarked after all filter, add it # if object unmarked after all filter, add it
@ -655,7 +674,7 @@ class DataSource(object):
return filtered_stix_objs return filtered_stix_objs
def deduplicate(self, stix_obj_list): def deduplicate(self, stix_obj_list):
'''deduplicate a list of STIX objects into a unique set """deduplicate a list of STIX objects into a unique set
reduces a set of STIX objects to unique set by looking reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version at 'id' and 'modified' fields - as a unique object version
@ -668,7 +687,7 @@ class DataSource(object):
(list): a unique set of the passed STIX object list (list): a unique set of the passed STIX object list
''' """
unique = [] unique = []
have = False have = False
for i in stix_obj_list: for i in stix_obj_list:

109
stix2/sources/taxii.py
View File

@ -3,27 +3,23 @@ from requests.auth import HTTPBasicAuth
from stix2.sources import DataSource from stix2.sources import DataSource
''' # TODO: -Should we make properties for the TAXIIDataSource address and other
TODO: # possible variables that are found in "self.taxii_info"
-Should we make properties for the TAXIIDataSource address and other possible variables
that are found in "self.taxii_info"
'''
TAXII_FILTERS = ['added_after', 'match[id]', 'match[type]', 'match[version]'] TAXII_FILTERS = ['added_after', 'match[id]', 'match[type]', 'match[version]']
class TAXIIDataSource(DataSource): class TAXIIDataSource(DataSource):
'''STIX 2.0 Data Source - TAXII 2.0 module''' """STIX 2.0 Data Source - TAXII 2.0 module"""
def __init__(self, api_root=None, auth=None, name="TAXII", ):
def __init__(self, api_root=None, auth=None, name="TAXII"):
super(TAXIIDataSource, self).__init__(name=name) super(TAXIIDataSource, self).__init__(name=name)
self.taxii_info = { self.taxii_info = {
"api_root": { "api_root": {
"url": api_root "url": api_root
}, },
"auth": auth "auth": auth
} }
@ -34,7 +30,8 @@ class TAXIIDataSource(DataSource):
resp = requests.get(coll_url, resp = requests.get(coll_url,
headers=headers, headers=headers,
auth=HTTPBasicAuth(self.taxii_info['auth']['user'], self.taxii_info['auth']['pass'])) auth=HTTPBasicAuth(self.taxii_info['auth']['user'],
self.taxii_info['auth']['pass']))
# TESTING # TESTING
# print("\n-------__init__() ----\n") # print("\n-------__init__() ----\n")
# print(resp.text) # print(resp.text)
@ -53,8 +50,10 @@ class TAXIIDataSource(DataSource):
if e == "collections": if e == "collections":
raise raise
# raise type(e), type(e)(e.message + # raise type(e), type(e)(e.message +
# "To connect to the TAXII collections, the API root resource must contain a collection endpoint URL. # "To connect to the TAXII collections, the API root
# This was not found in the API root resource received from the API root" ), sys.exc_info()[2] # resource must contain a collection endpoint URL.
# This was not found in the API root resource received
# from the API root" ), sys.exc_info()[2]
except requests.ConnectionError as e: except requests.ConnectionError as e:
raise raise
@ -62,20 +61,21 @@ class TAXIIDataSource(DataSource):
# "Attempting to connect to %s" % coll_url) # "Attempting to connect to %s" % coll_url)
def get(self, id_, _composite_filters=None): def get(self, id_, _composite_filters=None):
'''get STIX 2 object from TAXII source by specified 'id' """Get STIX 2.0 object from TAXII source by specified 'id'
NOTE: Notes:
-just pass _composite_filters to the query() as they are applied there Just pass _composite_filters to the query() as they are applied
-deduplication of results is also done within query() there. de-duplication of results is also done within query()
Args: Args:
id_ (str): id of STIX object to retrieve id_ (str): id of STIX object to retrieve
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one) _composite_filters (list): filters passed from a Composite Data
Source (if this data source is attached to one)
Returns: Returns:
''' """
# make query in TAXII query format since 'id' is TAXii field # make query in TAXII query format since 'id' is TAXii field
query = [ query = [
@ -94,20 +94,21 @@ class TAXIIDataSource(DataSource):
return stix_obj return stix_obj
def all_versions(self, id_, _composite_filters=None): def all_versions(self, id_, _composite_filters=None):
'''get all versions of STIX 2 object from TAXII source by specified 'id' """Get all versions of STIX 2.0 object from TAXII source by
specified 'id'
NOTE: Notes:
-just passes _composite_filters to the query() as they are applied there Just passes _composite_filters to the query() as they are applied
-deduplication of results is also done within query() there. de-duplication of results is also done within query()
Args: Args:
id_ (str): id of STIX objects to retrieve id_ (str): id of STIX objects to retrieve
_composite_filters (list): filters passed from a Composite Data
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one) Source (if this data source is attached to one)
Returns: Returns:
The query results with filters applied.
''' """
# make query in TAXII query format since 'id' is TAXII field # make query in TAXII query format since 'id' is TAXII field
query = [ query = [
@ -123,15 +124,16 @@ class TAXIIDataSource(DataSource):
return all_data return all_data
def query(self, query=None, _composite_filters=None): def query(self, query=None, _composite_filters=None):
'''query the TAXII data source for STIX objects matching the query """Query the TAXII data source for STIX objects matching the query
The final full query could contain filters from: The final full query could contain filters from:
-the current API call -the current API call
-Composite Data source filters (that are passed in via '_composite_filters') -Composite Data source filters (that are passed in via
'_composite_filters')
-TAXII data source filters that are attached -TAXII data source filters that are attached
TAXII filters ['added_after', 'match[<>]'] are extracted and sent to TAXII TAXII filters ['added_after', 'match[<>]'] are extracted and sent
if they are present to TAXII if they are present
TODO: Authentication for TAXII TODO: Authentication for TAXII
@ -139,12 +141,13 @@ class TAXIIDataSource(DataSource):
query(list): list of filters (dicts) to search on query(list): list of filters (dicts) to search on
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one) _composite_filters (list): filters passed from a
Composite Data Source (if this data source is attached to one)
Returns: Returns:
''' """
all_data = [] all_data = []
@ -157,24 +160,27 @@ class TAXIIDataSource(DataSource):
if _composite_filters: if _composite_filters:
query += _composite_filters query += _composite_filters
# seperate taxii query terms (can be done remotely) # separate taxii query terms (can be done remotely)
taxii_filters = self._parse_taxii_filters(query) taxii_filters = self._parse_taxii_filters(query)
# for each collection endpoint - send query request # for each collection endpoint - send query request
for collection in self.taxii_info['api_root']['collections']: for collection in self.taxii_info['api_root']['collections']:
coll_obj_url = self.taxii_info['api_root']['url'] + "/collections/" + str(collection['id']) + "/objects/" coll_obj_url = "/".join([self.taxii_info['api_root']['url'],
"collections", str(collection['id']),
"objects"])
headers = {} headers = {}
try: try:
resp = requests.get(coll_obj_url, resp = requests.get(coll_obj_url,
params=taxii_filters, params=taxii_filters,
headers=headers, headers=headers,
auth=HTTPBasicAuth(self.taxii_info['auth']['user'], self.taxii_info['auth']['pass'])) auth=HTTPBasicAuth(self.taxii_info['auth']['user'],
self.taxii_info['auth']['pass']))
# TESTING # TESTING
# print("\n-------query() ----\n") # print("\n-------query() ----\n")
# print("Request that was sent: \n") # print("Request that was sent: \n")
# print(resp.url) # print(resp.url)
# print("Reponse: \n") # print("Response: \n")
# print(json.dumps(resp.json(),indent=4)) # print(json.dumps(resp.json(),indent=4))
# print("\n") # print("\n")
# print(resp.status_code) # print(resp.status_code)
@ -194,9 +200,8 @@ class TAXIIDataSource(DataSource):
# raise type(e), type(e)(e.message + # raise type(e), type(e)(e.message +
# "Attempting to connect to %s" % coll_url) # "Attempting to connect to %s" % coll_url)
''' # TODO: Is there a way to collect exceptions while carrying
TODO: Is there a way to collect exceptions while carrying on then raise all of them at the end? # on then raise all of them at the end?
'''
# deduplicate data (before filtering as reduces wasted filtering) # deduplicate data (before filtering as reduces wasted filtering)
all_data = self.deduplicate(all_data) all_data = self.deduplicate(all_data)
@ -207,24 +212,25 @@ class TAXIIDataSource(DataSource):
return all_data return all_data
def _parse_taxii_filters(self, query): def _parse_taxii_filters(self, query):
'''parse out TAXII filters that the TAXII server can filter on """Parse out TAXII filters that the TAXII server can filter on
TAXII filters should be analgous to how they are supplied TAXII filters should be analgous to how they are supplied
in the url to the TAXII endpoint. For instance in the url to the TAXII endpoint. For instance
"?match[type]=indicator,sighting" should be in a query dict as follows "?match[type]=indicator,sighting" should be in a query dict as follows
{ {
"field":"match[type]" "field": "match[type]"
"op": "=", "op": "=",
"value":"indicator,sighting" "value": "indicator,sighting"
} }
Args: Args:
query (list): list of filters to extract which ones are TAXII specific query (list): list of filters to extract which ones are TAXII
specific.
Returns: Returns:
params (dict): dict of the TAXII filters but in format required for 'requests.get()' params (dict): dict of the TAXII filters but in format required
for 'requests.get()'.
''' """
params = {} params = {}
@ -234,12 +240,9 @@ class TAXIIDataSource(DataSource):
return params return params
def close(self): def close(self):
'''close down the Data Source - if any clean up is required """Close down the Data Source - if any clean up is required.
''' """
pass pass
''' # TODO: - getters/setters (properties) for TAXII config info
TODO:
- getters/setters (properties) for TAXII config info
'''