MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Code style changes.

stix2.1
Emmanuelle Vargas-Gonzalez 2017-05-26 15:24:33 -04:00
parent 97d8d732fc
commit 2a8af45ec2
2 changed files with 208 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
stix2/sources/__init__.py
View File

@ -1,4 +1,4 @@
'''
"""
Python STIX 2.0 Composite Data Source and Data Source (classes)
@ -6,24 +6,28 @@ Python STIX 2.0 Composite Data Source and Data Source (classes)
-Test everything
-add_filter(), remove_filter(), deduplicate() - if these functions remain the exact same for
both CompositeDataSource and DataSource, they just inherit/have module access to
-add_filter(), remove_filter(), deduplicate() - if these functions remain
the exact same for both CompositeDataSource and DataSource, they just
inherit/have module access to
'''
"""
import abc
import copy
import uuid
from six import iteritems
def make_id():
str(uuid.uuid4())
return str(uuid.uuid4())
# STIX 2.0 fields used to denote object version
STIX_VERSION_FIELDS = ['id', 'modified']
# currently, only STIX 2.0 common SDO fields (that are not compex objects) are supported for filtering on
# Currently, only STIX 2.0 common SDO fields (that are not compex objects)
# are supported for filtering on
STIX_COMMON_FIELDS = [
'type',
'id',
@ -32,11 +36,11 @@ STIX_COMMON_FIELDS = [
'modified',
'revoked',
'labels',
# 'external_references', #list of external references object type - not supported for filtering
# 'external_references', # list of external references object type - not supported for filtering
'object_references',
'object_marking_refs',
'granular_marking_refs',
# 'granular_markings' #list of granular-marking type - not supported for filtering
# 'granular_markings' # list of granular-marking type - not supported for filtering
]
@ -51,28 +55,26 @@ FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
class CompositeDataSource(object):
'''Composite Data Source
"""Composite Data Source
Acts as a controller for all the defined/configured STIX Data Sources
e.g. a user can defined n Data Sources - creating Data Source (objects)
for each. There is only one instance of this for any python STIX 2.0 application
for each. There is only one instance of this for any python STIX 2.0
application
'''
"""
def __init__(self, name="CompositeDataSource"):
'''
"""
Creates a new STIX Data Source.
Args:
'data_sources' (dict): a dict of DataSource objects; to be controlled and used by
the Data Source Controller object
'data_sources' (dict): a dict of DataSource objects; to be
controlled and used by the Data Source Controller object
filters :
name :
Returns:
'''
"""
self.id_ = make_id()
self.name = name
self.data_sources = {}
@ -80,7 +82,7 @@ class CompositeDataSource(object):
self.filter_allowed = {}
def get(self, id_):
'''retrieve STIX object by 'id'
"""Retrieve STIX object by 'id'
federated retrieve method-iterates through all STIX data sources
defined in the "data_sources" parameter. Each data source has a
@ -97,12 +99,12 @@ class CompositeDataSource(object):
Returns:
stix_obj (dict): the STIX object to be returned
'''
"""
all_data = []
# for every configured Data Source, call its retrieve handler
for ds_id, ds in self.data_sources.iteritems():
for ds_id, ds in iteritems(self.data_sources):
data = ds.get(id_=id_, _composite_filters=self.filters.values())
all_data += data
@ -116,7 +118,7 @@ class CompositeDataSource(object):
return stix_obj
def all_versions(self, id_):
'''retrieve STIX objects by 'id'
"""Retrieve STIX objects by 'id'
Federated all_versions retrieve method - iterates through all STIX data
sources defined in "data_sources"
@ -129,22 +131,23 @@ class CompositeDataSource(object):
Returns:
all_data (list): list of STIX objects that have the specified id
'''
"""
all_data = []
# retrieve STIX objects from all configured data sources
for ds_id, ds in self.data_sources.iteritems():
for ds_id, ds in iteritems(self.data_sources):
data = ds.all_versions(id_=id_, _composite_filters=self.filters.values())
all_data += data
# remove exact duplicates (where duplicates are STIX 2.0 objects with the same 'id' and 'modified' values)
# remove exact duplicates (where duplicates are STIX 2.0 objects
# with the same 'id' and 'modified' values)
if len(all_data) > 0:
all_data = self.deduplicate(all_data)
return all_data
def query(self, query=None):
'''composite data source query
"""composite data source query
Federate the query to all Data Sources attached
to the Composite Data Source
@ -155,32 +158,35 @@ class CompositeDataSource(object):
Returns:
all_data (list): list of STIX objects to be returned
'''
"""
if not query:
query = []
all_data = []
# federate query to all attached data sources, pass composite filters to them
for ds_id, ds in self.data_sources.iteritems():
# federate query to all attached data sources,
# pass composite filters to them
for ds_id, ds in iteritems(self.data_sources):
data = ds.query(query=query, _composite_filters=self.filters.values())
all_data += data
# remove exact duplicates (where duplicates are STIX 2.0 objects with the same 'id' and 'modified' values)
# remove exact duplicates (where duplicates are STIX 2.0
# objects with the same 'id' and 'modified' values)
if len(all_data) > 0:
all_data = self.deduplicate(all_data)
return all_data
def add_data_source(self, data_sources):
'''add/attach Data Source to the Composite Data Source instance
"""add/attach Data Source to the Composite Data Source instance
Args:
data_sources (list): a list of Data Source objects to attach to the Composite Data Source
data_sources (list): a list of Data Source objects to attach
to the Composite Data Source
Returns:
'''
"""
for ds in data_sources:
if issubclass(ds, DataSource):
@ -188,37 +194,40 @@ class CompositeDataSource(object):
# data source already attached to Composite Data Source
continue
# add data source to Composite Data Source (its id will be its key identifier)
# add data source to Composite Data Source
# (its id will be its key identifier)
self.data_sources[ds['id']] = ds
else:
# the Data Source object is not a proper subclass of DataSource Abstract Class
# the Data Source object is not a proper subclass
# of DataSource Abstract Class
# TODO: maybe log error?
continue
return
def remove_data_source(self, data_source_ids):
'''remove/detach Data Source from the Composite Data Source instance
"""remove/detach Data Source from the Composite Data Source instance
Args:
data_source_ids (list): a list of Data Source id's( which are strings )
data_source_ids (list): a list of Data Source
id's(which are strings)
Returns:
'''
"""
for id_ in data_source_ids:
try:
if self.data_sources[id_]:
del self.data_sources[id_]
except KeyError:
# Data Source 'id' was not found in CompositeDataSource's list of data sources
# Data Source 'id' was not found in CompositeDataSource's
# list of data sources
pass
return
def get_data_sources(self):
'''return all attached Data Sources
"""return all attached Data Sources
TODO: Make this a property?
@ -226,11 +235,11 @@ class CompositeDataSource(object):
Returns:
'''
"""
return copy.deepcopy(self.data_sources.values())
def add_filter(self, filters):
'''add/attach a filter to the Composite Data Source instance
"""add/attach a filter to the Composite Data Source instance
Args:
filters (list): list of filters (dict) to add to the Data Source
@ -238,7 +247,7 @@ class CompositeDataSource(object):
Returns:
status (list): list of status/error messages
'''
"""
status = []
errors = []
@ -268,12 +277,11 @@ class CompositeDataSource(object):
allowed = False
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
'''
Filter is added regardless of whether it fits requirements
to be a common filter. This is done because some filters
may be added and used by third party Data Sources, where
the filtering may be conducted within those plugins, just not here
'''
# Filter is added regardless of whether it fits requirements
# to be a common filter. This is done because some filters
# may be added and used by third party Data Sources, where the
# filtering may be conducted within those plugins, just not here
id_ = make_id()
filter_['id'] = id_
self.filters['id_'] = filter_
@ -302,7 +310,7 @@ class CompositeDataSource(object):
return ids, status
def remove_filter(self, filter_ids):
'''remove/detach a filter from the Data Source instance
"""Remove/detach a filter from the Data Source instance
Args:
filter_ids (list): list of filter id's (which are strings)
@ -310,7 +318,7 @@ class CompositeDataSource(object):
Returns:
'''
"""
for filter_id in filter_ids:
try:
@ -318,35 +326,36 @@ class CompositeDataSource(object):
del self.filters[filter_id]
del self.filter_allowed[filter_id]
except KeyError:
# filter id not found in list of filters attached to the Composite Data Source
# filter id not found in list of filters
# attached to the Composite Data Source
pass
return
def get_filters(self):
'''return filters attached to Composite Data Source
"""return filters attached to Composite Data Source
Args:
Returns:
(list): the list of filters currently attached to the Data Source
'''
"""
return copy.deepcopy(list(self.filters.values()))
def deduplicate(self, stix_obj_list):
'''deduplicate a list fo STIX objects to a unique set
"""deduplicate a list fo STIX objects to a unique set
Reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version is determined
by the combination of those fields
at 'id' and 'modified' fields - as a unique object version
is determined by the combination of those fields
Args:
stix_obj_list (list): list of STIX objects (dicts)
Returns:
(list): unique set of the passed list of STIX objects
'''
"""
unique = []
dont_have = False
@ -363,7 +372,7 @@ class CompositeDataSource(object):
class DataSource(object):
'''
"""
Abstract Data Source class for STIX 2.0
An implementer will create a concrete subclass from
@ -373,7 +382,7 @@ class DataSource(object):
supply them to a Composite Data Source which calls
the subclass methods when conducting STIX 2.0
data retrievals.
'''
"""
__metaclass__ = abc.ABCMeta
@ -385,87 +394,93 @@ class DataSource(object):
@abc.abstractmethod
def get(self, id_, _composite_filters=None):
'''
"""
Fill:
-implement the specific data source API calls, processing, functionality
requried for retrieving data from the data source
-implement the specific data source API calls, processing,
functionality required for retrieving data from the data source
Args:
id (str): the id of the STIX 2.0 object to retrieve. Should return a single object,
the most recent version of the object specified by the "id".
id_ (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object
specified by the "id".
_composite_filters (list): list of filters passed along from the Composite Data Filter
_composite_filters (list): list of filters passed along from
the Composite Data Filter.
Returns:
stix_obj (dictionary): the STIX object to be returned
'''
"""
stix_obj = None
return stix_obj
@abc.abstractmethod
def all_versions(self, id_, _composite_filters=None):
'''
"""
Fill:
-Similar to get() except returns list of all object versions of the specified "id".
-implement the specific data source API calls, processing, functionality
requried for retrieving data from the data source
-Similar to get() except returns list of all object versions of
the specified "id".
-implement the specific data source API calls, processing,
functionality required for retrieving data from the data source
Args:
id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects,
all the versions of the object specified by the "id".
id_ (str): The id of the STIX 2.0 object to retrieve. Should
return a list of objects, all the versions of the object
specified by the "id".
_composite_filters (list): list of filters passed from the Composite Data Source
_composite_filters (list): list of filters passed from the
Composite Data Source
Returns:
stix_objs (list): a list of STIX objects(where each object is a STIX object)
'''
stix_objs (list): a list of STIX objects (where each object is a
STIX object)
"""
stix_objs = []
return stix_objs
@abc.abstractmethod
def query(self, query, _composite_filters=None):
'''
"""
Fill:
-implement the specific data source API calls, processing, functionality
requried for retrieving query from the data source
-implement the specific data source API calls, processing,
functionality required for retrieving query from the data source
Args:
query (list): a list of filters (which collectively are the query) to conduct search on
query (list): a list of filters (which collectively are the query)
to conduct search on
_composite_filters (list): a list of filters passed from the Composite Data Source
_composite_filters (list): a list of filters passed from the
Composite Data Source
Returns:
'''
"""
stix_objs = []
return stix_objs
@abc.abstractmethod
def close(self):
'''
"""
Fill:
Close, release, shutdown any objects, contexts, variables
Args:
Returns:
(list): list of status/error messages
'''
"""
status = []
return status
def add_filter(self, filters):
'''add/attach a filter to the Data Source instance
"""add/attach a filter to the Data Source instance
Args:
filters (list): list of filters (dict) to add to the Data Source
@ -473,7 +488,7 @@ class DataSource(object):
Returns:
status (list): list of status/error messages
'''
"""
status = []
errors = []
@ -503,12 +518,11 @@ class DataSource(object):
allowed = False
errors.append("Filter 'value' type is not supported. The type(value) must be python immutable type or dictionary")
'''
Filter is added regardless of whether it fits requirements
to be a common filter. This is done because some filters
may be added and used by third party Data Sources, where
the filtering may be conducted within those plugins, just not here
'''
# Filter is added regardless of whether it fits requirements
# to be a common filter. This is done because some filters
# may be added and used by third party Data Sources, where the
# filtering may be conducted within those plugins, just not here
id_ = make_id()
filter_['id'] = id_
self.filters[id_] = filter_
@ -538,15 +552,16 @@ class DataSource(object):
return ids, status
def remove_filter(self, filter_ids):
'''remove/detach a filter from the Data Source instance
"""remove/detach a filter from the Data Source instance
Args:
filter_ids (list): list of filter ids to dettach/remove from Data Source
filter_ids (list): list of filter ids to dettach/remove
from Data Source
Returns:
'''
"""
for filter_id in filter_ids:
try:
if filter_id in self.filters:
@ -559,18 +574,19 @@ class DataSource(object):
return
def get_filters(self):
'''return copy of all filters currently attached to Data Source
"""return copy of all filters currently attached to Data Source
TODO: make this a property?
Returns:
(list): a copy of all the filters(dict) which are attached to Data Source
(list): a copy of all the filters(dict) which are attached
to Data Source
'''
"""
return copy.deepcopy(list(self.filters.values()))
def apply_common_filters(self, stix_objs, query):
'''evaluates filters against a set of STIX 2.0 objects
"""evaluates filters against a set of STIX 2.0 objects
Supports only STIX 2.0 common property fields
@ -579,9 +595,10 @@ class DataSource(object):
query (list): list of filters (combined form complete query)
Returns:
(list): list of STIX objects that successfully evaluate against the query
(list): list of STIX objects that successfully evaluate against
the query
'''
"""
filtered_stix_objs = []
@ -590,12 +607,14 @@ class DataSource(object):
clean = True
for filter_ in query:
# skip filter as filter was identified (when added) as not a common filter
# skip filter as filter was identified (when added) as
# not a common filter
if 'id' in filter_ and self.filter_allowed[filter_['id']] is False:
continue
# check filter "field" is in STIX object - if cant be applied due to STIX object,
# STIX object is discarded (i.e. did not make it through the filter)
# check filter "field" is in STIX object - if cant be applied
# due to STIX object, STIX object is discarded (i.e. did not
# make it through the filter)
if filter_['field'] not in stix_obj.keys():
break
@ -616,34 +635,34 @@ class DataSource(object):
else:
# filter operation not supported
continue
'''
#TODO: I think the rest of the operations only
#apply to timestamps, in which case I dont think
#simple operator usage (like below) works
elif filter_['op'] == ">":
if not stix_obj[filter_['field']] > filter_['value']:
clean = False
break
# TODO: I think the rest of the operations only
# apply to timestamps, in which case I don't think
# simple operator usage (like below) works
elif filter_['op'] == "<":
if not stix_obj[filter_['field']] < filter_['value']:
clean = False
break
elif filter_['op'] == ">=":
if not stix_obj[filter_['field']] >= filter_['value']:
clean = False
break
elif filter_['op'] == "<=":
if not stix_obj[filter_['field']] <= filter_['value']:
clean = False
break
'''
# elif filter_['op'] == ">":
# if not stix_obj[filter_['field']] > filter_['value']:
# clean = False
# break
#
# elif filter_['op'] == "<":
# if not stix_obj[filter_['field']] < filter_['value']:
# clean = False
# break
#
# elif filter_['op'] == ">=":
# if not stix_obj[filter_['field']] >= filter_['value']:
# clean = False
# break
#
# elif filter_['op'] == "<=":
# if not stix_obj[filter_['field']] <= filter_['value']:
# clean = False
# break
except TypeError:
# type mismatch of comparison operands - ignore filter, no error raised for now
# type mismatch of comparison operands - ignore filter,
# no error raised for now
pass
# if object unmarked after all filter, add it
@ -655,7 +674,7 @@ class DataSource(object):
return filtered_stix_objs
def deduplicate(self, stix_obj_list):
'''deduplicate a list of STIX objects into a unique set
"""deduplicate a list of STIX objects into a unique set
reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version
@ -668,7 +687,7 @@ class DataSource(object):
(list): a unique set of the passed STIX object list
'''
"""
unique = []
have = False
for i in stix_obj_list:

109
stix2/sources/taxii.py
View File

@ -3,27 +3,23 @@ from requests.auth import HTTPBasicAuth
from stix2.sources import DataSource
'''
TODO:
# TODO: -Should we make properties for the TAXIIDataSource address and other
# possible variables that are found in "self.taxii_info"
-Should we make properties for the TAXIIDataSource address and other possible variables
that are found in "self.taxii_info"
'''
TAXII_FILTERS = ['added_after', 'match[id]', 'match[type]', 'match[version]']
class TAXIIDataSource(DataSource):
'''STIX 2.0 Data Source - TAXII 2.0 module'''
def __init__(self, api_root=None, auth=None, name="TAXII", ):
"""STIX 2.0 Data Source - TAXII 2.0 module"""
def __init__(self, api_root=None, auth=None, name="TAXII"):
super(TAXIIDataSource, self).__init__(name=name)
self.taxii_info = {
"api_root": {
"url": api_root
},
"url": api_root
},
"auth": auth
}
@ -34,7 +30,8 @@ class TAXIIDataSource(DataSource):
resp = requests.get(coll_url,
headers=headers,
auth=HTTPBasicAuth(self.taxii_info['auth']['user'], self.taxii_info['auth']['pass']))
auth=HTTPBasicAuth(self.taxii_info['auth']['user'],
self.taxii_info['auth']['pass']))
# TESTING
# print("\n-------__init__() ----\n")
# print(resp.text)
@ -53,8 +50,10 @@ class TAXIIDataSource(DataSource):
if e == "collections":
raise
# raise type(e), type(e)(e.message +
# "To connect to the TAXII collections, the API root resource must contain a collection endpoint URL.
# This was not found in the API root resource received from the API root" ), sys.exc_info()[2]
# "To connect to the TAXII collections, the API root
# resource must contain a collection endpoint URL.
# This was not found in the API root resource received
# from the API root" ), sys.exc_info()[2]
except requests.ConnectionError as e:
raise
@ -62,20 +61,21 @@ class TAXIIDataSource(DataSource):
# "Attempting to connect to %s" % coll_url)
def get(self, id_, _composite_filters=None):
'''get STIX 2 object from TAXII source by specified 'id'
"""Get STIX 2.0 object from TAXII source by specified 'id'
NOTE:
-just pass _composite_filters to the query() as they are applied there
-deduplication of results is also done within query()
Notes:
Just pass _composite_filters to the query() as they are applied
there. de-duplication of results is also done within query()
Args:
id_ (str): id of STIX object to retrieve
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one)
_composite_filters (list): filters passed from a Composite Data
Source (if this data source is attached to one)
Returns:
'''
"""
# make query in TAXII query format since 'id' is TAXii field
query = [
@ -94,20 +94,21 @@ class TAXIIDataSource(DataSource):
return stix_obj
def all_versions(self, id_, _composite_filters=None):
'''get all versions of STIX 2 object from TAXII source by specified 'id'
"""Get all versions of STIX 2.0 object from TAXII source by
specified 'id'
NOTE:
-just passes _composite_filters to the query() as they are applied there
-deduplication of results is also done within query()
Notes:
Just passes _composite_filters to the query() as they are applied
there. de-duplication of results is also done within query()
Args:
id_ (str): id of STIX objects to retrieve
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one)
_composite_filters (list): filters passed from a Composite Data
Source (if this data source is attached to one)
Returns:
'''
The query results with filters applied.
"""
# make query in TAXII query format since 'id' is TAXII field
query = [
@ -123,15 +124,16 @@ class TAXIIDataSource(DataSource):
return all_data
def query(self, query=None, _composite_filters=None):
'''query the TAXII data source for STIX objects matching the query
"""Query the TAXII data source for STIX objects matching the query
The final full query could contain filters from:
-the current API call
-Composite Data source filters (that are passed in via '_composite_filters')
-Composite Data source filters (that are passed in via
'_composite_filters')
-TAXII data source filters that are attached
TAXII filters ['added_after', 'match[<>]'] are extracted and sent to TAXII
if they are present
TAXII filters ['added_after', 'match[<>]'] are extracted and sent
to TAXII if they are present
TODO: Authentication for TAXII
@ -139,12 +141,13 @@ class TAXIIDataSource(DataSource):
query(list): list of filters (dicts) to search on
_composite_filters (list): filters passed from a Composite Data Source (if this data source is attached to one)
_composite_filters (list): filters passed from a
Composite Data Source (if this data source is attached to one)
Returns:
'''
"""
all_data = []
@ -157,24 +160,27 @@ class TAXIIDataSource(DataSource):
if _composite_filters:
query += _composite_filters
# seperate taxii query terms (can be done remotely)
# separate taxii query terms (can be done remotely)
taxii_filters = self._parse_taxii_filters(query)
# for each collection endpoint - send query request
for collection in self.taxii_info['api_root']['collections']:
coll_obj_url = self.taxii_info['api_root']['url'] + "/collections/" + str(collection['id']) + "/objects/"
coll_obj_url = "/".join([self.taxii_info['api_root']['url'],
"collections", str(collection['id']),
"objects"])
headers = {}
try:
resp = requests.get(coll_obj_url,
params=taxii_filters,
headers=headers,
auth=HTTPBasicAuth(self.taxii_info['auth']['user'], self.taxii_info['auth']['pass']))
auth=HTTPBasicAuth(self.taxii_info['auth']['user'],
self.taxii_info['auth']['pass']))
# TESTING
# print("\n-------query() ----\n")
# print("Request that was sent: \n")
# print(resp.url)
# print("Reponse: \n")
# print("Response: \n")
# print(json.dumps(resp.json(),indent=4))
# print("\n")
# print(resp.status_code)
@ -194,9 +200,8 @@ class TAXIIDataSource(DataSource):
# raise type(e), type(e)(e.message +
# "Attempting to connect to %s" % coll_url)
'''
TODO: Is there a way to collect exceptions while carrying on then raise all of them at the end?
'''
# TODO: Is there a way to collect exceptions while carrying
# on then raise all of them at the end?
# deduplicate data (before filtering as reduces wasted filtering)
all_data = self.deduplicate(all_data)
@ -207,24 +212,25 @@ class TAXIIDataSource(DataSource):
return all_data
def _parse_taxii_filters(self, query):
'''parse out TAXII filters that the TAXII server can filter on
"""Parse out TAXII filters that the TAXII server can filter on
TAXII filters should be analgous to how they are supplied
in the url to the TAXII endpoint. For instance
"?match[type]=indicator,sighting" should be in a query dict as follows
{
"field":"match[type]"
"field": "match[type]"
"op": "=",
"value":"indicator,sighting"
"value": "indicator,sighting"
}
Args:
query (list): list of filters to extract which ones are TAXII specific
query (list): list of filters to extract which ones are TAXII
specific.
Returns:
params (dict): dict of the TAXII filters but in format required for 'requests.get()'
'''
params (dict): dict of the TAXII filters but in format required
for 'requests.get()'.
"""
params = {}
@ -234,12 +240,9 @@ class TAXIIDataSource(DataSource):
return params
def close(self):
'''close down the Data Source - if any clean up is required
"""Close down the Data Source - if any clean up is required.
'''
"""
pass
'''
TODO:
- getters/setters (properties) for TAXII config info
'''
# TODO: - getters/setters (properties) for TAXII config info