MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

Formatting changes, replace deduplicate() code in DataSource, missing super() calls to initialize objects.

stix2.1
Emmanuelle Vargas-Gonzalez 2017-08-11 08:10:20 -04:00
parent a4ead4f6e7
commit 86fd3778f5
5 changed files with 162 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

211
stix2/sources/__init__.py
View File

@ -9,9 +9,11 @@ Classes:
TODO:Test everything TODO:Test everything
NOTE: add_filter(), remove_filter(), deduplicate() - if these functions remain Notes:
the exact same for DataSource, DataSink, CompositeDataSource etc... -> just add_filter(), remove_filter(), deduplicate() - if these functions remain
make those functions an interface to inherit? the exact same for DataSource, DataSink, CompositeDataSource etc... -> just
make those functions an interface to inherit?
""" """
import copy import copy
@ -23,7 +25,8 @@ from six import iteritems
def make_id(): def make_id():
return str(uuid.uuid4()) return str(uuid.uuid4())
# Currently, only STIX 2.0 common SDO fields (that are not compex objects)
# Currently, only STIX 2.0 common SDO fields (that are not complex objects)
# are supported for filtering on # are supported for filtering on
STIX_COMMON_FIELDS = [ STIX_COMMON_FIELDS = [
"created", "created",
@ -59,6 +62,7 @@ class DataStore(object):
""" """
An implementer will create a concrete subclass from An implementer will create a concrete subclass from
this abstract class for the specific data store. this abstract class for the specific data store.
""" """
def __init__(self, name="DataStore"): def __init__(self, name="DataStore"):
self.name = name self.name = name
@ -69,29 +73,25 @@ class DataStore(object):
def get(self, stix_id): def get(self, stix_id):
""" """
Implement: Implement:
-translate API get() call to the appropriate DataSource call Translate API get() call to the appropriate DataSource call
Args: Args:
stix_id (str): the id of the STIX 2.0 object to retrieve. Should stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object return a single object, the most recent version of the object
specified by the "id". specified by the "id".
_composite_filters (list): list of filters passed along from
the Composite Data Filter.
Returns: Returns:
stix_obj (dictionary): the STIX object to be returned stix_obj (dictionary): the STIX object to be returned
""" """
return self.source.get(stix_id=stix_id) return self.source.get(stix_id=stix_id)
def all_versions(self, stix_id): def all_versions(self, stix_id):
""" """
Implement: Implement:
-translate all_versions() call to the appropriate DataSource call Translate all_versions() call to the appropriate DataSource call
Args: Args:
stix_id (str): the id of the STIX 2.0 object to retrieve. Should stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object return a single object, the most recent version of the object
specified by the "id". specified by the "id".
@ -102,21 +102,19 @@ class DataStore(object):
Returns: Returns:
stix_objs (list): a list of STIX objects (where each object is a stix_objs (list): a list of STIX objects (where each object is a
STIX object) STIX object)
""" """
return self.source.all_versions(stix_id=stix_id) return self.source.all_versions(stix_id=stix_id)
def query(self, query): def query(self, query):
""" """
Fill: Fill:
-implement the specific data source API calls, processing, Implement the specific data source API calls, processing,
functionality required for retrieving query from the data source functionality required for retrieving query from the data source
Args: Args:
query (list): a list of filters (which collectively are the query) query (list): a list of filters (which collectively are the query)
to conduct search on to conduct search on.
_composite_filters (list): a list of filters passed from the
Composite Data Source
Returns: Returns:
stix_objs (list): a list of STIX objects (where each object is a stix_objs (list): a list of STIX objects (where each object is a
@ -136,8 +134,13 @@ class DataStore(object):
class DataSink(object): class DataSink(object):
""" """
An implementer will create a concrete subclass from this Abstract class for defining a data sink. Intended for subclassing into
abstract class for the specific data sink. different sink components.
Attributes:
id (str): A unique UUIDv4 to identify this DataSink.
name (str): The descriptive name that identifies this DataSink.
""" """
def __init__(self, name="DataSink"): def __init__(self, name="DataSink"):
@ -147,16 +150,25 @@ class DataSink(object):
def add(self, stix_objs): def add(self, stix_objs):
""" """
Fill: Fill:
-implement the specific data sink API calls, processing, Implement the specific data sink API calls, processing,
functionality required for adding data to the sink functionality required for adding data to the sink
""" """
raise NotImplementedError() raise NotImplementedError()
class DataSource(object): class DataSource(object):
""" """
An implementer will create a concrete subclass from Abstract class for defining a data source. Intended for subclassing into
this abstract class for the specific data source. different source components.
Attributes:
id (str): A unique UUIDv4 to identify this DataSource.
name (str): The descriptive name that identifies this DataSource.
filters (dict): A collection of filters present in this DataSource.
filter_allowed (dict): A collection of the allowed filters in this
DataSource.
""" """
def __init__(self, name="DataSource"): def __init__(self, name="DataSource"):
@ -168,11 +180,10 @@ class DataSource(object):
def get(self, stix_id, _composite_filters=None): def get(self, stix_id, _composite_filters=None):
""" """
Fill: Fill:
-implement the specific data source API calls, processing, Implement the specific data source API calls, processing,
functionality required for retrieving data from the data source functionality required for retrieving data from the data source
Args: Args:
stix_id (str): the id of the STIX 2.0 object to retrieve. Should stix_id (str): the id of the STIX 2.0 object to retrieve. Should
return a single object, the most recent version of the object return a single object, the most recent version of the object
specified by the "id". specified by the "id".
@ -196,7 +207,7 @@ class DataSource(object):
functionality required for retrieving data from the data source functionality required for retrieving data from the data source
Args: Args:
id (str): The id of the STIX 2.0 object to retrieve. Should stix_id (str): The id of the STIX 2.0 object to retrieve. Should
return a list of objects, all the versions of the object return a list of objects, all the versions of the object
specified by the "id". specified by the "id".
@ -206,8 +217,8 @@ class DataSource(object):
Returns: Returns:
stix_objs (list): a list of STIX objects (where each object is a stix_objs (list): a list of STIX objects (where each object is a
STIX object) STIX object)
"""
"""
raise NotImplementedError() raise NotImplementedError()
def query(self, query, _composite_filters=None): def query(self, query, _composite_filters=None):
@ -225,13 +236,11 @@ class DataSource(object):
Returns: Returns:
""" """
raise NotImplementedError() raise NotImplementedError()
def add_filter(self, filters): def add_filter(self, filters):
"""add/attach a filter to the Data Source instance """Add/attach a filter to the Data Source instance
Args: Args:
filters (list): list of filters (dict) to add to the Data Source filters (list): list of filters (dict) to add to the Data Source
@ -240,14 +249,13 @@ class DataSource(object):
status (list): list of status/error messages status (list): list of status/error messages
""" """
status = [] status = []
errors = [] errors = []
ids = [] ids = []
allowed = True allowed = True
for filter_ in filters: for filter_ in filters:
# check required filter components ("field", "op", "value") exist # check required filter components ('field', 'op', 'value') exist
for field in FILTER_FIELDS: for field in FILTER_FIELDS:
if field not in filter_.keys(): if field not in filter_.keys():
allowed = False allowed = False
@ -306,14 +314,11 @@ class DataSource(object):
return ids, status return ids, status
def remove_filter(self, filter_ids): def remove_filter(self, filter_ids):
"""remove/detach a filter from the Data Source instance """Remove/detach a filter from the Data Source instance
Args: Args:
filter_ids (list): list of filter ids to dettach/remove filter_ids (list): list of filter ids to detach/remove
from Data Source from Data Source.
Returns:
""" """
for filter_id in filter_ids: for filter_id in filter_ids:
@ -328,7 +333,7 @@ class DataSource(object):
return return
def get_filters(self): def get_filters(self):
"""return copy of all filters currently attached to Data Source """Return copy of all filters currently attached to Data Source
TODO: make this a property? TODO: make this a property?
@ -340,7 +345,7 @@ class DataSource(object):
return copy.deepcopy(list(self.filters.values())) return copy.deepcopy(list(self.filters.values()))
def apply_common_filters(self, stix_objs, query): def apply_common_filters(self, stix_objs, query):
"""evaluates filters against a set of STIX 2.0 objects """Evaluates filters against a set of STIX 2.0 objects
Supports only STIX 2.0 common property fields Supports only STIX 2.0 common property fields
@ -350,10 +355,9 @@ class DataSource(object):
Returns: Returns:
(list): list of STIX objects that successfully evaluate against (list): list of STIX objects that successfully evaluate against
the query the query.
""" """
filtered_stix_objs = [] filtered_stix_objs = []
# evaluate objects against filter # evaluate objects against filter
@ -390,9 +394,9 @@ class DataSource(object):
return filtered_stix_objs return filtered_stix_objs
def deduplicate(self, stix_obj_list): def deduplicate(self, stix_obj_list):
"""deduplicate a list of STIX objects into a unique set """Deduplicate a list of STIX objects to a unique set
reduces a set of STIX objects to unique set by looking Reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version at 'id' and 'modified' fields - as a unique object version
is determined by the combination of those fields is determined by the combination of those fields
@ -400,30 +404,34 @@ class DataSource(object):
stix_obj_list (list): list of STIX objects (dicts) stix_obj_list (list): list of STIX objects (dicts)
Returns: Returns:
(list): a unique set of the passed STIX object list A list with a unique set of the passed list of STIX objects.
""" """
unique = [] unique_objs = {}
have = False
for i in stix_obj_list: for obj in stix_obj_list:
for j in unique: unique_objs[(obj['id'], obj['modified'])] = obj
if i['id'] == j['id'] and i['modified'] == j['modified']:
have = True return list(unique_objs.values())
break
if not have:
unique.append(i)
have = False
return unique
class CompositeDataSource(object): class CompositeDataSource(object):
"""Composite Data Source """Composite Data Source
Acts as a controller for all the defined/configured STIX Data Sources Acts as a controller for all the defined/configured STIX Data Sources
e.g. a user can defined n Data Sources - creating Data Source (objects) e.g. a user can define n Data Sources - creating Data Source (objects)
for each. There is only one instance of this for any python STIX 2.0 for each. There is only one instance of this for any python STIX 2.0
application application.
Attributes:
id (str): A UUIDv4 to identify this CompositeDataSource.
name (str): The name that identifies this CompositeDataSource.
data_sources (dict): A dictionary of DataSource objects; to be
controlled and used by the Data Source Controller object.
filters (dict): A collection of filters present in this
CompositeDataSource.
filter_allowed (dict): A collection of the allowed filters in this
CompositeDataSource.
""" """
def __init__(self, name="CompositeDataSource"): def __init__(self, name="CompositeDataSource"):
@ -431,11 +439,9 @@ class CompositeDataSource(object):
Creates a new STIX Data Source. Creates a new STIX Data Source.
Args: Args:
'data_sources' (dict): a dict of DataSource objects; to be name (str): A string containing the name to attach in the
controlled and used by the Data Source Controller object CompositeDataSource instance.
filters :
name :
""" """
self.id = make_id() self.id = make_id()
self.name = name self.name = name
@ -446,23 +452,23 @@ class CompositeDataSource(object):
def get(self, stix_id): def get(self, stix_id):
"""Retrieve STIX object by 'id' """Retrieve STIX object by 'id'
federated retrieve method-iterates through all STIX data sources Federated retrieve method-iterates through all STIX data sources
defined in the "data_sources" parameter. Each data source has a defined in the "data_sources" parameter. Each data source has a
specific API retrieve-like function and associated parameters. This specific API retrieve-like function and associated parameters. This
function does a federated retrieval and consolidation of the data function does a federated retrieval and consolidation of the data
returned from all the STIX data sources. returned from all the STIX data sources.
note: a composite data source will pass its attached filters to Notes:
each configured data source, pushing filtering to them to handle A composite data source will pass its attached filters to
each configured data source, pushing filtering to them to handle.
Args: Args:
id (str): the id of the STIX object to retrieve stix_id (str): the id of the STIX object to retrieve.
Returns: Returns:
stix_obj (dict): the STIX object to be returned stix_obj (dict): the STIX object to be returned.
""" """
all_data = [] all_data = []
# for every configured Data Source, call its retrieve handler # for every configured Data Source, call its retrieve handler
@ -485,14 +491,16 @@ class CompositeDataSource(object):
Federated all_versions retrieve method - iterates through all STIX data Federated all_versions retrieve method - iterates through all STIX data
sources defined in "data_sources" sources defined in "data_sources"
note: a composite data source will pass its attached filters to Notes:
each configured data source, pushing filtering to them to handle A composite data source will pass its attached filters to
each configured data source, pushing filtering to them to handle
Args: Args:
id_ (str): id of the STIX objects to retrieve stix_id (str): id of the STIX objects to retrieve
Returns: Returns:
all_data (list): list of STIX objects that have the specified id all_data (list): list of STIX objects that have the specified id
""" """
all_data = [] all_data = []
@ -509,10 +517,10 @@ class CompositeDataSource(object):
return all_data return all_data
def query(self, query=None): def query(self, query=None):
"""composite data source query """Composite data source query
Federate the query to all Data Sources attached Federate the query to all Data Sources attached to the
to the Composite Data Source Composite Data Source.
Args: Args:
query (list): list of filters to search on query (list): list of filters to search on
@ -540,16 +548,13 @@ class CompositeDataSource(object):
return all_data return all_data
def add_data_source(self, data_sources): def add_data_source(self, data_sources):
"""add/attach Data Source to the Composite Data Source instance """Add/attach Data Source to the Composite Data Source instance
Args: Args:
data_sources (list): a list of Data Source objects to attach data_sources (list): a list of Data Source objects to attach
to the Composite Data Source to the Composite Data Source
Returns:
""" """
for ds in data_sources: for ds in data_sources:
if issubclass(ds, DataSource): if issubclass(ds, DataSource):
if self.data_sources[ds['id']] in self.data_sources.keys(): if self.data_sources[ds['id']] in self.data_sources.keys():
@ -568,7 +573,7 @@ class CompositeDataSource(object):
return return
def remove_data_source(self, data_source_ids): def remove_data_source(self, data_source_ids):
"""remove/detach Data Source from the Composite Data Source instance """Remove/detach Data Source from the Composite Data Source instance
Args: Args:
data_source_ids (list): a list of Data Source data_source_ids (list): a list of Data Source
@ -590,17 +595,13 @@ class CompositeDataSource(object):
@property @property
def data_sources(self): def data_sources(self):
"""return all attached Data Sources """Return all attached Data Sources
Args:
Returns:
""" """
return copy.deepcopy(self.data_sources.values()) return copy.deepcopy(self.data_sources.values())
def add_filter(self, filters): def add_filter(self, filters):
"""add/attach a filter to the Composite Data Source instance """Add/attach a filter to the Composite Data Source instance
Args: Args:
filters (list): list of filters (dict) to add to the Data Source filters (list): list of filters (dict) to add to the Data Source
@ -609,7 +610,6 @@ class CompositeDataSource(object):
status (list): list of status/error messages status (list): list of status/error messages
""" """
status = [] status = []
errors = [] errors = []
ids = [] ids = []
@ -679,12 +679,9 @@ class CompositeDataSource(object):
Args: Args:
filter_ids (list): list of filter id's (which are strings) filter_ids (list): list of filter id's (which are strings)
dettach from the Composite Data Source detach from the Composite Data Source.
Returns:
""" """
for filter_id in filter_ids: for filter_id in filter_ids:
try: try:
if filter_id in self.filters: if filter_id in self.filters:
@ -699,7 +696,7 @@ class CompositeDataSource(object):
@property @property
def filters(self): def filters(self):
"""return filters attached to Composite Data Source """Return filters attached to Composite Data Source
Returns: Returns:
(list): the list of filters currently attached to the Data Source (list): the list of filters currently attached to the Data Source
@ -708,7 +705,7 @@ class CompositeDataSource(object):
return copy.deepcopy(list(self.filters.values())) return copy.deepcopy(list(self.filters.values()))
def deduplicate(self, stix_obj_list): def deduplicate(self, stix_obj_list):
"""deduplicate a list fo STIX objects to a unique set """Deduplicate a list of STIX objects to a unique set
Reduces a set of STIX objects to unique set by looking Reduces a set of STIX objects to unique set by looking
at 'id' and 'modified' fields - as a unique object version at 'id' and 'modified' fields - as a unique object version
@ -718,9 +715,9 @@ class CompositeDataSource(object):
stix_obj_list (list): list of STIX objects (dicts) stix_obj_list (list): list of STIX objects (dicts)
Returns: Returns:
(list): unique set of the passed list of STIX objects A list with a unique set of the passed list of STIX objects.
"""
"""
unique_objs = {} unique_objs = {}
for obj in stix_obj_list: for obj in stix_obj_list:
@ -729,13 +726,13 @@ class CompositeDataSource(object):
return list(unique_objs.values()) return list(unique_objs.values())
class STIXCommonPropertyFilters(): class STIXCommonPropertyFilters(object):
""" """
""" """
@classmethod @classmethod
def _all(cls, filter_, stix_obj_field): def _all(cls, filter_, stix_obj_field):
"""all filter operations (for filters whose value type can be applied to any operation type)""" """all filter operations (for filters whose value type can be applied to any operation type)"""
if filter_["op"] == '=': if filter_["op"] == "=":
return stix_obj_field == filter_["value"] return stix_obj_field == filter_["value"]
elif filter_["op"] == "!=": elif filter_["op"] == "!=":
return stix_obj_field != filter_["value"] return stix_obj_field != filter_["value"]
@ -791,14 +788,15 @@ class STIXCommonPropertyFilters():
@classmethod @classmethod
def external_references(cls, filter_, stix_obj): def external_references(cls, filter_, stix_obj):
""" """
stix object's can have a list of external references STIX object's can have a list of external references
external_references properties:
external_references.source_name (string)
external_references.description (string)
external_references.url (string)
external_references.hashes (hash, but for filtering purposes, a string)
external_references.external_id (string)
external-reference properties:
external_reference.source_name (string)
external_reference.description (string)
external_reference.url (string)
external_reference.hashes (hash, but for filtering purposes , a string)
external_reference.external_id (string)
""" """
for er in stix_obj["external_references"]: for er in stix_obj["external_references"]:
# grab er property name from filter field # grab er property name from filter field
@ -811,11 +809,12 @@ class STIXCommonPropertyFilters():
@classmethod @classmethod
def granular_markings(cls, filter_, stix_obj): def granular_markings(cls, filter_, stix_obj):
""" """
stix object's can have a list of granular marking references STIX object's can have a list of granular marking references
granular_markings properties:
granular_markings.marking_ref (id)
granular_markings.selectors (string)
granular-marking properties:
granular-marking.marking_ref (id)
granular-marking.selectors (string)
""" """
for gm in stix_obj["granular_markings"]: for gm in stix_obj["granular_markings"]:
# grab gm property name from filter field # grab gm property name from filter field

20
stix2/sources/filesystem.py
View File

@ -12,16 +12,15 @@ TODO: Test everything
import json import json
import os import os
from stix2.sources import DataSink, DataSource, DataStore, make_id
from stix2 import Bundle from stix2 import Bundle
from stix2.sources import DataSink, DataSource, DataStore
class FileSystemStore(DataStore): class FileSystemStore(DataStore):
""" """
""" """
def __init__(self, stix_dir="stix_data", name="FileSystemStore"): def __init__(self, stix_dir="stix_data", name="FileSystemStore"):
self.name = name super(FileSystemStore, self).__init__(name=name)
self.id = make_id()
self.source = FileSystemSource(stix_dir=stix_dir) self.source = FileSystemSource(stix_dir=stix_dir)
self.sink = FileSystemSink(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir)
@ -94,8 +93,11 @@ class FileSystemSource(DataSource):
def all_versions(self, stix_id, _composite_filters=None): def all_versions(self, stix_id, _composite_filters=None):
""" """
NOTE: since FileSystem sources/sinks dont handle mutliple verions of a STIX object, Notes:
this operation is futile. Pass call to get(). (Appoved by G.B.) Since FileSystem sources/sinks don't handle multiple versions
of a STIX object, this operation is futile. Pass call to get().
(Approved by G.B.)
""" """
# query = [ # query = [
@ -139,7 +141,7 @@ class FileSystemSource(DataSource):
if "type" in [filter_["field"] for filter_ in file_filters]: if "type" in [filter_["field"] for filter_ in file_filters]:
for filter_ in file_filters: for filter_ in file_filters:
if filter_["field"] == "type": if filter_["field"] == "type":
if filter_["op"] == '=': if filter_["op"] == "=":
include_paths.append(os.path.join(self.stix_dir, filter_["value"])) include_paths.append(os.path.join(self.stix_dir, filter_["value"]))
elif filter_["op"] == "!=": elif filter_["op"] == "!=":
declude_paths.append(os.path.join(self.stix_dir, filter_["value"])) declude_paths.append(os.path.join(self.stix_dir, filter_["value"]))
@ -167,8 +169,11 @@ class FileSystemSource(DataSource):
# may forgo the loading of STIX content into memory # may forgo the loading of STIX content into memory
if "id" in [filter_["field"] for filter_ in file_filters]: if "id" in [filter_["field"] for filter_ in file_filters]:
for filter_ in file_filters: for filter_ in file_filters:
if filter_["field"] == "id" and filter_["field"] == '=': if filter_["field"] == "id" and filter_["op"] == "=":
id_ = filter_["value"] id_ = filter_["value"]
break
else:
id_ = None
else: else:
id_ = None id_ = None
@ -188,7 +193,6 @@ class FileSystemSource(DataSource):
all_data.extend(self.apply_common_filters([stix_obj], query)) all_data.extend(self.apply_common_filters([stix_obj], query))
all_data = self.deduplicate(all_data) all_data = self.deduplicate(all_data)
return all_data return all_data
def _parse_file_filters(self, query): def _parse_file_filters(self, query):

57
stix2/sources/memory.py
View File

@ -11,9 +11,10 @@ TODO: Test everything.
TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) TODO: Use deduplicate() calls only when memory corpus is dirty (been added to)
can save a lot of time for successive queries can save a lot of time for successive queries
NOTE: Not worrying about STIX versioning. The in memory STIX data at anytime Notes:
will only hold one version of a STIX object. As such, when save() is called, Not worrying about STIX versioning. The in memory STIX data at anytime
the single versions of all the STIX objects are what is written to file. will only hold one version of a STIX object. As such, when save() is called,
the single versions of all the STIX objects are what is written to file.
""" """
@ -21,7 +22,7 @@ import json
import os import os
from stix2 import Bundle from stix2 import Bundle
from stix2.sources import DataSink, DataSource, DataStore, make_id from stix2.sources import DataSink, DataSource, DataStore
from stix2validator import validate_string from stix2validator import validate_string
@ -30,12 +31,13 @@ class MemoryStore(DataStore):
""" """
def __init__(self, stix_data=None, name="MemoryStore"): def __init__(self, stix_data=None, name="MemoryStore"):
""" """
Note: It doesnt make sense to create a MemoryStore by passing Notes:
in existing MemorySource and MemorySink because there could It doesn't make sense to create a MemoryStore by passing
be data concurrency issues. Just as easy to create new MemoryStore. in existing MemorySource and MemorySink because there could
be data concurrency issues. Just as easy to create new MemoryStore.
""" """
self.name = name super(MemoryStore, self).__init__(name=name)
self.id = make_id()
self.data = {} self.data = {}
if stix_data: if stix_data:
@ -46,7 +48,6 @@ class MemoryStore(DataStore):
# make dictionary of the objects for easy lookup # make dictionary of the objects for easy lookup
if r.is_valid: if r.is_valid:
for stix_obj in stix_data["objects"]: for stix_obj in stix_data["objects"]:
self.data[stix_obj["id"]] = stix_obj self.data[stix_obj["id"]] = stix_obj
else: else:
print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator") print("Error: json data passed to MemorySink() was found to not be validated by STIX 2 Validator")
@ -73,16 +74,16 @@ class MemoryStore(DataStore):
class MemorySink(DataSink): class MemorySink(DataSink):
""" """
""" """
def __init__(self, stix_data=None, name="MemorySink", _store=False): def __init__(self, stix_data=None, name="MemorySink", _store=False):
""" """
Args: Args:
stix_data (dictionary OR list): valid STIX 2.0 content in
data (dictionary OR list): valid STIX 2.0 content in bundle or a list bundle or a list.
name (string): optional name tag of the data source name (string): optional name tag of the data source
_store (bool): if the MemorySink is a part of a DataStore, in which case _store (bool): if the MemorySink is a part of a DataStore,
"stix_data" is a direct reference to shared memory with DataSource in which case "stix_data" is a direct reference to
shared memory with DataSource.
""" """
super(MemorySink, self).__init__(name=name) super(MemorySink, self).__init__(name=name)
@ -152,11 +153,12 @@ class MemorySource(DataSource):
def __init__(self, stix_data=None, name="MemorySource", _store=False): def __init__(self, stix_data=None, name="MemorySource", _store=False):
""" """
Args: Args:
stix_data (dictionary OR list): valid STIX 2.0 content in
data (dictionary OR list): valid STIX 2.0 content in bundle or list bundle or list.
name (string): optional name tag of the data source name (string): optional name tag of the data source.
_store (bool): if the MemorySource is a part of a DataStore, in which case _store (bool): if the MemorySource is a part of a DataStore,
"stix_data" is a direct reference to shared memory with DataSink in which case "stix_data" is a direct reference to shared
memory with DataSink.
""" """
super(MemorySource, self).__init__(name=name) super(MemorySource, self).__init__(name=name)
@ -167,7 +169,7 @@ class MemorySource(DataSource):
self.data = {} self.data = {}
if stix_data: if stix_data:
if type(stix_data) == dict: if type(stix_data) == dict:
# stix objects are in a bundle # STIX objects are in a bundle
# verify STIX json data # verify STIX json data
r = validate_string(json.dumps(stix_data)) r = validate_string(json.dumps(stix_data))
# make dictionary of the objects for easy lookup # make dictionary of the objects for easy lookup
@ -179,7 +181,7 @@ class MemorySource(DataSource):
print(r) print(r)
self.data = {} self.data = {}
elif type(stix_data) == list: elif type(stix_data) == list:
# stix objects are in a list # STIX objects are in a list
for stix_obj in stix_data: for stix_obj in stix_data:
r = validate_string(json.dumps(stix_obj)) r = validate_string(json.dumps(stix_obj))
if r.is_valid: if r.is_valid:
@ -219,8 +221,11 @@ class MemorySource(DataSource):
def all_versions(self, stix_id, _composite_filters=None): def all_versions(self, stix_id, _composite_filters=None):
""" """
NOTE: since Memory sources/sinks dont handle mutliple verions of a STIX object, Notes:
this operation is futile. Translate call to get(). (Appoved by G.B.) Since Memory sources/sinks don't handle multiple versions of a
STIX object, this operation is futile. Translate call to get().
(Approved by G.B.)
""" """
# query = [ # query = [
@ -237,9 +242,7 @@ class MemorySource(DataSource):
def query(self, query=None, _composite_filters=None): def query(self, query=None, _composite_filters=None):
""" """
""" """
if query is None: if query is None:
query = [] query = []
@ -250,7 +253,7 @@ class MemorySource(DataSource):
query.extend(_composite_filters) query.extend(_composite_filters)
# deduplicate data before filtering -> Deduplication is not required as Memory only ever holds one version of an object # deduplicate data before filtering -> Deduplication is not required as Memory only ever holds one version of an object
# all_data = self.depuplicate(all_data) # all_data = self.deduplicate(all_data)
# apply STIX common property filters # apply STIX common property filters
all_data = self.apply_common_filters(self.data.values(), query) all_data = self.apply_common_filters(self.data.values(), query)

28
stix2/sources/taxii.py
View File

@ -11,7 +11,6 @@ TODO: Test everything
""" """
import json import json
import uuid
from stix2.sources import DataSink, DataSource, DataStore, make_id from stix2.sources import DataSink, DataSource, DataStore, make_id
@ -27,8 +26,8 @@ class TAXIICollectionStore(DataStore):
Args: Args:
collection (taxii2.Collection): Collection instance collection (taxii2.Collection): Collection instance
"""
"""
self.name = name self.name = name
self.id = make_id() self.id = make_id()
self.source = TAXIICollectionSource(collection) self.source = TAXIICollectionSource(collection)
@ -38,7 +37,6 @@ class TAXIICollectionStore(DataStore):
class TAXIICollectionSink(DataSink): class TAXIICollectionSink(DataSink):
""" """
""" """
def __init__(self, collection, name="TAXIICollectionSink"): def __init__(self, collection, name="TAXIICollectionSink"):
super(TAXIICollectionSink, self).__init__(name=name) super(TAXIICollectionSink, self).__init__(name=name)
@ -51,7 +49,7 @@ class TAXIICollectionSink(DataSink):
@staticmethod @staticmethod
def create_bundle(objects): def create_bundle(objects):
return dict(id="bundle--" + str(uuid.uuid4()), return dict(id="bundle--%s" % make_id(),
objects=objects, objects=objects,
spec_version="2.0", spec_version="2.0",
type="bundle") type="bundle")
@ -137,15 +135,17 @@ class TAXIICollectionSource(DataSource):
return all_data return all_data
def _parse_taxii_filters(self, query): def _parse_taxii_filters(self, query):
"""Parse out TAXII filters that the TAXII server can filter on """Parse out TAXII filters that the TAXII server can filter on.
For instance Notes:
"?match[type]=indicator,sighting" should be in a query dict as follows For instance - "?match[type]=indicator,sighting" should be in a
{ query dict as follows:
"field": "type"
"op": "=", {
"value": "indicator,sighting" "field": "type"
} "op": "=",
"value": "indicator,sighting"
}
Args: Args:
query (list): list of filters to extract which ones are TAXII query (list): list of filters to extract which ones are TAXII
@ -154,8 +154,8 @@ class TAXIICollectionSource(DataSource):
Returns: Returns:
params (dict): dict of the TAXII filters but in format required params (dict): dict of the TAXII filters but in format required
for 'requests.get()'. for 'requests.get()'.
"""
"""
params = {} params = {}
for filter_ in query: for filter_ in query:
@ -163,6 +163,6 @@ class TAXIICollectionSource(DataSource):
if filter_["field"] == "added_after": if filter_["field"] == "added_after":
params[filter_["field"]] = filter_["value"] params[filter_["field"]] = filter_["value"]
else: else:
taxii_field = "match[" + filter_["field"] + ']' taxii_field = "match[%s]" % filter_["field"]
params[taxii_field] = filter_["value"] params[taxii_field] = filter_["value"]
return params return params

2
stix2/test/test_data_sources.py
View File

@ -74,7 +74,7 @@ def test_parse_taxii_filters():
assert taxii_filters == expected_params assert taxii_filters == expected_params
@pytest.skip @pytest.mark.skip(reason="test_add_get_remove_filter() - Determine what are we testing.")
def test_add_get_remove_filter(): def test_add_get_remove_filter():
# First 3 filters are valid, remaining fields are erroneous in some way # First 3 filters are valid, remaining fields are erroneous in some way