diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py index 78f7555..c43f309 100644 --- a/stix2/datastore/__init__.py +++ b/stix2/datastore/__init__.py @@ -16,7 +16,7 @@ import uuid from six import with_metaclass -from stix2.datastore.filters import Filter +from stix2.datastore.filters import Filter, FilterSet from stix2.utils import deduplicate @@ -220,13 +220,13 @@ class DataSource(with_metaclass(ABCMeta)): Attributes: id (str): A unique UUIDv4 to identify this DataSource. - filters (set): A collection of filters attached to this DataSource. + filters (FilterSet): A collection of filters attached to this DataSource. """ def __init__(self): super(DataSource, self).__init__() self.id = make_id() - self.filters = set() + self.filters = FilterSet() @abstractmethod def get(self, stix_id): @@ -420,7 +420,7 @@ class CompositeDataSource(DataSource): Args: stix_id (str): the id of the STIX object to retrieve. - _composite_filters (list): a list of filters passed from a + _composite_filters (FilterSet): a collection of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to another parent CompositeDataSource), not user supplied. @@ -432,11 +432,12 @@ class CompositeDataSource(DataSource): raise AttributeError('CompositeDataSource has no data sources') all_data = [] - all_filters = set() - all_filters.update(self.filters) + all_filters = FilterSet() + + all_filters.add(self.filters) if _composite_filters: - all_filters.update(_composite_filters) + all_filters.add(_composite_filters) # for every configured Data Source, call its retrieve handler for ds in self.data_sources: @@ -466,7 +467,7 @@ class CompositeDataSource(DataSource): Args: stix_id (str): id of the STIX objects to retrieve. - _composite_filters (list): a list of filters passed from a + _composite_filters (FilterSet): a collection of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied. @@ -478,12 +479,12 @@ class CompositeDataSource(DataSource): raise AttributeError('CompositeDataSource has no data sources') all_data = [] - all_filters = set() + all_filters = FilterSet() - all_filters.update(self.filters) + all_filters.add(self.filters) if _composite_filters: - all_filters.update(_composite_filters) + all_filters.add(_composite_filters) # retrieve STIX objects from all configured data sources for ds in self.data_sources: @@ -505,7 +506,7 @@ class CompositeDataSource(DataSource): Args: query (list): list of filters to search on. - _composite_filters (list): a list of filters passed from a + _composite_filters (FilterSet): a collection of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached to a parent CompositeDataSource), not user supplied. @@ -517,17 +518,17 @@ class CompositeDataSource(DataSource): raise AttributeError('CompositeDataSource has no data sources') if not query: - # don't mess with the query (i.e. convert to a set, as that's done + # don't mess with the query (i.e. deduplicate, as that's done # within the specific DataSources that are called) query = [] all_data = [] + all_filters = FilterSet() - all_filters = set() - all_filters.update(self.filters) + all_filters.add(self.filters) if _composite_filters: - all_filters.update(_composite_filters) + all_filters.add(_composite_filters) # federate query to all attached data sources, # pass composite filters to id diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index a6f31cf..c13b02c 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -8,7 +8,7 @@ import os from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin -from stix2.datastore.filters import Filter, apply_common_filters +from stix2.datastore.filters import Filter, FilterSet, apply_common_filters from stix2.utils import deduplicate, get_class_hierarchy_names @@ -165,7 +165,7 @@ class FileSystemSource(DataSource): Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -195,7 +195,7 @@ class FileSystemSource(DataSource): Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -217,7 +217,7 @@ class FileSystemSource(DataSource): Args: query (list): list of filters to search on - _composite_filters (set): set of filters passed from the + _composite_filters (FilterSet): collection of filters passed from the CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -231,20 +231,13 @@ class FileSystemSource(DataSource): all_data = [] - if query is None: - query = set() - else: - if not isinstance(query, list): - # make sure dont make set from a Filter object, - # need to make a set from a list of Filter objects (even if just one Filter) - query = [query] - query = set(query) + query = FilterSet(query) # combine all query filters if self.filters: - query.update(self.filters) + query.add(self.filters) if _composite_filters: - query.update(_composite_filters) + query.add(_composite_filters) # extract any filters that are for "type" or "id" , as we can then do # filtering before reading in the STIX objects. A STIX 'type' filter @@ -343,8 +336,8 @@ class FileSystemSource(DataSource): search space of a FileSystemStore (or FileSystemSink). """ - file_filters = set() + file_filters = [] for filter_ in query: if filter_.property == "id" or filter_.property == "type": - file_filters.add(filter_) + file_filters.append(filter_) return file_filters diff --git a/stix2/datastore/filters.py b/stix2/datastore/filters.py index 9065b61..0946694 100644 --- a/stix2/datastore/filters.py +++ b/stix2/datastore/filters.py @@ -17,6 +17,23 @@ except NameError: pass +def deduplicate_filters(filters): + """utility for deduplicating list of filters, this + is used when 'set()' cannot be used as one of the + filter values is a dict (or non-hashable type) + + Args: + filters (list): a list of filters + + Returns: list of unique filters + """ + unique_filters = [] + for filter_ in filters: + if filter_ not in unique_filters: + unique_filters.append(filter_) + return unique_filters + + def _check_filter_components(prop, op, value): """Check that filter meets minimum validity. @@ -168,3 +185,39 @@ def _check_filter(filter_, stix_obj): else: # Check if property matches return filter_._check_property(stix_obj[prop]) + + +class FilterSet(object): + """ """ + + def __init__(self, filters=None): + """ """ + self._filters = [] + if filters: + self.add(filters) + + def __iter__(self): + """ """ + for f in self._filters: + yield f + + def add(self, filters): + """ """ + if not isinstance(filters, FilterSet) and not isinstance(filters, list): + filters = [filters] + + for f in filters: + if f not in self._filters: + self._filters.append(f) + + return + + def remove(self, filters): + """ """ + if not isinstance(filters, FilterSet) and not isinstance(filters, list): + filters = [filters] + + for f in filters: + self._filters.remove(f) + + return diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index e057271..e6f0fd2 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -18,7 +18,7 @@ import os from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin -from stix2.datastore.filters import Filter, apply_common_filters +from stix2.datastore.filters import Filter, FilterSet, apply_common_filters def _add(store, stix_data=None, version=None): @@ -197,7 +197,7 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: @@ -236,7 +236,7 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: @@ -258,7 +258,7 @@ class MemorySource(DataSource): Args: query (list): list of filters to search on - _composite_filters (set): set of filters passed from the + _composite_filters (FilterSet): collection of filters passed from the CompositeDataSource, not user supplied Returns: @@ -269,19 +269,15 @@ class MemorySource(DataSource): """ if query is None: - query = set() + query = FilterSet() else: - if not isinstance(query, list): - # make sure don't make set from a Filter object, - # need to make a set from a list of Filter objects (even if just one Filter) - query = [query] - query = set(query) + query = FilterSet(query) # combine all query filters if self.filters: - query.update(self.filters) + query.add(self.filters) if _composite_filters: - query.update(_composite_filters) + query.add(_composite_filters) # Apply STIX common property filters. all_data = list(apply_common_filters(self._data.values(), query)) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 0a58763..faa2669 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -6,7 +6,7 @@ from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin -from stix2.datastore.filters import Filter, apply_common_filters +from stix2.datastore.filters import Filter, FilterSet, apply_common_filters from stix2.utils import deduplicate TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -120,7 +120,7 @@ class TAXIICollectionSource(DataSource): Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -132,11 +132,12 @@ class TAXIICollectionSource(DataSource): """ # combine all query filters - query = set() + query = FilterSet() + if self.filters: - query.update(self.filters) + query.add(self.filters) if _composite_filters: - query.update(_composite_filters) + query.add(_composite_filters) # dont extract TAXII filters from query (to send to TAXII endpoint) # as directly retrieveing a STIX object by ID @@ -164,7 +165,7 @@ class TAXIICollectionSource(DataSource): Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - _composite_filters (set): set of filters passed from the parent + _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -198,7 +199,7 @@ class TAXIICollectionSource(DataSource): Args: query (list): list of filters to search on - _composite_filters (set): set of filters passed from the + _composite_filters (FilterSet): collection of filters passed from the CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -209,20 +210,13 @@ class TAXIICollectionSource(DataSource): parsed into python STIX objects and then returned. """ - if query is None: - query = set() - else: - if not isinstance(query, list): - # make sure dont make set from a Filter object, - # need to make a set from a list of Filter objects (even if just one Filter) - query = [query] - query = set(query) + query = FilterSet(query) # combine all query filters if self.filters: - query.update(self.filters) + query.add(self.filters) if _composite_filters: - query.update(_composite_filters) + query.add(_composite_filters) # parse taxii query params (that can be applied remotely) taxii_filters = self._parse_taxii_filters(query) @@ -268,17 +262,16 @@ class TAXIICollectionSource(DataSource): Args: - query (set): set of filters to extract which ones are TAXII + query (list): list of filters to extract which ones are TAXII specific. - Returns: - taxii_filters (set): set of the TAXII filters + Returns: a list of the TAXII filters """ - taxii_filters = set() + taxii_filters = [] for filter_ in query: if filter_.property in TAXII_FILTERS: - taxii_filters.add(filter_) + taxii_filters.append(filter_) return taxii_filters