WIP - just at stash point

stix2/datastore/__init__.py

@@ -16,7 +16,7 @@ import uuid
 
 from six import with_metaclass
 
-from stix2.datastore.filters import Filter
+from stix2.datastore.filters import Filter, FilterSet
 from stix2.utils import deduplicate
 
 
@@ -220,13 +220,13 @@ class DataSource(with_metaclass(ABCMeta)):
 
     Attributes:
         id (str): A unique UUIDv4 to identify this DataSource.
-        filters (set): A collection of filters attached to this DataSource.
+        filters (FilterSet): A collection of filters attached to this DataSource.
 
     """
     def __init__(self):
         super(DataSource, self).__init__()
         self.id = make_id()
-        self.filters = set()
+        self.filters = FilterSet()
 
     @abstractmethod
     def get(self, stix_id):
@@ -420,7 +420,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): the id of the STIX object to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is attached
                 to another parent CompositeDataSource), not user supplied.
 
@@ -432,11 +432,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
+        all_filters = FilterSet()
-        all_filters.update(self.filters)
+
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # for every configured Data Source, call its retrieve handler
         for ds in self.data_sources:
@@ -466,7 +467,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             stix_id (str): id of the STIX objects to retrieve.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -478,12 +479,12 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         all_data = []
-        all_filters = set()
+        all_filters = FilterSet()
 
-        all_filters.update(self.filters)
+        all_filters.add(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # retrieve STIX objects from all configured data sources
         for ds in self.data_sources:
@@ -505,7 +506,7 @@ class CompositeDataSource(DataSource):
 
         Args:
             query (list): list of filters to search on.
-            _composite_filters (list): a list of filters passed from a
+            _composite_filters (FilterSet): a collection of filters passed from a
                 CompositeDataSource (i.e. if this CompositeDataSource is
                 attached to a parent CompositeDataSource), not user supplied.
 
@@ -517,17 +518,17 @@ class CompositeDataSource(DataSource):
             raise AttributeError('CompositeDataSource has no data sources')
 
         if not query:
-            # don't mess with the query (i.e. convert to a set, as that's done
+            # don't mess with the query (i.e. deduplicate, as that's done
             # within the specific DataSources that are called)
             query = []
 
         all_data = []
+        all_filters = FilterSet()
 
-        all_filters = set()
+        all_filters.add(self.filters)
-        all_filters.update(self.filters)
 
         if _composite_filters:
-            all_filters.update(_composite_filters)
+            all_filters.add(_composite_filters)
 
         # federate query to all attached data sources,
         # pass composite filters to id

stix2/datastore/filesystem.py

@@ -8,7 +8,7 @@ import os
 
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate, get_class_hierarchy_names
 
 
@@ -165,7 +165,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -195,7 +195,7 @@ class FileSystemSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -217,7 +217,7 @@ class FileSystemSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -231,20 +231,13 @@ class FileSystemSource(DataSource):
 
         all_data = []
 
-        if query is None:
+        query = FilterSet(query)
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # extract any filters that are for "type" or "id" , as we can then do
         # filtering before reading in the STIX objects. A STIX 'type' filter
@@ -343,8 +336,8 @@ class FileSystemSource(DataSource):
         search space of a FileSystemStore (or FileSystemSink).
 
         """
-        file_filters = set()
+        file_filters = []
         for filter_ in query:
             if filter_.property == "id" or filter_.property == "type":
-                file_filters.add(filter_)
+                file_filters.append(filter_)
         return file_filters

stix2/datastore/filters.py

@@ -17,6 +17,23 @@ except NameError:
     pass
 
 
+def deduplicate_filters(filters):
+    """utility for deduplicating list of filters, this
+    is used when 'set()' cannot be used as one of the
+    filter values is a dict (or non-hashable type)
+
+    Args:
+        filters (list): a list of filters
+
+    Returns: list of unique filters
+    """
+    unique_filters = []
+    for filter_ in filters:
+        if filter_ not in unique_filters:
+            unique_filters.append(filter_)
+    return unique_filters
+
+
 def _check_filter_components(prop, op, value):
     """Check that filter meets minimum validity.
 
@@ -168,3 +185,39 @@ def _check_filter(filter_, stix_obj):
     else:
         # Check if property matches
         return filter_._check_property(stix_obj[prop])
+
+
+class FilterSet(object):
+    """ """
+
+    def __init__(self, filters=None):
+        """ """
+        self._filters = []
+        if filters:
+            self.add(filters)
+
+    def __iter__(self):
+        """ """
+        for f in self._filters:
+            yield f
+
+    def add(self, filters):
+        """ """
+        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+            filters = [filters]
+
+        for f in filters:
+            if f not in self._filters:
+                self._filters.append(f)
+
+        return
+
+    def remove(self, filters):
+        """ """
+        if not isinstance(filters, FilterSet) and not isinstance(filters, list):
+            filters = [filters]
+
+        for f in filters:
+            self._filters.remove(f)
+
+        return

stix2/datastore/memory.py

@@ -18,7 +18,7 @@ import os
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 
 
 def _add(store, stix_data=None, version=None):
@@ -197,7 +197,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -236,7 +236,7 @@ class MemorySource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX 2 object to retrieve.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -258,7 +258,7 @@ class MemorySource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
 
         Returns:
@@ -269,19 +269,15 @@ class MemorySource(DataSource):
 
         """
         if query is None:
-            query = set()
+            query = FilterSet()
         else:
-            if not isinstance(query, list):
+            query = FilterSet(query)
-                # make sure don't make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # Apply STIX common property filters.
         all_data = list(apply_common_filters(self._data.values(), query))

stix2/datastore/taxii.py

@@ -6,7 +6,7 @@ from requests.exceptions import HTTPError
 from stix2.base import _STIXBase
 from stix2.core import Bundle, parse
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
-from stix2.datastore.filters import Filter, apply_common_filters
+from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
 from stix2.utils import deduplicate
 
 TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
@@ -120,7 +120,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX object to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -132,11 +132,12 @@ class TAXIICollectionSource(DataSource):
 
         """
         # combine all query filters
-        query = set()
+        query = FilterSet()
+
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # dont extract TAXII filters from query (to send to TAXII endpoint)
         # as directly retrieveing a STIX object by ID
@@ -164,7 +165,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             stix_id (str): The STIX ID of the STIX objects to be retrieved.
-            _composite_filters (set): set of filters passed from the parent
+            _composite_filters (FilterSet): collection of filters passed from the parent
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -198,7 +199,7 @@ class TAXIICollectionSource(DataSource):
 
         Args:
             query (list): list of filters to search on
-            _composite_filters (set): set of filters passed from the
+            _composite_filters (FilterSet): collection of filters passed from the
                 CompositeDataSource, not user supplied
             version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
                 None, use latest version.
@@ -209,20 +210,13 @@ class TAXIICollectionSource(DataSource):
                 parsed into python STIX objects and then returned.
 
         """
-        if query is None:
+        query = FilterSet(query)
-            query = set()
-        else:
-            if not isinstance(query, list):
-                # make sure dont make set from a Filter object,
-                # need to make a set from a list of Filter objects (even if just one Filter)
-                query = [query]
-            query = set(query)
 
         # combine all query filters
         if self.filters:
-            query.update(self.filters)
+            query.add(self.filters)
         if _composite_filters:
-            query.update(_composite_filters)
+            query.add(_composite_filters)
 
         # parse taxii query params (that can be applied remotely)
         taxii_filters = self._parse_taxii_filters(query)
@@ -268,17 +262,16 @@ class TAXIICollectionSource(DataSource):
 
 
         Args:
-            query (set): set of filters to extract which ones are TAXII
+            query (list): list of filters to extract which ones are TAXII
                 specific.
 
-        Returns:
+        Returns: a list of the TAXII filters
-            taxii_filters (set): set of the TAXII filters
 
         """
-        taxii_filters = set()
+        taxii_filters = []
 
         for filter_ in query:
             if filter_.property in TAXII_FILTERS:
-                taxii_filters.add(filter_)
+                taxii_filters.append(filter_)
 
         return taxii_filters