MISP
/
cti-python-stix2
mirror of https://github.com/MISP/cti-python-stix2
2
0
Fork 0
Code Issues Releases Wiki Activity

WIP - just at stash point

stix2.0
= 2018-04-11 13:36:52 -04:00
parent e92db2417a
commit 27647091a5
5 changed files with 102 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
stix2/datastore/__init__.py
View File

@ -16,7 +16,7 @@ import uuid
from six import with_metaclass
from stix2.datastore.filters import Filter
from stix2.datastore.filters import Filter, FilterSet
from stix2.utils import deduplicate
@ -220,13 +220,13 @@ class DataSource(with_metaclass(ABCMeta)):
Attributes:
id (str): A unique UUIDv4 to identify this DataSource.
filters (set): A collection of filters attached to this DataSource.
filters (FilterSet): A collection of filters attached to this DataSource.
"""
def __init__(self):
super(DataSource, self).__init__()
self.id = make_id()
self.filters = set()
self.filters = FilterSet()
@abstractmethod
def get(self, stix_id):
@ -420,7 +420,7 @@ class CompositeDataSource(DataSource):
Args:
stix_id (str): the id of the STIX object to retrieve.
_composite_filters (list): a list of filters passed from a
_composite_filters (FilterSet): a collection of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is attached
to another parent CompositeDataSource), not user supplied.
@ -432,11 +432,12 @@ class CompositeDataSource(DataSource):
raise AttributeError('CompositeDataSource has no data sources')
all_data = []
all_filters = set()
all_filters.update(self.filters)
all_filters = FilterSet()
all_filters.add(self.filters)
if _composite_filters:
all_filters.update(_composite_filters)
all_filters.add(_composite_filters)
# for every configured Data Source, call its retrieve handler
for ds in self.data_sources:
@ -466,7 +467,7 @@ class CompositeDataSource(DataSource):
Args:
stix_id (str): id of the STIX objects to retrieve.
_composite_filters (list): a list of filters passed from a
_composite_filters (FilterSet): a collection of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is
attached to a parent CompositeDataSource), not user supplied.
@ -478,12 +479,12 @@ class CompositeDataSource(DataSource):
raise AttributeError('CompositeDataSource has no data sources')
all_data = []
all_filters = set()
all_filters = FilterSet()
all_filters.update(self.filters)
all_filters.add(self.filters)
if _composite_filters:
all_filters.update(_composite_filters)
all_filters.add(_composite_filters)
# retrieve STIX objects from all configured data sources
for ds in self.data_sources:
@ -505,7 +506,7 @@ class CompositeDataSource(DataSource):
Args:
query (list): list of filters to search on.
_composite_filters (list): a list of filters passed from a
_composite_filters (FilterSet): a collection of filters passed from a
CompositeDataSource (i.e. if this CompositeDataSource is
attached to a parent CompositeDataSource), not user supplied.
@ -517,17 +518,17 @@ class CompositeDataSource(DataSource):
raise AttributeError('CompositeDataSource has no data sources')
if not query:
# don't mess with the query (i.e. convert to a set, as that's done
# don't mess with the query (i.e. deduplicate, as that's done
# within the specific DataSources that are called)
query = []
all_data = []
all_filters = FilterSet()
all_filters = set()
all_filters.update(self.filters)
all_filters.add(self.filters)
if _composite_filters:
all_filters.update(_composite_filters)
all_filters.add(_composite_filters)
# federate query to all attached data sources,
# pass composite filters to id

25
stix2/datastore/filesystem.py
View File

@ -8,7 +8,7 @@ import os
from stix2.core import Bundle, parse
from stix2.datastore import DataSink, DataSource, DataStoreMixin
from stix2.datastore.filters import Filter, apply_common_filters
from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
from stix2.utils import deduplicate, get_class_hierarchy_names
@ -165,7 +165,7 @@ class FileSystemSource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX object to be retrieved.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -195,7 +195,7 @@ class FileSystemSource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX objects to be retrieved.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -217,7 +217,7 @@ class FileSystemSource(DataSource):
Args:
query (list): list of filters to search on
_composite_filters (set): set of filters passed from the
_composite_filters (FilterSet): collection of filters passed from the
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -231,20 +231,13 @@ class FileSystemSource(DataSource):
all_data = []
if query is None:
query = set()
else:
if not isinstance(query, list):
# make sure dont make set from a Filter object,
# need to make a set from a list of Filter objects (even if just one Filter)
query = [query]
query = set(query)
query = FilterSet(query)
# combine all query filters
if self.filters:
query.update(self.filters)
query.add(self.filters)
if _composite_filters:
query.update(_composite_filters)
query.add(_composite_filters)
# extract any filters that are for "type" or "id" , as we can then do
# filtering before reading in the STIX objects. A STIX 'type' filter
@ -343,8 +336,8 @@ class FileSystemSource(DataSource):
search space of a FileSystemStore (or FileSystemSink).
"""
file_filters = set()
file_filters = []
for filter_ in query:
if filter_.property == "id" or filter_.property == "type":
file_filters.add(filter_)
file_filters.append(filter_)
return file_filters

53
stix2/datastore/filters.py
View File

@ -17,6 +17,23 @@ except NameError:
pass
def deduplicate_filters(filters):
"""utility for deduplicating list of filters, this
is used when 'set()' cannot be used as one of the
filter values is a dict (or non-hashable type)
Args:
filters (list): a list of filters
Returns: list of unique filters
"""
unique_filters = []
for filter_ in filters:
if filter_ not in unique_filters:
unique_filters.append(filter_)
return unique_filters
def _check_filter_components(prop, op, value):
"""Check that filter meets minimum validity.
@ -168,3 +185,39 @@ def _check_filter(filter_, stix_obj):
else:
# Check if property matches
return filter_._check_property(stix_obj[prop])
class FilterSet(object):
""" """
def __init__(self, filters=None):
""" """
self._filters = []
if filters:
self.add(filters)
def __iter__(self):
""" """
for f in self._filters:
yield f
def add(self, filters):
""" """
if not isinstance(filters, FilterSet) and not isinstance(filters, list):
filters = [filters]
for f in filters:
if f not in self._filters:
self._filters.append(f)
return
def remove(self, filters):
""" """
if not isinstance(filters, FilterSet) and not isinstance(filters, list):
filters = [filters]
for f in filters:
self._filters.remove(f)
return

20
stix2/datastore/memory.py
View File

@ -18,7 +18,7 @@ import os
from stix2.base import _STIXBase
from stix2.core import Bundle, parse
from stix2.datastore import DataSink, DataSource, DataStoreMixin
from stix2.datastore.filters import Filter, apply_common_filters
from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
def _add(store, stix_data=None, version=None):
@ -197,7 +197,7 @@ class MemorySource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX object to be retrieved.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
@ -236,7 +236,7 @@ class MemorySource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX 2 object to retrieve.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
@ -258,7 +258,7 @@ class MemorySource(DataSource):
Args:
query (list): list of filters to search on
_composite_filters (set): set of filters passed from the
_composite_filters (FilterSet): collection of filters passed from the
CompositeDataSource, not user supplied
Returns:
@ -269,19 +269,15 @@ class MemorySource(DataSource):
"""
if query is None:
query = set()
query = FilterSet()
else:
if not isinstance(query, list):
# make sure don't make set from a Filter object,
# need to make a set from a list of Filter objects (even if just one Filter)
query = [query]
query = set(query)
query = FilterSet(query)
# combine all query filters
if self.filters:
query.update(self.filters)
query.add(self.filters)
if _composite_filters:
query.update(_composite_filters)
query.add(_composite_filters)
# Apply STIX common property filters.
all_data = list(apply_common_filters(self._data.values(), query))

37
stix2/datastore/taxii.py
View File

@ -6,7 +6,7 @@ from requests.exceptions import HTTPError
from stix2.base import _STIXBase
from stix2.core import Bundle, parse
from stix2.datastore import DataSink, DataSource, DataStoreMixin
from stix2.datastore.filters import Filter, apply_common_filters
from stix2.datastore.filters import Filter, FilterSet, apply_common_filters
from stix2.utils import deduplicate
TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
@ -120,7 +120,7 @@ class TAXIICollectionSource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX object to be retrieved.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -132,11 +132,12 @@ class TAXIICollectionSource(DataSource):
"""
# combine all query filters
query = set()
query = FilterSet()
if self.filters:
query.update(self.filters)
query.add(self.filters)
if _composite_filters:
query.update(_composite_filters)
query.add(_composite_filters)
# dont extract TAXII filters from query (to send to TAXII endpoint)
# as directly retrieveing a STIX object by ID
@ -164,7 +165,7 @@ class TAXIICollectionSource(DataSource):
Args:
stix_id (str): The STIX ID of the STIX objects to be retrieved.
_composite_filters (set): set of filters passed from the parent
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -198,7 +199,7 @@ class TAXIICollectionSource(DataSource):
Args:
query (list): list of filters to search on
_composite_filters (set): set of filters passed from the
_composite_filters (FilterSet): collection of filters passed from the
CompositeDataSource, not user supplied
version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If
None, use latest version.
@ -209,20 +210,13 @@ class TAXIICollectionSource(DataSource):
parsed into python STIX objects and then returned.
"""
if query is None:
query = set()
else:
if not isinstance(query, list):
# make sure dont make set from a Filter object,
# need to make a set from a list of Filter objects (even if just one Filter)
query = [query]
query = set(query)
query = FilterSet(query)
# combine all query filters
if self.filters:
query.update(self.filters)
query.add(self.filters)
if _composite_filters:
query.update(_composite_filters)
query.add(_composite_filters)
# parse taxii query params (that can be applied remotely)
taxii_filters = self._parse_taxii_filters(query)
@ -268,17 +262,16 @@ class TAXIICollectionSource(DataSource):
Args:
query (set): set of filters to extract which ones are TAXII
query (list): list of filters to extract which ones are TAXII
specific.
Returns:
taxii_filters (set): set of the TAXII filters
Returns: a list of the TAXII filters
"""
taxii_filters = set()
taxii_filters = []
for filter_ in query:
if filter_.property in TAXII_FILTERS:
taxii_filters.add(filter_)
taxii_filters.append(filter_)
return taxii_filters