commit
458b32d124
|
@ -16,55 +16,18 @@ Notes:
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import collections
|
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
from six import iteritems
|
from six import iteritems
|
||||||
|
|
||||||
|
from stix2.sources.filters import (FILTER_OPS, FILTER_VALUE_TYPES,
|
||||||
class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])):
|
STIX_COMMON_FIELDS, STIX_COMMON_FILTERS_MAP)
|
||||||
__slots__ = ()
|
|
||||||
|
|
||||||
def __new__(cls, field, op, value):
|
|
||||||
# If value is a list, convert it to a tuple so it is hashable.
|
|
||||||
if isinstance(value, list):
|
|
||||||
value = tuple(value)
|
|
||||||
self = super(Filter, cls).__new__(cls, field, op, value)
|
|
||||||
return self
|
|
||||||
|
|
||||||
|
|
||||||
def make_id():
|
def make_id():
|
||||||
return str(uuid.uuid4())
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
|
||||||
# Currently, only STIX 2.0 common SDO fields (that are not complex objects)
|
|
||||||
# are supported for filtering on
|
|
||||||
STIX_COMMON_FIELDS = [
|
|
||||||
"created",
|
|
||||||
"created_by_ref",
|
|
||||||
"external_references.source_name",
|
|
||||||
"external_references.description",
|
|
||||||
"external_references.url",
|
|
||||||
"external_references.hashes",
|
|
||||||
"external_references.external_id",
|
|
||||||
"granular_markings.marking_ref",
|
|
||||||
"granular_markings.selectors",
|
|
||||||
"id",
|
|
||||||
"labels",
|
|
||||||
"modified",
|
|
||||||
"object_marking_refs",
|
|
||||||
"revoked",
|
|
||||||
"type",
|
|
||||||
"granular_markings"
|
|
||||||
]
|
|
||||||
|
|
||||||
# Supported filter operations
|
|
||||||
FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
|
|
||||||
|
|
||||||
# Supported filter value types
|
|
||||||
FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
|
|
||||||
|
|
||||||
|
|
||||||
class DataStore(object):
|
class DataStore(object):
|
||||||
"""
|
"""
|
||||||
An implementer will create a concrete subclass from
|
An implementer will create a concrete subclass from
|
||||||
|
@ -290,34 +253,31 @@ class DataSource(object):
|
||||||
for stix_obj in stix_objs:
|
for stix_obj in stix_objs:
|
||||||
clean = True
|
clean = True
|
||||||
for filter_ in query:
|
for filter_ in query:
|
||||||
try:
|
# skip filter as filter was identified (when added) as
|
||||||
# skip filter as filter was identified (when added) as
|
# not a common filter
|
||||||
# not a common filter
|
if filter_.field not in STIX_COMMON_FIELDS:
|
||||||
if filter_.field not in STIX_COMMON_FIELDS:
|
raise ValueError("Error, field: {0} is not supported for filtering on.".format(filter_.field))
|
||||||
raise Exception("Error, field: {0} is not supported for filtering on.".format(filter_.field))
|
|
||||||
|
|
||||||
# For properties like granular_markings and external_references
|
# For properties like granular_markings and external_references
|
||||||
# need to break the first property from the string.
|
# need to break the first property from the string.
|
||||||
if "." in filter_.field:
|
if "." in filter_.field:
|
||||||
field = filter_.field.split(".")[0]
|
field = filter_.field.split(".")[0]
|
||||||
else:
|
else:
|
||||||
field = filter_.field
|
field = filter_.field
|
||||||
|
|
||||||
# check filter "field" is in STIX object - if cant be
|
# check filter "field" is in STIX object - if cant be
|
||||||
# applied due to STIX object, STIX object is discarded
|
# applied due to STIX object, STIX object is discarded
|
||||||
# (i.e. did not make it through the filter)
|
# (i.e. did not make it through the filter)
|
||||||
if field not in stix_obj.keys():
|
if field not in stix_obj.keys():
|
||||||
clean = False
|
clean = False
|
||||||
break
|
break
|
||||||
|
|
||||||
match = getattr(STIXCommonPropertyFilters, field)(filter_, stix_obj)
|
match = STIX_COMMON_FILTERS_MAP[filter_.field.split('.')[0]](filter_, stix_obj)
|
||||||
if not match:
|
if not match:
|
||||||
clean = False
|
clean = False
|
||||||
break
|
break
|
||||||
elif match == -1:
|
elif match == -1:
|
||||||
raise Exception("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field))
|
raise ValueError("Error, filter operator: {0} not supported for specified field: {1}".format(filter_.op, filter_.field))
|
||||||
except Exception as e:
|
|
||||||
raise ValueError(e)
|
|
||||||
|
|
||||||
# if object unmarked after all filters, add it
|
# if object unmarked after all filters, add it
|
||||||
if clean:
|
if clean:
|
||||||
|
@ -532,140 +492,3 @@ class CompositeDataSource(DataSource):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return self.data_sources.values()
|
return self.data_sources.values()
|
||||||
|
|
||||||
|
|
||||||
class STIXCommonPropertyFilters(object):
|
|
||||||
"""
|
|
||||||
"""
|
|
||||||
@classmethod
|
|
||||||
def _all(cls, filter_, stix_obj_field):
|
|
||||||
"""all filter operations (for filters whose value type can be applied to any operation type)"""
|
|
||||||
if filter_.op == "=":
|
|
||||||
return stix_obj_field == filter_.value
|
|
||||||
elif filter_.op == "!=":
|
|
||||||
return stix_obj_field != filter_.value
|
|
||||||
elif filter_.op == "in":
|
|
||||||
return stix_obj_field in filter_.value
|
|
||||||
elif filter_.op == ">":
|
|
||||||
return stix_obj_field > filter_.value
|
|
||||||
elif filter_.op == "<":
|
|
||||||
return stix_obj_field < filter_.value
|
|
||||||
elif filter_.op == ">=":
|
|
||||||
return stix_obj_field >= filter_.value
|
|
||||||
elif filter_.op == "<=":
|
|
||||||
return stix_obj_field <= filter_.value
|
|
||||||
else:
|
|
||||||
return -1
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _id(cls, filter_, stix_obj_id):
|
|
||||||
"""base filter types"""
|
|
||||||
if filter_.op == "=":
|
|
||||||
return stix_obj_id == filter_.value
|
|
||||||
elif filter_.op == "!=":
|
|
||||||
return stix_obj_id != filter_.value
|
|
||||||
else:
|
|
||||||
return -1
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _boolean(cls, filter_, stix_obj_field):
|
|
||||||
if filter_.op == "=":
|
|
||||||
return stix_obj_field == filter_.value
|
|
||||||
elif filter_.op == "!=":
|
|
||||||
return stix_obj_field != filter_.value
|
|
||||||
else:
|
|
||||||
return -1
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _string(cls, filter_, stix_obj_field):
|
|
||||||
return cls._all(filter_, stix_obj_field)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _timestamp(cls, filter_, stix_obj_timestamp):
|
|
||||||
return cls._all(filter_, stix_obj_timestamp)
|
|
||||||
|
|
||||||
# STIX 2.0 Common Property filters
|
|
||||||
@classmethod
|
|
||||||
def created(cls, filter_, stix_obj):
|
|
||||||
return cls._timestamp(filter_, stix_obj["created"])
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def created_by_ref(cls, filter_, stix_obj):
|
|
||||||
return cls._id(filter_, stix_obj["created_by_ref"])
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def external_references(cls, filter_, stix_obj):
|
|
||||||
"""
|
|
||||||
STIX object's can have a list of external references
|
|
||||||
|
|
||||||
external_references properties:
|
|
||||||
external_references.source_name (string)
|
|
||||||
external_references.description (string)
|
|
||||||
external_references.url (string)
|
|
||||||
external_references.hashes (hash, but for filtering purposes, a string)
|
|
||||||
external_references.external_id (string)
|
|
||||||
|
|
||||||
"""
|
|
||||||
for er in stix_obj["external_references"]:
|
|
||||||
# grab er property name from filter field
|
|
||||||
filter_field = filter_.field.split(".")[1]
|
|
||||||
r = cls._string(filter_, er[filter_field])
|
|
||||||
if r:
|
|
||||||
return r
|
|
||||||
return False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def granular_markings(cls, filter_, stix_obj):
|
|
||||||
"""
|
|
||||||
STIX object's can have a list of granular marking references
|
|
||||||
|
|
||||||
granular_markings properties:
|
|
||||||
granular_markings.marking_ref (id)
|
|
||||||
granular_markings.selectors (string)
|
|
||||||
|
|
||||||
"""
|
|
||||||
for gm in stix_obj["granular_markings"]:
|
|
||||||
# grab gm property name from filter field
|
|
||||||
filter_field = filter_.field.split(".")[1]
|
|
||||||
|
|
||||||
if filter_field == "marking_ref":
|
|
||||||
return cls._id(filter_, gm[filter_field])
|
|
||||||
|
|
||||||
elif filter_field == "selectors":
|
|
||||||
for selector in gm[filter_field]:
|
|
||||||
r = cls._string(filter_, selector)
|
|
||||||
if r:
|
|
||||||
return r
|
|
||||||
return False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def id(cls, filter_, stix_obj):
|
|
||||||
return cls._id(filter_, stix_obj["id"])
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def labels(cls, filter_, stix_obj):
|
|
||||||
for label in stix_obj["labels"]:
|
|
||||||
r = cls._string(filter_, label)
|
|
||||||
if r:
|
|
||||||
return r
|
|
||||||
return False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def modified(cls, filter_, stix_obj):
|
|
||||||
return cls._timestamp(filter_, stix_obj["modified"])
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def object_marking_refs(cls, filter_, stix_obj):
|
|
||||||
for marking_id in stix_obj["object_marking_refs"]:
|
|
||||||
r = cls._id(filter_, marking_id)
|
|
||||||
if r:
|
|
||||||
return r
|
|
||||||
return False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def revoked(cls, filter_, stix_obj):
|
|
||||||
return cls._boolean(filter_, stix_obj["revoked"])
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def type(cls, filter_, stix_obj):
|
|
||||||
return cls._string(filter_, stix_obj["type"])
|
|
||||||
|
|
|
@ -77,9 +77,7 @@ class FileSystemSource(DataSource):
|
||||||
def get(self, stix_id, _composite_filters=None):
|
def get(self, stix_id, _composite_filters=None):
|
||||||
"""
|
"""
|
||||||
"""
|
"""
|
||||||
query = [
|
query = [Filter("id", "=", stix_id)]
|
||||||
Filter("id", "=", stix_id)
|
|
||||||
]
|
|
||||||
|
|
||||||
all_data = self.query(query=query, _composite_filters=_composite_filters)
|
all_data = self.query(query=query, _composite_filters=_composite_filters)
|
||||||
|
|
||||||
|
@ -91,21 +89,10 @@ class FileSystemSource(DataSource):
|
||||||
"""
|
"""
|
||||||
Notes:
|
Notes:
|
||||||
Since FileSystem sources/sinks don't handle multiple versions
|
Since FileSystem sources/sinks don't handle multiple versions
|
||||||
of a STIX object, this operation is futile. Pass call to get().
|
of a STIX object, this operation is unnecessary. Pass call to get().
|
||||||
(Approved by G.B.)
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# query = [
|
|
||||||
# {
|
|
||||||
# "field": "id",
|
|
||||||
# "op": "=",
|
|
||||||
# "value": stix_id
|
|
||||||
# }
|
|
||||||
# ]
|
|
||||||
|
|
||||||
# all_data = self.query(query=query, _composite_filters=_composite_filters)
|
|
||||||
|
|
||||||
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
|
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
|
||||||
|
|
||||||
def query(self, query=None, _composite_filters=None):
|
def query(self, query=None, _composite_filters=None):
|
||||||
|
|
|
@ -0,0 +1,204 @@
|
||||||
|
"""
|
||||||
|
Filters for Python STIX 2.0 DataSources, DataSinks, DataStores
|
||||||
|
|
||||||
|
Classes:
|
||||||
|
Filter
|
||||||
|
|
||||||
|
TODO: The script at the bottom of the module works (to capture
|
||||||
|
all the callable filter methods), however it causes this module
|
||||||
|
to be imported by itself twice. Not sure how big of deal that is,
|
||||||
|
or if cleaner solution possible.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import types
|
||||||
|
|
||||||
|
# Currently, only STIX 2.0 common SDO fields (that are not complex objects)
|
||||||
|
# are supported for filtering on
|
||||||
|
STIX_COMMON_FIELDS = [
|
||||||
|
"created",
|
||||||
|
"created_by_ref",
|
||||||
|
"external_references.source_name",
|
||||||
|
"external_references.description",
|
||||||
|
"external_references.url",
|
||||||
|
"external_references.hashes",
|
||||||
|
"external_references.external_id",
|
||||||
|
"granular_markings.marking_ref",
|
||||||
|
"granular_markings.selectors",
|
||||||
|
"id",
|
||||||
|
"labels",
|
||||||
|
"modified",
|
||||||
|
"object_marking_refs",
|
||||||
|
"revoked",
|
||||||
|
"type",
|
||||||
|
"granular_markings"
|
||||||
|
]
|
||||||
|
|
||||||
|
# Supported filter operations
|
||||||
|
FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<=']
|
||||||
|
|
||||||
|
# Supported filter value types
|
||||||
|
FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple]
|
||||||
|
|
||||||
|
# filter lookup map - STIX 2 common fields -> filter method
|
||||||
|
STIX_COMMON_FILTERS_MAP = {}
|
||||||
|
|
||||||
|
|
||||||
|
class Filter(collections.namedtuple("Filter", ['field', 'op', 'value'])):
|
||||||
|
__slots__ = ()
|
||||||
|
|
||||||
|
def __new__(cls, field, op, value):
|
||||||
|
# If value is a list, convert it to a tuple so it is hashable.
|
||||||
|
if isinstance(value, list):
|
||||||
|
value = tuple(value)
|
||||||
|
self = super(Filter, cls).__new__(cls, field, op, value)
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
# primitive type filters
|
||||||
|
|
||||||
|
def _all_filter(filter_, stix_obj_field):
|
||||||
|
"""all filter operations (for filters whose value type can be applied to any operation type)"""
|
||||||
|
if filter_.op == "=":
|
||||||
|
return stix_obj_field == filter_.value
|
||||||
|
elif filter_.op == "!=":
|
||||||
|
return stix_obj_field != filter_.value
|
||||||
|
elif filter_.op == "in":
|
||||||
|
return stix_obj_field in filter_.value
|
||||||
|
elif filter_.op == ">":
|
||||||
|
return stix_obj_field > filter_.value
|
||||||
|
elif filter_.op == "<":
|
||||||
|
return stix_obj_field < filter_.value
|
||||||
|
elif filter_.op == ">=":
|
||||||
|
return stix_obj_field >= filter_.value
|
||||||
|
elif filter_.op == "<=":
|
||||||
|
return stix_obj_field <= filter_.value
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
|
||||||
|
def _id_filter(filter_, stix_obj_id):
|
||||||
|
"""base filter types"""
|
||||||
|
if filter_.op == "=":
|
||||||
|
return stix_obj_id == filter_.value
|
||||||
|
elif filter_.op == "!=":
|
||||||
|
return stix_obj_id != filter_.value
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
|
||||||
|
def _boolean_filter(filter_, stix_obj_field):
|
||||||
|
if filter_.op == "=":
|
||||||
|
return stix_obj_field == filter_.value
|
||||||
|
elif filter_.op == "!=":
|
||||||
|
return stix_obj_field != filter_.value
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
|
||||||
|
def _string_filter(filter_, stix_obj_field):
|
||||||
|
return _all_filter(filter_, stix_obj_field)
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp_filter(filter_, stix_obj_timestamp):
|
||||||
|
return _all_filter(filter_, stix_obj_timestamp)
|
||||||
|
|
||||||
|
# STIX 2.0 Common Property filters
|
||||||
|
# The naming of these functions is important as
|
||||||
|
# they are used to index a mapping dictionary from
|
||||||
|
# STIX common field names to these filter functions.
|
||||||
|
#
|
||||||
|
# REQUIRED naming scheme:
|
||||||
|
# "check_<STIX field name>_filter"
|
||||||
|
|
||||||
|
|
||||||
|
def check_created_filter(filter_, stix_obj):
|
||||||
|
return _timestamp_filter(filter_, stix_obj["created"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_created_by_ref_filter(filter_, stix_obj):
|
||||||
|
return _id_filter(filter_, stix_obj["created_by_ref"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_external_references_filter(filter_, stix_obj):
|
||||||
|
"""
|
||||||
|
STIX object's can have a list of external references
|
||||||
|
|
||||||
|
external_references properties:
|
||||||
|
external_references.source_name (string)
|
||||||
|
external_references.description (string)
|
||||||
|
external_references.url (string)
|
||||||
|
external_references.hashes (hash, but for filtering purposes, a string)
|
||||||
|
external_references.external_id (string)
|
||||||
|
|
||||||
|
"""
|
||||||
|
for er in stix_obj["external_references"]:
|
||||||
|
# grab er property name from filter field
|
||||||
|
filter_field = filter_.field.split(".")[1]
|
||||||
|
r = _string_filter(filter_, er[filter_field])
|
||||||
|
if r:
|
||||||
|
return r
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check_granular_markings_filter(filter_, stix_obj):
|
||||||
|
"""
|
||||||
|
STIX object's can have a list of granular marking references
|
||||||
|
|
||||||
|
granular_markings properties:
|
||||||
|
granular_markings.marking_ref (id)
|
||||||
|
granular_markings.selectors (string)
|
||||||
|
|
||||||
|
"""
|
||||||
|
for gm in stix_obj["granular_markings"]:
|
||||||
|
# grab gm property name from filter field
|
||||||
|
filter_field = filter_.field.split(".")[1]
|
||||||
|
|
||||||
|
if filter_field == "marking_ref":
|
||||||
|
return _id_filter(filter_, gm[filter_field])
|
||||||
|
|
||||||
|
elif filter_field == "selectors":
|
||||||
|
for selector in gm[filter_field]:
|
||||||
|
r = _string_filter(filter_, selector)
|
||||||
|
if r:
|
||||||
|
return r
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check_id_filter(filter_, stix_obj):
|
||||||
|
return _id_filter(filter_, stix_obj["id"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_labels_filter(filter_, stix_obj):
|
||||||
|
for label in stix_obj["labels"]:
|
||||||
|
r = _string_filter(filter_, label)
|
||||||
|
if r:
|
||||||
|
return r
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check_modified_filter(filter_, stix_obj):
|
||||||
|
return _timestamp_filter(filter_, stix_obj["modified"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_object_marking_refs_filter(filter_, stix_obj):
|
||||||
|
for marking_id in stix_obj["object_marking_refs"]:
|
||||||
|
r = _id_filter(filter_, marking_id)
|
||||||
|
if r:
|
||||||
|
return r
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check_revoked_filter(filter_, stix_obj):
|
||||||
|
return _boolean_filter(filter_, stix_obj["revoked"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_type_filter(filter_, stix_obj):
|
||||||
|
return _string_filter(filter_, stix_obj["type"])
|
||||||
|
|
||||||
|
|
||||||
|
# Create mapping of field names to filter functions
|
||||||
|
for name, obj in dict(globals()).items():
|
||||||
|
if "check_" in name and isinstance(obj, types.FunctionType):
|
||||||
|
field_name = "_".join(name.split("_")[1:-1])
|
||||||
|
STIX_COMMON_FILTERS_MAP[field_name] = obj
|
|
@ -24,7 +24,8 @@ import os
|
||||||
from stix2validator import validate_string
|
from stix2validator import validate_string
|
||||||
|
|
||||||
from stix2 import Bundle
|
from stix2 import Bundle
|
||||||
from stix2.sources import DataSink, DataSource, DataStore, Filter
|
from stix2.sources import DataSink, DataSource, DataStore
|
||||||
|
from stix2.sources.filters import Filter
|
||||||
|
|
||||||
|
|
||||||
class MemoryStore(DataStore):
|
class MemoryStore(DataStore):
|
||||||
|
@ -205,9 +206,7 @@ class MemorySource(DataSource):
|
||||||
return stix_obj
|
return stix_obj
|
||||||
|
|
||||||
# if there are filters from the composite level, process full query
|
# if there are filters from the composite level, process full query
|
||||||
query = [
|
query = [Filter("id", "=", stix_id)]
|
||||||
Filter("id", "=", stix_id)
|
|
||||||
]
|
|
||||||
|
|
||||||
all_data = self.query(query=query, _composite_filters=_composite_filters)
|
all_data = self.query(query=query, _composite_filters=_composite_filters)
|
||||||
|
|
||||||
|
@ -220,21 +219,10 @@ class MemorySource(DataSource):
|
||||||
"""
|
"""
|
||||||
Notes:
|
Notes:
|
||||||
Since Memory sources/sinks don't handle multiple versions of a
|
Since Memory sources/sinks don't handle multiple versions of a
|
||||||
STIX object, this operation is futile. Translate call to get().
|
STIX object, this operation is unnecessary. Translate call to get().
|
||||||
(Approved by G.B.)
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# query = [
|
|
||||||
# {
|
|
||||||
# "field": "id",
|
|
||||||
# "op": "=",
|
|
||||||
# "value": stix_id
|
|
||||||
# }
|
|
||||||
# ]
|
|
||||||
|
|
||||||
# all_data = self.query(query=query, _composite_filters=_composite_filters)
|
|
||||||
|
|
||||||
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
|
return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)]
|
||||||
|
|
||||||
def query(self, query=None, _composite_filters=None):
|
def query(self, query=None, _composite_filters=None):
|
||||||
|
|
|
@ -12,7 +12,8 @@ TODO: Test everything
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from stix2.sources import DataSink, DataSource, DataStore, Filter, make_id
|
from stix2.sources import DataSink, DataSource, DataStore, make_id
|
||||||
|
from stix2.sources.filters import Filter
|
||||||
|
|
||||||
TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
|
TAXII_FILTERS = ['added_after', 'id', 'type', 'version']
|
||||||
|
|
||||||
|
@ -130,11 +131,7 @@ class TAXIICollectionSource(DataSource):
|
||||||
For instance - "?match[type]=indicator,sighting" should be in a
|
For instance - "?match[type]=indicator,sighting" should be in a
|
||||||
query dict as follows:
|
query dict as follows:
|
||||||
|
|
||||||
{
|
Filter("type", "=", "indicator,sighting")
|
||||||
"field": "type"
|
|
||||||
"op": "=",
|
|
||||||
"value": "indicator,sighting"
|
|
||||||
}
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
query (list): list of filters to extract which ones are TAXII
|
query (list): list of filters to extract which ones are TAXII
|
||||||
|
|
|
@ -2,7 +2,8 @@ import pytest
|
||||||
from taxii2client import Collection
|
from taxii2client import Collection
|
||||||
|
|
||||||
from stix2.sources import (CompositeDataSource, DataSink, DataSource,
|
from stix2.sources import (CompositeDataSource, DataSink, DataSource,
|
||||||
DataStore, Filter, make_id, taxii)
|
DataStore, make_id, taxii)
|
||||||
|
from stix2.sources.filters import Filter
|
||||||
from stix2.sources.memory import MemorySource
|
from stix2.sources.memory import MemorySource
|
||||||
|
|
||||||
COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
|
COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'
|
||||||
|
@ -18,107 +19,110 @@ def collection():
|
||||||
return Collection(COLLECTION_URL, MockTAXIIClient())
|
return Collection(COLLECTION_URL, MockTAXIIClient())
|
||||||
|
|
||||||
|
|
||||||
STIX_OBJS1 = [
|
@pytest.fixture
|
||||||
{
|
def ds():
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
return DataSource()
|
||||||
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
|
||||||
"labels": [
|
|
||||||
"url-watchlist"
|
|
||||||
],
|
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
|
||||||
"name": "Malicious site hosting downloader",
|
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
|
||||||
"type": "indicator",
|
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
|
||||||
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
|
||||||
"labels": [
|
|
||||||
"url-watchlist"
|
|
||||||
],
|
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
|
||||||
"name": "Malicious site hosting downloader",
|
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
|
||||||
"type": "indicator",
|
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
|
||||||
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
|
||||||
"labels": [
|
|
||||||
"url-watchlist"
|
|
||||||
],
|
|
||||||
"modified": "2017-01-27T13:49:53.936Z",
|
|
||||||
"name": "Malicious site hosting downloader",
|
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
|
||||||
"type": "indicator",
|
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
|
||||||
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
|
||||||
"labels": [
|
|
||||||
"url-watchlist"
|
|
||||||
],
|
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
|
||||||
"name": "Malicious site hosting downloader",
|
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
|
||||||
"type": "indicator",
|
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
|
||||||
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
|
||||||
"labels": [
|
|
||||||
"url-watchlist"
|
|
||||||
],
|
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
|
||||||
"name": "Malicious site hosting downloader",
|
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
|
||||||
"type": "indicator",
|
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
STIX_OBJS2 = [
|
|
||||||
{
|
IND1 = {
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
||||||
"labels": [
|
"labels": [
|
||||||
"url-watchlist"
|
"url-watchlist"
|
||||||
],
|
],
|
||||||
"modified": "2017-01-31T13:49:53.935Z",
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
"name": "Malicious site hosting downloader",
|
"name": "Malicious site hosting downloader",
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
"type": "indicator",
|
"type": "indicator",
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
},
|
}
|
||||||
{
|
IND2 = {
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
||||||
"labels": [
|
"labels": [
|
||||||
"url-watchlist"
|
"url-watchlist"
|
||||||
],
|
],
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
"name": "Malicious site hosting downloader",
|
"name": "Malicious site hosting downloader",
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
"type": "indicator",
|
"type": "indicator",
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
},
|
}
|
||||||
{
|
IND3 = {
|
||||||
"created": "2017-01-27T13:49:53.935Z",
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
||||||
"labels": [
|
"labels": [
|
||||||
"url-watchlist"
|
"url-watchlist"
|
||||||
],
|
],
|
||||||
"modified": "2017-01-27T13:49:53.935Z",
|
"modified": "2017-01-27T13:49:53.936Z",
|
||||||
"name": "Malicious site hosting downloader",
|
"name": "Malicious site hosting downloader",
|
||||||
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
"type": "indicator",
|
"type": "indicator",
|
||||||
"valid_from": "2017-01-27T13:49:53.935382Z"
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
}
|
}
|
||||||
]
|
IND4 = {
|
||||||
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
|
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
||||||
|
"labels": [
|
||||||
|
"url-watchlist"
|
||||||
|
],
|
||||||
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
|
"name": "Malicious site hosting downloader",
|
||||||
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
|
"type": "indicator",
|
||||||
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
|
}
|
||||||
|
IND5 = {
|
||||||
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
|
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
||||||
|
"labels": [
|
||||||
|
"url-watchlist"
|
||||||
|
],
|
||||||
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
|
"name": "Malicious site hosting downloader",
|
||||||
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
|
"type": "indicator",
|
||||||
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
|
}
|
||||||
|
IND6 = {
|
||||||
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
|
"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",
|
||||||
|
"labels": [
|
||||||
|
"url-watchlist"
|
||||||
|
],
|
||||||
|
"modified": "2017-01-31T13:49:53.935Z",
|
||||||
|
"name": "Malicious site hosting downloader",
|
||||||
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
|
"type": "indicator",
|
||||||
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
|
}
|
||||||
|
IND7 = {
|
||||||
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
|
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
||||||
|
"labels": [
|
||||||
|
"url-watchlist"
|
||||||
|
],
|
||||||
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
|
"name": "Malicious site hosting downloader",
|
||||||
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
|
"type": "indicator",
|
||||||
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
|
}
|
||||||
|
IND8 = {
|
||||||
|
"created": "2017-01-27T13:49:53.935Z",
|
||||||
|
"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",
|
||||||
|
"labels": [
|
||||||
|
"url-watchlist"
|
||||||
|
],
|
||||||
|
"modified": "2017-01-27T13:49:53.935Z",
|
||||||
|
"name": "Malicious site hosting downloader",
|
||||||
|
"pattern": "[url:value = 'http://x4z9arb.cn/4712']",
|
||||||
|
"type": "indicator",
|
||||||
|
"valid_from": "2017-01-27T13:49:53.935382Z"
|
||||||
|
}
|
||||||
|
|
||||||
|
STIX_OBJS2 = [IND6, IND7, IND8]
|
||||||
|
STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5]
|
||||||
|
|
||||||
|
|
||||||
def test_ds_smoke():
|
def test_ds_smoke():
|
||||||
|
@ -172,7 +176,7 @@ def test_parse_taxii_filters():
|
||||||
assert taxii_filters == expected_params
|
assert taxii_filters == expected_params
|
||||||
|
|
||||||
|
|
||||||
def test_add_get_remove_filter():
|
def test_add_get_remove_filter(ds):
|
||||||
|
|
||||||
# First 3 filters are valid, remaining fields are erroneous in some way
|
# First 3 filters are valid, remaining fields are erroneous in some way
|
||||||
valid_filters = [
|
valid_filters = [
|
||||||
|
@ -186,8 +190,6 @@ def test_add_get_remove_filter():
|
||||||
Filter('created', '=', object()),
|
Filter('created', '=', object()),
|
||||||
]
|
]
|
||||||
|
|
||||||
ds = DataSource()
|
|
||||||
|
|
||||||
assert len(ds.filters) == 0
|
assert len(ds.filters) == 0
|
||||||
|
|
||||||
ds.add_filter(valid_filters[0])
|
ds.add_filter(valid_filters[0])
|
||||||
|
@ -225,7 +227,7 @@ def test_add_get_remove_filter():
|
||||||
ds.add_filters(valid_filters)
|
ds.add_filters(valid_filters)
|
||||||
|
|
||||||
|
|
||||||
def test_apply_common_filters():
|
def test_apply_common_filters(ds):
|
||||||
stix_objs = [
|
stix_objs = [
|
||||||
{
|
{
|
||||||
"created": "2017-01-27T13:49:53.997Z",
|
"created": "2017-01-27T13:49:53.997Z",
|
||||||
|
@ -286,8 +288,6 @@ def test_apply_common_filters():
|
||||||
Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
|
Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"),
|
||||||
]
|
]
|
||||||
|
|
||||||
ds = DataSource()
|
|
||||||
|
|
||||||
resp = ds.apply_common_filters(stix_objs, [filters[0]])
|
resp = ds.apply_common_filters(stix_objs, [filters[0]])
|
||||||
ids = [r['id'] for r in resp]
|
ids = [r['id'] for r in resp]
|
||||||
assert stix_objs[0]['id'] in ids
|
assert stix_objs[0]['id'] in ids
|
||||||
|
@ -327,61 +327,65 @@ def test_apply_common_filters():
|
||||||
assert resp[0]['id'] == stix_objs[2]['id']
|
assert resp[0]['id'] == stix_objs[2]['id']
|
||||||
assert len(resp) == 1
|
assert len(resp) == 1
|
||||||
|
|
||||||
# These are used with STIX_OBJS2
|
|
||||||
more_filters = [
|
|
||||||
Filter("modified", "<", "2017-01-28T13:49:53.935Z"),
|
|
||||||
Filter("modified", ">", "2017-01-28T13:49:53.935Z"),
|
|
||||||
Filter("modified", ">=", "2017-01-27T13:49:53.935Z"),
|
|
||||||
Filter("modified", "<=", "2017-01-27T13:49:53.935Z"),
|
|
||||||
Filter("modified", "?", "2017-01-27T13:49:53.935Z"),
|
|
||||||
Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"),
|
|
||||||
Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f"),
|
|
||||||
Filter("notacommonproperty", "=", "bar"),
|
|
||||||
]
|
|
||||||
|
|
||||||
resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[0]])
|
def test_filters0(ds):
|
||||||
|
resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])
|
||||||
assert resp[0]['id'] == STIX_OBJS2[1]['id']
|
assert resp[0]['id'] == STIX_OBJS2[1]['id']
|
||||||
assert len(resp) == 2
|
assert len(resp) == 2
|
||||||
|
|
||||||
resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[1]])
|
|
||||||
|
def test_filters1(ds):
|
||||||
|
resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])
|
||||||
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
||||||
assert len(resp) == 1
|
assert len(resp) == 1
|
||||||
|
|
||||||
resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[2]])
|
|
||||||
|
def test_filters2(ds):
|
||||||
|
resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])
|
||||||
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
||||||
assert len(resp) == 3
|
assert len(resp) == 3
|
||||||
|
|
||||||
resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[3]])
|
|
||||||
|
def test_filters3(ds):
|
||||||
|
resp = ds.apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])
|
||||||
assert resp[0]['id'] == STIX_OBJS2[1]['id']
|
assert resp[0]['id'] == STIX_OBJS2[1]['id']
|
||||||
assert len(resp) == 2
|
assert len(resp) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_filters4(ds):
|
||||||
|
fltr4 = Filter("modified", "?", "2017-01-27T13:49:53.935Z")
|
||||||
with pytest.raises(ValueError) as excinfo:
|
with pytest.raises(ValueError) as excinfo:
|
||||||
ds.apply_common_filters(STIX_OBJS2, [more_filters[4]])
|
ds.apply_common_filters(STIX_OBJS2, [fltr4])
|
||||||
|
|
||||||
assert str(excinfo.value) == ("Error, filter operator: {0} not supported "
|
assert str(excinfo.value) == ("Error, filter operator: {0} not supported "
|
||||||
"for specified field: {1}").format(more_filters[4].op,
|
"for specified field: {1}").format(fltr4.op, fltr4.field)
|
||||||
more_filters[4].field)
|
|
||||||
|
|
||||||
resp = ds.apply_common_filters(STIX_OBJS2, [more_filters[5]])
|
|
||||||
|
def test_filters5(ds):
|
||||||
|
resp = ds.apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])
|
||||||
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
assert resp[0]['id'] == STIX_OBJS2[0]['id']
|
||||||
assert len(resp) == 1
|
assert len(resp) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_filters6(ds):
|
||||||
|
fltr6 = Filter("id", "?", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")
|
||||||
with pytest.raises(ValueError) as excinfo:
|
with pytest.raises(ValueError) as excinfo:
|
||||||
ds.apply_common_filters(STIX_OBJS2, [more_filters[6]])
|
ds.apply_common_filters(STIX_OBJS2, [fltr6])
|
||||||
|
|
||||||
assert str(excinfo.value) == ("Error, filter operator: {0} not supported "
|
assert str(excinfo.value) == ("Error, filter operator: {0} not supported "
|
||||||
"for specified field: {1}").format(more_filters[6].op,
|
"for specified field: {1}").format(fltr6.op, fltr6.field)
|
||||||
more_filters[6].field)
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_filters7(ds):
|
||||||
|
fltr7 = Filter("notacommonproperty", "=", "bar")
|
||||||
with pytest.raises(ValueError) as excinfo:
|
with pytest.raises(ValueError) as excinfo:
|
||||||
ds.apply_common_filters(STIX_OBJS2, [more_filters[7]])
|
ds.apply_common_filters(STIX_OBJS2, [fltr7])
|
||||||
|
|
||||||
assert str(excinfo.value) == ("Error, field: {0} is not supported for "
|
assert str(excinfo.value) == ("Error, field: {0} is not supported for "
|
||||||
"filtering on.".format(more_filters[7].field))
|
"filtering on.".format(fltr7.field))
|
||||||
|
|
||||||
|
|
||||||
def test_deduplicate():
|
def test_deduplicate(ds):
|
||||||
ds = DataSource()
|
|
||||||
unique = ds.deduplicate(STIX_OBJS1)
|
unique = ds.deduplicate(STIX_OBJS1)
|
||||||
|
|
||||||
# Only 3 objects are unique
|
# Only 3 objects are unique
|
||||||
|
|
Loading…
Reference in New Issue