From 239291253383639594d2df6ebf83c7252b0548b0 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 May 2018 11:28:34 -0400 Subject: [PATCH 1/8] handle TAXII client/server errors according to decided policy --- stix2/datastore/taxii.py | 63 +++++++++++++++++--- stix2/test/test_datastore_taxii.py | 96 +++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 10 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 872a510..e3f4e45 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -51,7 +51,27 @@ class TAXIICollectionSink(DataSink): """ def __init__(self, collection, allow_custom=False): super(TAXIICollectionSink, self).__init__() - self.collection = collection + try: + # we have to execute .can_write first in isolation because the + # attribute access could trigger a taxii2client.ValidationError which + # we catch here as a ValueError (its parent class). Later, we need to + # have the ability to also raise a different ValueError based on the + # value of .can_write + writeable = collection.can_write + + except (HTTPError, ValueError) as e: + e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached. TAXII Collection Error: " + + e.message) + raise + + if writeable: + # now past taxii2client possible exceptions, check value for local exceptions + self.collection = collection + else: + raise ValueError("The TAXII Collection object provided does not have write access" + " to the underlying linked Collection resource") + self.allow_custom = allow_custom def add(self, stix_data, version=None): @@ -111,7 +131,27 @@ class TAXIICollectionSource(DataSource): """ def __init__(self, collection, allow_custom=True): super(TAXIICollectionSource, self).__init__() - self.collection = collection + try: + # we have to execute .can_read first in isolation because the + # attribute access could trigger a taxii2client.ValidationError which + # we catch here as a ValueError (its parent class). Later, we need to + # have the ability to also raise a different ValueError based on the + # value of .can_read + writeable = collection.can_read + + except (HTTPError, ValueError) as e: + e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached. TAXII Collection Error: " + + e.message) + raise + + if writeable: + # now past taxii2client possible exceptions, check value for local exceptions + self.collection = collection + else: + raise ValueError("The TAXII Collection object provided does not have read access" + " to the underlying linked Collection resource") + self.allow_custom = allow_custom def get(self, stix_id, version=None, _composite_filters=None): @@ -145,9 +185,12 @@ class TAXIICollectionSource(DataSource): stix_objs = self.collection.get_object(stix_id)["objects"] stix_obj = list(apply_common_filters(stix_objs, query)) - except HTTPError: - # if resource not found or access is denied from TAXII server, return None - stix_obj = [] + except HTTPError as err: + if err.response.status_code == 404: + # if resource not found or access is denied from TAXII server, return None + stix_obj = [] + else: + raise if len(stix_obj): stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version) @@ -231,13 +274,17 @@ class TAXIICollectionSource(DataSource): # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) - # apply local (CompositeDataSource, TAXIICollectionSource and query) filters + # a pply local (CompositeDataSource, TAXIICollectionSource and query) filters query.remove(taxii_filters) all_data = list(apply_common_filters(all_data, query)) - except HTTPError: + except HTTPError as err: # if resources not found or access is denied from TAXII server, return empty list - all_data = [] + if err.response.status_code == 404: + err.message = ("The requested STIX objects for the TAXII Collection resource defined in" + " the supplied TAXII Collection object is either not found or access is" + " denied. Received error: " + err.message) + raise # parse python STIX objects from the STIX object dicts stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index 098f944..c753afc 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -2,6 +2,8 @@ import json from medallion.filters.basic_filter import BasicFilter import pytest +from requests.exceptions import HTTPError +from requests.models import Response from taxii2client import Collection, _filter_kwargs_to_query_params from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource, @@ -36,7 +38,12 @@ class MockTAXIICollectionEndpoint(Collection): ("id", "type", "version"), [] ) - return Bundle(objects=objs) + if objs: + return Bundle(objects=objs) + else: + resp = Response() + resp.status_code = 404 + resp.raise_for_status() def get_object(self, id, version=None): self._verify_can_read() @@ -51,7 +58,12 @@ class MockTAXIICollectionEndpoint(Collection): ("version",), [] ) - return Bundle(objects=objs) + if objs: + return Bundle(objects=objs) + else: + resp = Response() + resp.status_code = 404 + resp.raise_for_status() @pytest.fixture @@ -71,6 +83,23 @@ def collection(stix_objs1): return mock +@pytest.fixture +def collection_no_rw_access(stix_objs1): + mock = MockTAXIICollectionEndpoint(COLLECTION_URL, **{ + "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", + "title": "Not writeable or readable Collection", + "description": "This collection is a dropbox for submitting indicators", + "can_read": False, + "can_write": False, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + }) + + mock.objects.extend(stix_objs1) + return mock + + def test_ds_taxii(collection): ds = TAXIICollectionSource(collection) assert ds.collection is not None @@ -292,3 +321,66 @@ def test_get_all_versions(collection): indicators = ds.all_versions('indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f') # There are 3 indicators but 2 share the same 'modified' timestamp assert len(indicators) == 2 + + +def test_can_read_error(collection_no_rw_access): + """create a TAXIICOllectionSource with a taxii2client.Collection + instance that does not have read access, check ValueError exception is raised""" + with pytest.raises(ValueError) as excinfo: + TAXIICollectionSource(collection_no_rw_access) + assert "Collection object provided does not have read access" in str(excinfo.value) + + +def test_can_write_error(collection_no_rw_access): + """create a TAXIICOllectionSink with a taxii2client.Collection + instance that does not have write access, check ValueError exception is raised""" + with pytest.raises(ValueError) as excinfo: + TAXIICollectionSink(collection_no_rw_access) + assert "Collection object provided does not have write access" in str(excinfo.value) + + +def test_bad_collection(): + """this triggers a real connectivity issue (HTTPError: 503 ServerError) """ + with pytest.raises(HTTPError) as excinfo: + mock = MockTAXIICollectionEndpoint("http://doenstexist118482.org", verify=False) + TAXIICollectionStore(mock) + assert "Collection object provided could not be reached. TAXII Collection Error:" in str(excinfo.value.message) + assert "HTTPError" in str(excinfo.type) + + +def test_get_404(collection): + """a TAXIICollectionSource.get() call that receives an HTTP 404 response + code from the taxii2client should be be returned as None. + + TAXII spec states that a TAXII server can return a 404 for + nonexistent resources or lack of access. Decided that None is acceptable + reponse to imply that state of the TAXII endpoint. + """ + ds = TAXIICollectionStore(collection) + + # this will raise 404 from mock TAXII Client but TAXIICollectionStore + # should handle gracefully and return None + stix_obj = ds.get("indicator--1") # this will raise 404 from + assert stix_obj is None + + +def test_all_versions_404(collection): + """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404 + response code from the taxii2client should be returned as an exception""" + ds = TAXIICollectionStore(collection) + with pytest.raises(HTTPError) as excinfo: + ds.all_versions("indicator--1") + assert "is either not found or access is denied" in str(excinfo.value.message) + assert "404" in str(excinfo.value) + + +def test_query_404(collection): + """ a TAXIICollectionSource.query() call that recieves an HTTP 404 + response code from the taxii2client should be returned as an exception""" + ds = TAXIICollectionStore(collection) + query = [Filter("type", "=", "malware")] + + with pytest.raises(HTTPError) as excinfo: + ds.query(query=query) + assert "is either not found or access is denied" in str(excinfo.value.message) + assert "404" in str(excinfo.value) From 0d3f80f2fec627ca00e72d031c15d38f2352e1a6 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 May 2018 15:41:46 -0400 Subject: [PATCH 2/8] removing taxii 503 error test as not reproducible in Travis environment --- stix2/datastore/taxii.py | 57 ++++++++++++++---------------- stix2/test/test_datastore_taxii.py | 9 ----- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index e3f4e45..776baf7 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -9,6 +9,13 @@ from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, FilterSet, apply_common_filters from stix2.utils import deduplicate +try: + from taxii2client import ValidationError + _taxii2_client = True +except ImportError: + _taxii2_client = False + + TAXII_FILTERS = ['added_after', 'id', 'type', 'version'] @@ -51,27 +58,22 @@ class TAXIICollectionSink(DataSink): """ def __init__(self, collection, allow_custom=False): super(TAXIICollectionSink, self).__init__() - try: - # we have to execute .can_write first in isolation because the - # attribute access could trigger a taxii2client.ValidationError which - # we catch here as a ValueError (its parent class). Later, we need to - # have the ability to also raise a different ValueError based on the - # value of .can_write - writeable = collection.can_write + if not _taxii2_client: + raise ImportError("taxii2client library is required for usage of TAXIICollectionSink") - except (HTTPError, ValueError) as e: + try: + if collection.can_write: + self.collection = collection + else: + raise ValueError("The TAXII Collection object provided does not have write access" + " to the underlying linked Collection resource") + + except (HTTPError, ValidationError) as e: e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" " Collection object provided could not be reached. TAXII Collection Error: " + e.message) raise - if writeable: - # now past taxii2client possible exceptions, check value for local exceptions - self.collection = collection - else: - raise ValueError("The TAXII Collection object provided does not have write access" - " to the underlying linked Collection resource") - self.allow_custom = allow_custom def add(self, stix_data, version=None): @@ -131,27 +133,22 @@ class TAXIICollectionSource(DataSource): """ def __init__(self, collection, allow_custom=True): super(TAXIICollectionSource, self).__init__() - try: - # we have to execute .can_read first in isolation because the - # attribute access could trigger a taxii2client.ValidationError which - # we catch here as a ValueError (its parent class). Later, we need to - # have the ability to also raise a different ValueError based on the - # value of .can_read - writeable = collection.can_read + if not _taxii2_client: + raise ImportError("taxii2client library is required for usage of TAXIICollectionSource") - except (HTTPError, ValueError) as e: + try: + if collection.can_read: + self.collection = collection + else: + raise ValueError("The TAXII Collection object provided does not have read access" + " to the underlying linked Collection resource") + + except (HTTPError, ValidationError) as e: e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" " Collection object provided could not be reached. TAXII Collection Error: " + e.message) raise - if writeable: - # now past taxii2client possible exceptions, check value for local exceptions - self.collection = collection - else: - raise ValueError("The TAXII Collection object provided does not have read access" - " to the underlying linked Collection resource") - self.allow_custom = allow_custom def get(self, stix_id, version=None, _composite_filters=None): diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index c753afc..af8fecd 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -339,15 +339,6 @@ def test_can_write_error(collection_no_rw_access): assert "Collection object provided does not have write access" in str(excinfo.value) -def test_bad_collection(): - """this triggers a real connectivity issue (HTTPError: 503 ServerError) """ - with pytest.raises(HTTPError) as excinfo: - mock = MockTAXIICollectionEndpoint("http://doenstexist118482.org", verify=False) - TAXIICollectionStore(mock) - assert "Collection object provided could not be reached. TAXII Collection Error:" in str(excinfo.value.message) - assert "HTTPError" in str(excinfo.type) - - def test_get_404(collection): """a TAXIICollectionSource.get() call that receives an HTTP 404 response code from the taxii2client should be be returned as None. From 2b4c5bf264d15ba5dfbc1151fe80a34de00c0a91 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 May 2018 17:42:19 -0400 Subject: [PATCH 3/8] handling and modifying exception messages in a manner acceptable by all python versions --- stix2/datastore/taxii.py | 35 ++++++++++++++++++++---------- stix2/test/test_datastore_taxii.py | 4 ++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 776baf7..4a047a8 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -69,9 +69,13 @@ class TAXIICollectionSink(DataSink): " to the underlying linked Collection resource") except (HTTPError, ValidationError) as e: - e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" - " Collection object provided could not be reached. TAXII Collection Error: " - + e.message) + added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached.") + if not e.args: + e.args = (added_context,) + else: + e.args = (added_context,) + e.args + raise self.allow_custom = allow_custom @@ -144,9 +148,13 @@ class TAXIICollectionSource(DataSource): " to the underlying linked Collection resource") except (HTTPError, ValidationError) as e: - e.message = ("The underlying TAXII Collection resource defined in the supplied TAXII" - " Collection object provided could not be reached. TAXII Collection Error: " - + e.message) + added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached.") + if not e.args: + e.args = (added_context,) + else: + e.args = (added_context,) + e.args + raise self.allow_custom = allow_custom @@ -275,12 +283,17 @@ class TAXIICollectionSource(DataSource): query.remove(taxii_filters) all_data = list(apply_common_filters(all_data, query)) - except HTTPError as err: + except HTTPError as e: # if resources not found or access is denied from TAXII server, return empty list - if err.response.status_code == 404: - err.message = ("The requested STIX objects for the TAXII Collection resource defined in" - " the supplied TAXII Collection object is either not found or access is" - " denied. Received error: " + err.message) + if e.response.status_code == 404: + added_context = ("The requested STIX objects for the TAXII Collection resource defined in" + " the supplied TAXII Collection object is either not found or access is" + " denied. Received error: ") + if not e.args: + e.args = (added_context,) + else: + e.args = (added_context,) + e.args + raise # parse python STIX objects from the STIX object dicts diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index af8fecd..a153ce8 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -361,7 +361,7 @@ def test_all_versions_404(collection): ds = TAXIICollectionStore(collection) with pytest.raises(HTTPError) as excinfo: ds.all_versions("indicator--1") - assert "is either not found or access is denied" in str(excinfo.value.message) + assert "is either not found or access is denied" in str(excinfo.value) assert "404" in str(excinfo.value) @@ -373,5 +373,5 @@ def test_query_404(collection): with pytest.raises(HTTPError) as excinfo: ds.query(query=query) - assert "is either not found or access is denied" in str(excinfo.value.message) + assert "is either not found or access is denied" in str(excinfo.value) assert "404" in str(excinfo.value) From a3313bc08fb754877bbcab53e9193c835a439e34 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 16 May 2018 13:23:50 -0400 Subject: [PATCH 4/8] creating native DataSourceError exception class to wrap taxii client and server errors --- stix2/datastore/__init__.py | 26 ++++++++++++++-- stix2/datastore/taxii.py | 49 ++++++++++-------------------- stix2/test/test_datastore_taxii.py | 14 ++++----- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py index 7fdf515..ac03959 100644 --- a/stix2/datastore/__init__.py +++ b/stix2/datastore/__init__.py @@ -24,15 +24,35 @@ def make_id(): return str(uuid.uuid4()) +class DataSourceError(Exception): + """General DataSource error instance, used primarily for wrapping + lower level errors + + Args: + message (str): error message + root_exception (Exception): Exception instance of root exception + in the case that DataSourceError is wrapping a lower level or + other exception + """ + def __init__(self, message, root_exception=None): + self.message = message + self.root_exception = root_exception + + def __str__(self): + if self.root_exception: + self.message = "{} \"{}\"".format(self.message, self.root_exception) + return self.message + + class DataStoreMixin(object): """Provides mechanisms for storing and retrieving STIX data. The specific behavior can be customized by subclasses. Args: source (DataSource): An existing DataSource to use - as this DataStore's DataSource component + as this DataStore's DataSource component sink (DataSink): An existing DataSink to use - as this DataStore's DataSink component + as this DataStore's DataSink component Attributes: id (str): A unique UUIDv4 to identify this DataStore. @@ -129,7 +149,7 @@ class DataStoreMixin(object): obj (STIX object OR dict OR str): The STIX object (or its ID) whose relationships will be looked up. relationship_type (str): Only retrieve Relationships of this type. - If None, all relationships will be returned, regardless of type. + If None, all relationships will be returned, regardless of type. source_only (bool): Only retrieve Relationships for which this object is the source_ref. Default: False. target_only (bool): Only retrieve Relationships for which this diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 4a047a8..88897a9 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -5,7 +5,8 @@ from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse -from stix2.datastore import DataSink, DataSource, DataStoreMixin +from stix2.datastore import (DataSink, DataSource, DataSourceError, + DataStoreMixin) from stix2.datastore.filters import Filter, FilterSet, apply_common_filters from stix2.utils import deduplicate @@ -65,18 +66,12 @@ class TAXIICollectionSink(DataSink): if collection.can_write: self.collection = collection else: - raise ValueError("The TAXII Collection object provided does not have write access" - " to the underlying linked Collection resource") + raise DataSourceError("The TAXII Collection object provided does not have write access" + " to the underlying linked Collection resource") except (HTTPError, ValidationError) as e: - added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII" - " Collection object provided could not be reached.") - if not e.args: - e.args = (added_context,) - else: - e.args = (added_context,) + e.args - - raise + raise DataSourceError("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached. Receved error:", e) self.allow_custom = allow_custom @@ -144,18 +139,12 @@ class TAXIICollectionSource(DataSource): if collection.can_read: self.collection = collection else: - raise ValueError("The TAXII Collection object provided does not have read access" - " to the underlying linked Collection resource") + raise DataSourceError("The TAXII Collection object provided does not have read access" + " to the underlying linked Collection resource") except (HTTPError, ValidationError) as e: - added_context = ("The underlying TAXII Collection resource defined in the supplied TAXII" - " Collection object provided could not be reached.") - if not e.args: - e.args = (added_context,) - else: - e.args = (added_context,) + e.args - - raise + raise DataSourceError("The underlying TAXII Collection resource defined in the supplied TAXII" + " Collection object provided could not be reached. Recieved error:", e) self.allow_custom = allow_custom @@ -190,12 +179,12 @@ class TAXIICollectionSource(DataSource): stix_objs = self.collection.get_object(stix_id)["objects"] stix_obj = list(apply_common_filters(stix_objs, query)) - except HTTPError as err: - if err.response.status_code == 404: + except HTTPError as e: + if e.response.status_code == 404: # if resource not found or access is denied from TAXII server, return None stix_obj = [] else: - raise + raise DataSourceError("TAXII Collection resource returned error", e) if len(stix_obj): stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version) @@ -286,15 +275,9 @@ class TAXIICollectionSource(DataSource): except HTTPError as e: # if resources not found or access is denied from TAXII server, return empty list if e.response.status_code == 404: - added_context = ("The requested STIX objects for the TAXII Collection resource defined in" - " the supplied TAXII Collection object is either not found or access is" - " denied. Received error: ") - if not e.args: - e.args = (added_context,) - else: - e.args = (added_context,) + e.args - - raise + raise DataSourceError("The requested STIX objects for the TAXII Collection resource defined in" + " the supplied TAXII Collection object are either not found or access is" + " denied. Received error: ", e) # parse python STIX objects from the STIX object dicts stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index a153ce8..78a3284 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -2,12 +2,12 @@ import json from medallion.filters.basic_filter import BasicFilter import pytest -from requests.exceptions import HTTPError from requests.models import Response from taxii2client import Collection, _filter_kwargs_to_query_params from stix2 import (Bundle, TAXIICollectionSink, TAXIICollectionSource, TAXIICollectionStore, ThreatActor) +from stix2.datastore import DataSourceError from stix2.datastore.filters import Filter COLLECTION_URL = 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/' @@ -326,7 +326,7 @@ def test_get_all_versions(collection): def test_can_read_error(collection_no_rw_access): """create a TAXIICOllectionSource with a taxii2client.Collection instance that does not have read access, check ValueError exception is raised""" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(DataSourceError) as excinfo: TAXIICollectionSource(collection_no_rw_access) assert "Collection object provided does not have read access" in str(excinfo.value) @@ -334,7 +334,7 @@ def test_can_read_error(collection_no_rw_access): def test_can_write_error(collection_no_rw_access): """create a TAXIICOllectionSink with a taxii2client.Collection instance that does not have write access, check ValueError exception is raised""" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(DataSourceError) as excinfo: TAXIICollectionSink(collection_no_rw_access) assert "Collection object provided does not have write access" in str(excinfo.value) @@ -359,9 +359,9 @@ def test_all_versions_404(collection): """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404 response code from the taxii2client should be returned as an exception""" ds = TAXIICollectionStore(collection) - with pytest.raises(HTTPError) as excinfo: + with pytest.raises(DataSourceError) as excinfo: ds.all_versions("indicator--1") - assert "is either not found or access is denied" in str(excinfo.value) + assert "are either not found or access is denied" in str(excinfo.value) assert "404" in str(excinfo.value) @@ -371,7 +371,7 @@ def test_query_404(collection): ds = TAXIICollectionStore(collection) query = [Filter("type", "=", "malware")] - with pytest.raises(HTTPError) as excinfo: + with pytest.raises(DataSourceError) as excinfo: ds.query(query=query) - assert "is either not found or access is denied" in str(excinfo.value) + assert "are either not found or access is denied" in str(excinfo.value) assert "404" in str(excinfo.value) From 8fc421e7d43b9004841d109d8809da338c4a7c54 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 16 May 2018 15:17:34 -0400 Subject: [PATCH 5/8] think the mock client was erroneous for get_object(), couldnt get object by id, just returned random object --- stix2/test/test_datastore_taxii.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index 78a3284..dbade77 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -45,17 +45,17 @@ class MockTAXIICollectionEndpoint(Collection): resp.status_code = 404 resp.raise_for_status() - def get_object(self, id, version=None): + def get_object(self, id_, version=None): self._verify_can_read() - query_params = None + kwargs = {"id": id_} if version: - query_params = _filter_kwargs_to_query_params({"version": version}) - if query_params: - query_params = json.loads(query_params) + kwargs.update({"version": version}) + + query_params = _filter_kwargs_to_query_params(kwargs) full_filter = BasicFilter(query_params or {}) objs = full_filter.process_filter( self.objects, - ("version",), + ("version", "id"), [] ) if objs: From 6004ec597c432badeb57ca350b2ff22d2cdf1051 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 18 May 2018 11:19:05 -0400 Subject: [PATCH 6/8] solution to get 404 mocking problem --- stix2/test/test_datastore_taxii.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index dbade77..be8adaa 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -45,17 +45,17 @@ class MockTAXIICollectionEndpoint(Collection): resp.status_code = 404 resp.raise_for_status() - def get_object(self, id_, version=None): + def get_object(self, id, version=None): self._verify_can_read() - kwargs = {"id": id_} + query_params = None if version: - kwargs.update({"version": version}) - - query_params = _filter_kwargs_to_query_params(kwargs) + query_params = _filter_kwargs_to_query_params({"version": version}) + if query_params: + query_params = json.loads(query_params) full_filter = BasicFilter(query_params or {}) objs = full_filter.process_filter( self.objects, - ("version", "id"), + ("version",), [] ) if objs: @@ -339,7 +339,7 @@ def test_can_write_error(collection_no_rw_access): assert "Collection object provided does not have write access" in str(excinfo.value) -def test_get_404(collection): +def test_get_404(): """a TAXIICollectionSource.get() call that receives an HTTP 404 response code from the taxii2client should be be returned as None. @@ -347,7 +347,16 @@ def test_get_404(collection): nonexistent resources or lack of access. Decided that None is acceptable reponse to imply that state of the TAXII endpoint. """ - ds = TAXIICollectionStore(collection) + + class TAXIICollection404(): + can_read = True + + def get_object(self, id, version=None): + resp = Response() + resp.status_code = 404 + resp.raise_for_status() + + ds = TAXIICollectionSource(TAXIICollection404()) # this will raise 404 from mock TAXII Client but TAXIICollectionStore # should handle gracefully and return None From e484b7c25f6fac6209b927a480e3415a1510a3cc Mon Sep 17 00:00:00 2001 From: = Date: Fri, 18 May 2018 11:45:40 -0400 Subject: [PATCH 7/8] formatting --- stix2/datastore/taxii.py | 2 +- stix2/test/test_datastore_taxii.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/stix2/datastore/taxii.py b/stix2/datastore/taxii.py index 88897a9..c815e12 100644 --- a/stix2/datastore/taxii.py +++ b/stix2/datastore/taxii.py @@ -268,7 +268,7 @@ class TAXIICollectionSource(DataSource): # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) - # a pply local (CompositeDataSource, TAXIICollectionSource and query) filters + # apply local (CompositeDataSource, TAXIICollectionSource and query) filters query.remove(taxii_filters) all_data = list(apply_common_filters(all_data, query)) diff --git a/stix2/test/test_datastore_taxii.py b/stix2/test/test_datastore_taxii.py index be8adaa..8675378 100644 --- a/stix2/test/test_datastore_taxii.py +++ b/stix2/test/test_datastore_taxii.py @@ -326,6 +326,7 @@ def test_get_all_versions(collection): def test_can_read_error(collection_no_rw_access): """create a TAXIICOllectionSource with a taxii2client.Collection instance that does not have read access, check ValueError exception is raised""" + with pytest.raises(DataSourceError) as excinfo: TAXIICollectionSource(collection_no_rw_access) assert "Collection object provided does not have read access" in str(excinfo.value) @@ -334,6 +335,7 @@ def test_can_read_error(collection_no_rw_access): def test_can_write_error(collection_no_rw_access): """create a TAXIICOllectionSink with a taxii2client.Collection instance that does not have write access, check ValueError exception is raised""" + with pytest.raises(DataSourceError) as excinfo: TAXIICollectionSink(collection_no_rw_access) assert "Collection object provided does not have write access" in str(excinfo.value) @@ -343,9 +345,9 @@ def test_get_404(): """a TAXIICollectionSource.get() call that receives an HTTP 404 response code from the taxii2client should be be returned as None. - TAXII spec states that a TAXII server can return a 404 for - nonexistent resources or lack of access. Decided that None is acceptable - reponse to imply that state of the TAXII endpoint. + TAXII spec states that a TAXII server can return a 404 for nonexistent + resources or lack of access. Decided that None is acceptable reponse + to imply that state of the TAXII endpoint. """ class TAXIICollection404(): @@ -360,14 +362,16 @@ def test_get_404(): # this will raise 404 from mock TAXII Client but TAXIICollectionStore # should handle gracefully and return None - stix_obj = ds.get("indicator--1") # this will raise 404 from + stix_obj = ds.get("indicator--1") assert stix_obj is None def test_all_versions_404(collection): """ a TAXIICollectionSource.all_version() call that recieves an HTTP 404 response code from the taxii2client should be returned as an exception""" + ds = TAXIICollectionStore(collection) + with pytest.raises(DataSourceError) as excinfo: ds.all_versions("indicator--1") assert "are either not found or access is denied" in str(excinfo.value) @@ -377,6 +381,7 @@ def test_all_versions_404(collection): def test_query_404(collection): """ a TAXIICollectionSource.query() call that recieves an HTTP 404 response code from the taxii2client should be returned as an exception""" + ds = TAXIICollectionStore(collection) query = [Filter("type", "=", "malware")] From 3fb184028924ac45ae7e8bf269a4282ac6464642 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 May 2018 10:03:06 -0400 Subject: [PATCH 8/8] safer error class --- stix2/datastore/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py index ac03959..c2963e2 100644 --- a/stix2/datastore/__init__.py +++ b/stix2/datastore/__init__.py @@ -40,8 +40,9 @@ class DataSourceError(Exception): def __str__(self): if self.root_exception: - self.message = "{} \"{}\"".format(self.message, self.root_exception) - return self.message + return "{} \"{}\"".format(self.message, self.root_exception) + else: + return self.message class DataStoreMixin(object): @@ -149,7 +150,7 @@ class DataStoreMixin(object): obj (STIX object OR dict OR str): The STIX object (or its ID) whose relationships will be looked up. relationship_type (str): Only retrieve Relationships of this type. - If None, all relationships will be returned, regardless of type. + If None, all relationships will be returned, regardless of type. source_only (bool): Only retrieve Relationships for which this object is the source_ref. Default: False. target_only (bool): Only retrieve Relationships for which this