Merge pull request #526 from chisholm/fix_taxii20_datasource
Fix TAXIICollectionSource.query() to propagate HTTPErrorsmain
commit
e64d23d1dd
|
@ -296,19 +296,10 @@ class TAXIICollectionSource(DataSource):
|
||||||
|
|
||||||
# query TAXII collection
|
# query TAXII collection
|
||||||
all_data = []
|
all_data = []
|
||||||
|
paged_request = tcv21.as_pages if isinstance(self.collection, tcv21.Collection) else tcv20.as_pages
|
||||||
try:
|
try:
|
||||||
paged_request = tcv21.as_pages if isinstance(self.collection, tcv21.Collection) else tcv20.as_pages
|
|
||||||
|
|
||||||
for resource in paged_request(self.collection.get_objects, per_request=self.items_per_page, **taxii_filters_dict):
|
for resource in paged_request(self.collection.get_objects, per_request=self.items_per_page, **taxii_filters_dict):
|
||||||
all_data.extend(resource.get("objects", []))
|
all_data.extend(resource.get("objects", []))
|
||||||
|
|
||||||
# deduplicate data (before filtering as reduces wasted filtering)
|
|
||||||
all_data = deduplicate(all_data)
|
|
||||||
|
|
||||||
# apply local (CompositeDataSource, TAXIICollectionSource and query) filters
|
|
||||||
query.remove(taxii_filters)
|
|
||||||
all_data = list(apply_common_filters(all_data, query))
|
|
||||||
|
|
||||||
except HTTPError as e:
|
except HTTPError as e:
|
||||||
# if resources not found or access is denied from TAXII server, return empty list
|
# if resources not found or access is denied from TAXII server, return empty list
|
||||||
if e.response.status_code == 404:
|
if e.response.status_code == 404:
|
||||||
|
@ -318,6 +309,21 @@ class TAXIICollectionSource(DataSource):
|
||||||
" denied. Received error: ", e,
|
" denied. Received error: ", e,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# TAXII 2.0 paging can result in a 416 (Range Not Satisfiable) if
|
||||||
|
# the server isn't sending Content-Range headers, so the pager just
|
||||||
|
# goes until it runs out of pages. So 416 can't be treated as a
|
||||||
|
# real error, just an end-of-pages condition. For other codes,
|
||||||
|
# propagate the exception.
|
||||||
|
elif e.response.status_code != 416:
|
||||||
|
raise
|
||||||
|
|
||||||
|
# deduplicate data (before filtering as reduces wasted filtering)
|
||||||
|
all_data = deduplicate(all_data)
|
||||||
|
|
||||||
|
# apply local (CompositeDataSource, TAXIICollectionSource and query) filters
|
||||||
|
query.remove(taxii_filters)
|
||||||
|
all_data = list(apply_common_filters(all_data, query))
|
||||||
|
|
||||||
# parse python STIX objects from the STIX object dicts
|
# parse python STIX objects from the STIX object dicts
|
||||||
stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
|
stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data]
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue