some progress on CompositeDS guide, but includes some bug fixes in the DataSource API

docs/guide/DataStore, CompositeDataSource, Filters.ipynb

@@ -55,9 +55,52 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"indicator\",\n",
+      "    \"id\": \"indicator--797ae2b5-3f7a-44c5-8ecd-33ba22fdc2b5\",\n",
+      "    \"created\": \"2017-10-04T19:27:41.000Z\",\n",
+      "    \"modified\": \"2017-10-04T19:27:41.000Z\",\n",
+      "    \"labels\": [\n",
+      "        \"malicious-activity\"\n",
+      "    ],\n",
+      "    \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n",
+      "    \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n",
+      "    \"valid_from\": \"2017-10-04T19:27:41Z\",\n",
+      "    \"kill_chain_phases\": [\n",
+      "        {\n",
+      "            \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n",
+      "            \"phase_name\": \"delivery\"\n",
+      "        }\n",
+      "    ]\n",
+      "}\n",
+      "{\n",
+      "    \"type\": \"indicator\",\n",
+      "    \"id\": \"indicator--11913f42-2d52-4b9d-842f-94bf06819a66\",\n",
+      "    \"created\": \"2017-10-04T19:27:41.000Z\",\n",
+      "    \"modified\": \"2017-10-04T19:27:41.000Z\",\n",
+      "    \"labels\": [\n",
+      "        \"malicious-activity\"\n",
+      "    ],\n",
+      "    \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n",
+      "    \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n",
+      "    \"valid_from\": \"2017-10-04T19:27:41Z\",\n",
+      "    \"kill_chain_phases\": [\n",
+      "        {\n",
+      "            \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n",
+      "            \"phase_name\": \"delivery\"\n",
+      "        }\n",
+      "    ]\n",
+      "}\n"
+     ]
+    }
+   ],
    "source": [
     "from taxii2client import Collection\n",
     "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n",
@@ -74,7 +117,7 @@
     "cs.add_data_sources([fs, ts])\n",
     "\n",
     "# get an object that is only in the filesystem\n",
-    "ta = cs.get('threat-actor--d10825d4-7b74-4cca-a0fc-7c097e08ccd2')\n",
+    "ta = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n",
     "print(ta)\n",
     "\n",
     "# get an object that is only in the TAXII collection\n",
@@ -178,7 +221,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from stix2 import MemoryStore, FileSystemStore, FileSystemSource\n",

stix2/sources/__init__.py

@@ -257,7 +257,8 @@ class CompositeDataSource(DataSource):
         # for every configured Data Source, call its retrieve handler
         for ds in self.data_sources:
             data = ds.get(stix_id=stix_id, _composite_filters=all_filters)
-            all_data.append(data)
+            if data:
+                all_data.append(data)
 
         # remove duplicate versions
         if len(all_data) > 0:

stix2/sources/filesystem.py

@@ -135,7 +135,7 @@ class FileSystemSource(DataSource):
         self._stix_dir = os.path.abspath(stix_dir)
 
         if not os.path.exists(self._stix_dir):
-            print("Error: directory path for STIX data does not exist")
+            raise ValueError("directory path for STIX data does not exist: %s" % self._stix_dir)
 
     @property
     def stix_dir(self):
@@ -160,9 +160,12 @@ class FileSystemSource(DataSource):
 
         all_data = self.query(query=query, _composite_filters=_composite_filters)
 
-        stix_obj = sorted(all_data, key=lambda k: k['modified'])[0]
+        if len(all_data):
+            stix_obj = parse(sorted(all_data, key=lambda k: k['modified'])[0])
+        else:
+            stix_obj = None
 
-        return parse(stix_obj)
+        return stix_obj
 
     def all_versions(self, stix_id, _composite_filters=None):
         """retrieve STIX object from file directory via STIX ID, all versions

stix2/sources/taxii.py

@@ -124,7 +124,6 @@ class TAXIICollectionSource(DataSource):
 
         # dont extract TAXII filters from query (to send to TAXII endpoint)
         # as directly retrieveing a STIX object by ID
-
         stix_objs = self.collection.get_object(stix_id)["objects"]
 
         stix_obj = [stix_obj for stix_obj in apply_common_filters(stix_objs, query)]