diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index 7fc0997..49ea4fd 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -52,6 +52,19 @@ "print = json_print" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# without this configuration, only last print() call is outputted in cells\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "InteractiveShell.ast_node_interactivity = \"all\"" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -85,50 +98,229 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--797ae2b5-3f7a-44c5-8ecd-33ba22fdc2b5\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--11913f42-2d52-4b9d-842f-94bf06819a66\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "intrusion-set",\n",
+       "    "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:31:53.197Z",\n",
+       "    "modified": "2017-05-31T21:31:53.197Z",\n",
+       "    "name": "DragonOK",\n",
+       "    "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]",\n",
+       "    "aliases": [\n",
+       "        "DragonOK"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "mitre-attack",\n",
+       "            "url": "https://attack.mitre.org/wiki/Group/G0017",\n",
+       "            "external_id": "G0017"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Operation Quantum Entanglement",\n",
+       "            "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.",\n",
+       "            "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Symbiotic APT Groups",\n",
+       "            "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.",\n",
+       "            "url": "https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&D%20Track%20Insight%20into%20Symbiotic%20APT.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "New DragonOK",\n",
+       "            "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.",\n",
+       "            "url": "http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/"\n",
+       "        }\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--02b90f02-a96a-43ee-88f1-1e87297941f2",\n",
+       "    "created": "2017-11-13T07:00:24.000Z",\n",
+       "    "modified": "2017-11-13T07:00:24.000Z",\n",
+       "    "name": "Ransomware IP Blocklist",\n",
+       "    "description": "IP Blocklist address from abuse.ch",\n",
+       "    "pattern": "[ ipv4-addr:value = '91.237.247.24' ]",\n",
+       "    "valid_from": "2017-11-13T07:00:24Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity",\n",
+       "        "Ransomware",\n",
+       "        "Botnet",\n",
+       "        "C&C"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "abuse.ch",\n",
+       "            "url": "https://ransomwaretracker.abuse.ch/blocklist/"\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -136,22 +328,22 @@ "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n", "\n", "# create FileSystemStore\n", - "fs = FileSystemSource(\"/tmp/stix2_data\")\n", + "fs = FileSystemSource(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/\")\n", "\n", "# create TAXIICollectionSource\n", - "colxn = Collection('https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/')\n", + "colxn = Collection('https://test.freetaxii.com:8000/osint/collections/a9c22eaf-0f3e-482c-8bb4-45ae09e75d9b/')\n", "ts = TAXIICollectionSource(colxn)\n", "\n", "# add them both to the CompositeDataSource\n", "cs = CompositeDataSource()\n", - "cs.add_data_sources([fs, ts])\n", + "cs.add_data_sources([fs,ts])\n", "\n", "# get an object that is only in the filesystem\n", - "ta = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", - "print(ta)\n", + "intrusion_set = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", + "print(intrusion_set)\n", "\n", "# get an object that is only in the TAXII collection\n", - "ind = cs.get('indicator--37a6a5de-a5b9-425a-903a-4ae9cbf1ff3f')\n", + "ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n", "print(ind)\n" ] }, @@ -197,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": true }, @@ -231,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": true }, @@ -266,9 +458,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "cti-python-stix2", "language": "python", - "name": "python2" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 8eb5069..512fb58 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,6 +1,7 @@ """ Python STIX 2.x TAXIICollectionStore """ +from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse @@ -121,9 +122,13 @@ class TAXIICollectionSource(DataSource): # dont extract TAXII filters from query (to send to TAXII endpoint) # as directly retrieveing a STIX object by ID - stix_objs = self.collection.get_object(stix_id)["objects"] + try: + stix_objs = self.collection.get_object(stix_id)["objects"] + stix_obj = list(apply_common_filters(stix_objs, query)) - stix_obj = list(apply_common_filters(stix_objs, query)) + except HTTPError: + # if resource not found or access is denied from TAXII server, return None + stix_obj = [] if len(stix_obj): stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) @@ -209,13 +214,18 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters)["objects"] + try: + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] - # deduplicate data (before filtering as reduces wasted filtering) - all_data = deduplicate(all_data) + # deduplicate data (before filtering as reduces wasted filtering) + all_data = deduplicate(all_data) - # apply local (CompositeDataSource, TAXIICollectionSource and query filters) - all_data = list(apply_common_filters(all_data, query)) + # apply local (CompositeDataSource, TAXIICollectionSource and query filters) + all_data = list(apply_common_filters(all_data, query)) + + except HTTPError: + # if resources not found or access is denied from TAXII server, return empty list + all_data = [] # parse python STIX objects from the STIX object dicts stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] diff --git a/stix2/utils.py b/stix2/utils.py index f23dbe2..f0d43f1 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,5 +1,5 @@ """Utility functions and classes for the stix2 library.""" - +# HACK from collections import Mapping import copy import datetime as dt