diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index ba8ad53..31635ba 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -52,6 +52,19 @@ "print = json_print" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# without this configuration, only last print() call is outputted in cells\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "InteractiveShell.ast_node_interactivity = \"all\"" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -85,50 +98,229 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--797ae2b5-3f7a-44c5-8ecd-33ba22fdc2b5\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n", - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--11913f42-2d52-4b9d-842f-94bf06819a66\",\n", - " \"created\": \"2017-10-04T19:27:41.000Z\",\n", - " \"modified\": \"2017-10-04T19:27:41.000Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"name\": \"Emerging Threats - Block Rules - Compromised IPs\",\n", - " \"pattern\": \"[ ipv4-addr:value = '98.138.19.88' ]\",\n", - " \"valid_from\": \"2017-10-04T19:27:41Z\",\n", - " \"kill_chain_phases\": [\n", - " {\n", - " \"kill_chain_name\": \"lockheed-martin-cyber-kill-chain\",\n", - " \"phase_name\": \"delivery\"\n", - " }\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "intrusion-set",\n",
+       "    "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a",\n",
+       "    "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",\n",
+       "    "created": "2017-05-31T21:31:53.197Z",\n",
+       "    "modified": "2017-05-31T21:31:53.197Z",\n",
+       "    "name": "DragonOK",\n",
+       "    "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]",\n",
+       "    "aliases": [\n",
+       "        "DragonOK"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "mitre-attack",\n",
+       "            "url": "https://attack.mitre.org/wiki/Group/G0017",\n",
+       "            "external_id": "G0017"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Operation Quantum Entanglement",\n",
+       "            "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.",\n",
+       "            "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "Symbiotic APT Groups",\n",
+       "            "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.",\n",
+       "            "url": "https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&D%20Track%20Insight%20into%20Symbiotic%20APT.pdf"\n",
+       "        },\n",
+       "        {\n",
+       "            "source_name": "New DragonOK",\n",
+       "            "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.",\n",
+       "            "url": "http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/"\n",
+       "        }\n",
+       "    ],\n",
+       "    "object_marking_refs": [\n",
+       "        "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--02b90f02-a96a-43ee-88f1-1e87297941f2",\n",
+       "    "created": "2017-11-13T07:00:24.000Z",\n",
+       "    "modified": "2017-11-13T07:00:24.000Z",\n",
+       "    "name": "Ransomware IP Blocklist",\n",
+       "    "description": "IP Blocklist address from abuse.ch",\n",
+       "    "pattern": "[ ipv4-addr:value = '91.237.247.24' ]",\n",
+       "    "valid_from": "2017-11-13T07:00:24Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity",\n",
+       "        "Ransomware",\n",
+       "        "Botnet",\n",
+       "        "C&C"\n",
+       "    ],\n",
+       "    "external_references": [\n",
+       "        {\n",
+       "            "source_name": "abuse.ch",\n",
+       "            "url": "https://ransomwaretracker.abuse.ch/blocklist/"\n",
+       "        }\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -136,22 +328,22 @@ "from stix2 import CompositeDataSource, FileSystemSource, TAXIICollectionSource\n", "\n", "# create FileSystemStore\n", - "fs = FileSystemSource(\"/tmp/stix2_data\")\n", + "fs = FileSystemSource(\"/home/michael/cti-python-stix2/stix2/test/stix2_data/\")\n", "\n", "# create TAXIICollectionSource\n", - "colxn = Collection('https://test.freetaxii.com:8000/api1/collections/9cfa669c-ee94-4ece-afd2-f8edac37d8fd/')\n", + "colxn = Collection('https://test.freetaxii.com:8000/osint/collections/a9c22eaf-0f3e-482c-8bb4-45ae09e75d9b/')\n", "ts = TAXIICollectionSource(colxn)\n", "\n", "# add them both to the CompositeDataSource\n", "cs = CompositeDataSource()\n", - "cs.add_data_sources([fs, ts])\n", + "cs.add_data_sources([fs,ts])\n", "\n", "# get an object that is only in the filesystem\n", - "ta = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", - "print(ta)\n", + "intrusion_set = cs.get('intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a')\n", + "print(intrusion_set)\n", "\n", "# get an object that is only in the TAXII collection\n", - "ind = cs.get('indicator--37a6a5de-a5b9-425a-903a-4ae9cbf1ff3f')\n", + "ind = cs.get('indicator--02b90f02-a96a-43ee-88f1-1e87297941f2')\n", "print(ind)\n" ] }, @@ -197,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": true }, @@ -231,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": true }, @@ -537,9 +729,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "cti-python-stix2", "language": "python", - "name": "python2" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index 4b5bd6f..f494e6e 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -82,7 +82,7 @@ " /STIX2 Domain Object type\n", "```\n", "\n", - "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", + "The master STIX2 content directory contains subdirectories, each of which aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\", etc.). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", "\n", "```\n", "stix2_content/\n", @@ -109,11 +109,11 @@ "\n", "[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n", "\n", - "Use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", + "For use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. They can also be used individually when STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", "\n", "### FileSystem API\n", "\n", - "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query). The format of the STIX2 content targeted by the FileSystem suite is JSON files. When STIX2 content (in JSON) is retrieved by the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", + "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query): The format of the STIX2 content targeted by the FileSystem suite is JSON files. When the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) retrieves STIX2 content (in JSON) from disk, it will attempt to parse the content into full-featured python-stix2 objects and returned as such. \n", "\n", "A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", "\n", @@ -264,7 +264,7 @@ "source": [ "from stix2 import FileSystemSource\n", "\"\"\"\n", - "Working with FileSystemSource for retrieveing STIX content.\n", + "Working with FileSystemSource for retrieving STIX content.\n", "\"\"\"\n", "\n", "# create FileSystemSource\n", diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb index 75c0475..d651525 100644 --- a/docs/guide/memory.ipynb +++ b/docs/guide/memory.ipynb @@ -62,10 +62,11 @@ "\n", "\n", "### Memory API\n", + "A note on adding and retreiving STIX content to the Memory suite: As mentioned, under the hood the Memory suite is an internal, in-memory dictionary. STIX content that is to be added can be in the following forms: python-stix2 objects, (Python) dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python-stix2 objects or as (Python) dictionaries, reducing and converting any of the aforementioned types to one of those. Additionally, whatever form the STIX object is stored as, is how it will be returned when retrieved. python-stix2 objects, and json-encoded strings (of STIX content) are stored as python-stix2 objects, while (Python) dictionaries (of STIX objects) are stored as (Python) dictionaries.\n", "\n", - "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) and [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). These methods both add STIX content to an internal dictionary (maintained by [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore)). STIX content that is to be added can be in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as, is how it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dictionaries (of STIX objects) are stored as Python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). Also, for [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, individually or in a Bundle. \n", + "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file): For [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, as an individual STIX object or in a Bundle. When the JSON is loaded, the STIX objects are parsed into python-stix2 objects before being stored in the in-memory dictionary.\n", "\n", - "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", + "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file): This method dumps all STIX content that is in the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored in (i.e. supplied to) the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n", "\n", "### Memory Examples\n", "\n", @@ -74,26 +75,101 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--d91ef175-8a82-470a-a610-bbd2ee8a1516\",\n", - " \"created\": \"2017-09-29T19:52:16.930Z\",\n", - " \"modified\": \"2017-09-29T19:52:16.930Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"description\": \"Crusades C2 implant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']\",\n", - " \"valid_from\": \"2017-09-29T19:52:16.930909Z\"\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--2f61e4e7-0891-4e09-b79a-66f5e594fec0",\n",
+       "    "created": "2017-11-17T17:01:31.590Z",\n",
+       "    "modified": "2017-11-17T17:01:31.590Z",\n",
+       "    "description": "Crusades C2 implant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '54b7e05e39a59428743635242e4a867c932140a999f52a1e54fa7ee6a440c73b']",\n",
+       "    "valid_from": "2017-11-17T17:01:31.590939Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -115,26 +191,101 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"indicator\",\n", - " \"id\": \"indicator--79fdaad7-c461-49bb-ad1d-caa5e9c51c90\",\n", - " \"created\": \"2017-09-29T19:52:17.021Z\",\n", - " \"modified\": \"2017-09-29T19:52:17.021Z\",\n", - " \"labels\": [\n", - " \"malicious-activity\"\n", - " ],\n", - " \"description\": \"Crusades stage 2 implant variant\",\n", - " \"pattern\": \"[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']\",\n", - " \"valid_from\": \"2017-09-29T19:52:17.021728Z\"\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--ddb765ba-ff1e-4285-bf33-1f6d08f583d6",\n",
+       "    "created": "2017-11-17T17:01:31.799Z",\n",
+       "    "modified": "2017-11-17T17:01:31.799Z",\n",
+       "    "description": "Crusades stage 2 implant variant",\n",
+       "    "pattern": "[file:hashes.'SHA-256' = '31a45e777e4d58b97f4c43e38006f8cd6580ddabc4037905b2fad734712b582c']",\n",
+       "    "valid_from": "2017-11-17T17:01:31.799228Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -157,89 +308,216 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": 5, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "-----------------------\n", - "{'name': 'Urban2', 'created': '2017-09-12T13:26:18.023Z', 'labels': ['rootkit'], 'modified': '2017-09-12T13:26:18.023Z', 'type': 'malware', 'id': 'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'}\n", - "-----------------------\n", - "{\n", - " \"type\": \"malware\",\n", - " \"id\": \"malware--2b3dd412-18a5-4e81-8742-4977068eb3eb\",\n", - " \"created\": \"2017-09-29T19:52:17.028Z\",\n", - " \"modified\": \"2017-09-29T19:52:17.028Z\",\n", - " \"name\": \"Alexios\",\n", - " \"labels\": [\n", - " \"rootkit\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "malware",\n",
+       "    "id": "malware--e8170e70-522f-4ec3-aa22-afb55bfad0b0",\n",
+       "    "created": "2017-11-17T17:01:31.806Z",\n",
+       "    "modified": "2017-11-17T17:01:31.806Z",\n",
+       "    "name": "Alexios",\n",
+       "    "labels": [\n",
+       "        "rootkit"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "from stix2 import Filter\n", "\n", - "# add dictionary (of STIX object) to MemoryStore\n", - "# (this dict would assumably come from output of another source,\n", - "# i.e. a loaded json file, NOT manually created as done here for sample purposes)\n", - "\n", - "malware = {\n", - " \"type\": \"malware\",\n", - " \"id\" : \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\",\n", - " \"labels\": [\"rootkit\"],\n", - " \"name\": \"Urban2\",\n", - " \"created\": \"2017-09-12T13:26:18.023Z\",\n", - " \"modified\": \"2017-09-12T13:26:18.023Z\"\n", - "}\n", - "\n", - "mem.add(malware)\n", - "\n", - "results = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])\n", - "for r in results:\n", - " # note that python STIX objects are pretty-printed\n", - " # due to some python dunder method magic, but normal\n", - " # python dictionaries are not by default. Thus the\n", - " # python STIX objects and python STIX dictionaries\n", - " # that match the above query can be easily identified visually\n", - " print(\"-----------------------\")\n", - " print(r)" + "mal = mem.query([Filter(\"labels\",\"=\", \"rootkit\")])[0]\n", + "print(mal)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"report\",\n", - " \"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\n", - " \"created\": \"2017-05-08T18:34:08.042Z\",\n", - " \"modified\": \"2017-05-08T18:34:08.042Z\",\n", - " \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\",\n", - " \"published\": \"2017-05-08T10:24:11.011Z\",\n", - " \"object_refs\": [\n", - " \"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"\n", - " ],\n", - " \"labels\": [\n", - " \"threat-report\"\n", - " ]\n", - "}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "report",\n",
+       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
+       "    "created": "2017-05-08T18:34:08.042Z",\n",
+       "    "modified": "2017-05-08T18:34:08.042Z",\n",
+       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
+       "    "published": "2017-05-08T10:24:11.011Z",\n",
+       "    "object_refs": [\n",
+       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
+       "    ],\n",
+       "    "labels": [\n",
+       "        "threat-report"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "from stix2 import Filter\n", "\n", "# add json formatted string to MemoryStore\n", - "# Again, would NOT manual create json-formatted string\n", + "# Again, would NOT manually create json-formatted string\n", "# but taken as an output form from another source\n", "report = '{\"type\": \"report\",\"id\": \"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\",\"labels\": [\"threat-report\"], \"name\": \"The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.\", \"published\": \"2017-05-08T10:24:11.011Z\", \"object_refs\":[\"malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4\"], \"created\": \"2017-05-08T18:34:08.042Z\", \"modified\": \"2017-05-08T18:34:08.042Z\"}'\n", "\n", @@ -257,15 +535,103 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{u'name': u'The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.', u'created': u'2017-05-08T18:34:08.042Z', u'labels': [u'threat-report'], u'modified': u'2017-05-08T18:34:08.042Z', u'object_refs': [u'malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4'], u'published': u'2017-05-08T10:24:11.011Z', u'type': u'report', u'id': u'report--2add14d6-bbf3-4308-bb8e-226d314a08e4'}\n" - ] + "data": { + "text/html": [ + "
{\n",
+       "    "type": "report",\n",
+       "    "id": "report--2add14d6-bbf3-4308-bb8e-226d314a08e4",\n",
+       "    "created": "2017-05-08T18:34:08.042Z",\n",
+       "    "modified": "2017-05-08T18:34:08.042Z",\n",
+       "    "name": "The Crusades: Looking into the relentless infiltration of Israels digital infrastructure.",\n",
+       "    "published": "2017-05-08T10:24:11.011Z",\n",
+       "    "object_refs": [\n",
+       "        "malware--2daa14d6-cbf3-4308-bb8e-226d324a08e4"\n",
+       "    ],\n",
+       "    "labels": [\n",
+       "        "threat-report"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -280,17 +646,15 @@ "report = mem_2.get(\"report--2add14d6-bbf3-4308-bb8e-226d314a08e4\")\n", "\n", "# for visualpurposes\n", - "# Note: Since STIX content was added to MemoryStore as json,\n", - "# it is maintained as python dictionaries ( as opposed to STIX objects)\n", "print(report)" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "cti-python-stix2", "language": "python", - "name": "python2" + "name": "cti-python-stix2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 2f8905b..b0f0cea 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -58,9 +58,9 @@ "source": [ "## TAXIICollection\n", "\n", - "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) for pushing and retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) for retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", + "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) pushes and retrieves STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) retrieves STIX content from local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) pushes STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", "\n", - "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", + "A note on TAXII2 searching/filtering of STIX content: TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", "\n", "### TAXIICollection API\n", "\n", diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index e92c525..f4311be 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -1,8 +1,6 @@ """ Python STIX 2.0 FileSystem Source/Sink -TODO: - Test everything """ import json @@ -22,7 +20,12 @@ class FileSystemStore(DataStore): Args: stix_dir (str): path to directory of STIX objects - bundlify (bool): Whether to wrap objects in bundles when saving them. + allow_custom (bool): whether to allow custom STIX content to be + pushed/retrieved. Defaults to True for FileSystemSource side(retrieving data) + and False for FileSystemSink side(pushing data). However, when + parameter is supplied, it will be applied to both FileSystemSource + and FileSystemSink. + bundlify (bool): whether to wrap objects in bundles when saving them. Default: False. Attributes: @@ -30,10 +33,16 @@ class FileSystemStore(DataStore): sink (FileSystemSink): FileSystemSink """ - def __init__(self, stix_dir, bundlify=False): + def __init__(self, stix_dir, allow_custom=None, bundlify=False): + if allow_custom is None: + allow_custom_source = True + allow_custom_sink = False + else: + allow_custom_sink = allow_custom_source = allow_custom + super(FileSystemStore, self).__init__( - source=FileSystemSource(stix_dir=stix_dir), - sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + source=FileSystemSource(stix_dir=stix_dir, allow_custom=allow_custom_source), + sink=FileSystemSink(stix_dir=stix_dir, allow_custom=allow_custom_sink, bundlify=bundlify) ) @@ -46,13 +55,16 @@ class FileSystemSink(DataSink): Args: stix_dir (str): path to directory of STIX objects. + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSource. Default: False bundlify (bool): Whether to wrap objects in bundles when saving them. Default: False. """ - def __init__(self, stix_dir, bundlify=False): + def __init__(self, stix_dir, allow_custom=False, bundlify=False): super(FileSystemSink, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.allow_custom = allow_custom self.bundlify = bundlify if not os.path.exists(self._stix_dir): @@ -71,20 +83,18 @@ class FileSystemSink(DataSink): os.makedirs(os.path.dirname(path)) if self.bundlify: - stix_obj = Bundle(stix_obj) + stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom) with open(path, "w") as f: f.write(str(stix_obj)) - def add(self, stix_data=None, allow_custom=False, version=None): + def add(self, stix_data=None, version=None): """Add STIX objects to file directory. Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or list of), dict (or list of), or a STIX 2.0 json encoded string. - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -100,24 +110,24 @@ class FileSystemSink(DataSink): self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) + stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) if stix_data["type"] == "bundle": # extract STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) else: # adding json-formatted STIX - self._check_path_and_write(stix_data) + self._check_path_and_write(stix_data,) elif isinstance(stix_data, Bundle): # recursively add individual STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects for stix_obj in stix_data: - self.add(stix_obj, allow_custom=allow_custom, version=version) + self.add(stix_obj, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), " @@ -134,11 +144,14 @@ class FileSystemSource(DataSource): Args: stix_dir (str): path to directory of STIX objects + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSink. Default: True """ - def __init__(self, stix_dir): + def __init__(self, stix_dir, allow_custom=True): super(FileSystemSource, self).__init__() self._stix_dir = os.path.abspath(stix_dir) + self.allow_custom = allow_custom if not os.path.exists(self._stix_dir): raise ValueError("directory path for STIX data does not exist: %s" % self._stix_dir) @@ -147,15 +160,13 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -167,7 +178,7 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + all_data = self.query(query=query, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -176,7 +187,7 @@ class FileSystemSource(DataSource): return stix_obj - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions @@ -186,8 +197,6 @@ class FileSystemSource(DataSource): stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -197,9 +206,9 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)] + return [self.get(stix_id=stix_id, version=version, _composite_filters=_composite_filters)] - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + def query(self, query=None, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -210,8 +219,6 @@ class FileSystemSource(DataSource): query (list): list of filters to search on _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -221,6 +228,7 @@ class FileSystemSource(DataSource): parsed into a python STIX objects and then returned. """ + all_data = [] if query is None: @@ -304,7 +312,7 @@ class FileSystemSource(DataSource): all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 308d0d0..5d08d7c 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -1,9 +1,6 @@ """ Python STIX 2.0 Memory Source/Sink -TODO: - Run through tests again, lot of changes. - TODO: Use deduplicate() calls only when memory corpus is dirty (been added to) can save a lot of time for successive queries @@ -24,7 +21,7 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None, allow_custom=False, version=None): +def _add(store, stix_data=None, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. @@ -32,8 +29,6 @@ def _add(store, stix_data=None, allow_custom=False, version=None): Args: stix_data (list OR dict OR STIX object): STIX objects to be added - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -46,28 +41,19 @@ def _add(store, stix_data=None, allow_custom=False, version=None): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom, version=version) + _add(store, stix_obj, version=version) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data - elif isinstance(stix_data, str): - # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) - if stix_data["type"] == "bundle": - # recurse on each STIX object in bundle - for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom, version=version) - else: - _add(store, stix_data, allow_custom=allow_custom, version=version) - elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj, allow_custom=allow_custom, version=version) + _add(store, stix_obj, version=version) else: - raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") + raise TypeError("stix_data expected to be a python-stix2 object (or list of), JSON formatted STIX (or list of)," + " or a JSON formatted STIX bundle. stix_data was of type: " + str(type(stix_data))) class MemoryStore(DataStore): @@ -81,8 +67,9 @@ class MemoryStore(DataStore): Args: stix_data (list OR dict OR STIX object): STIX content to be added - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom STIX content. + Only applied when export/input functions called, i.e. + load_from_file() and save_to_file(). Defaults to True. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -92,11 +79,11 @@ class MemoryStore(DataStore): sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False, version=None): + def __init__(self, stix_data=None, allow_custom=True, version=None): self._data = {} if stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) super(MemoryStore, self).__init__( source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True), @@ -109,8 +96,6 @@ class MemoryStore(DataStore): Args: file_path (str): file path to write STIX data to - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. """ return self.sink.save_to_file(*args, **kwargs) @@ -123,8 +108,6 @@ class MemoryStore(DataStore): Args: file_path (str): file path to load STIX data from - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -141,37 +124,39 @@ class MemorySink(DataSink): Args: stix_data (dict OR list): valid STIX 2.0 content in bundle or a list. - _store (bool): if the MemorySink is a part of a DataStore, + _store (bool): whether the MemorySink is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSource. Not user supplied - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom objects/properties + when exporting STIX content to file. + Default: True. Attributes: _data (dict): the in-memory dict that holds STIX objects. - If apart of a MemoryStore, dict is shared between with - a MemorySource + If part of a MemoryStore, the dict is shared with a MemorySource """ - def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): + def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): super(MemorySink, self).__init__() self._data = {} + self.allow_custom = allow_custom if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) - def add(self, stix_data, allow_custom=False, version=None): - _add(self, stix_data, allow_custom=allow_custom, version=version) + def add(self, stix_data, version=None): + _add(self, stix_data, version=version) add.__doc__ = _add.__doc__ - def save_to_file(self, file_path, allow_custom=False): + def save_to_file(self, file_path): file_path = os.path.abspath(file_path) + if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(list(self._data.values()), allow_custom=allow_custom))) + f.write(str(Bundle(list(self._data.values()), allow_custom=self.allow_custom))) save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ @@ -188,23 +173,24 @@ class MemorySource(DataSource): _store (bool): if the MemorySource is a part of a DataStore, in which case "stix_data" is a direct reference to shared memory with DataSink. Not user supplied - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + allow_custom (bool): whether to allow custom objects/properties + when importing STIX content from file. + Default: True. Attributes: _data (dict): the in-memory dict that holds STIX objects. - If apart of a MemoryStore, dict is shared between with - a MemorySink + If part of a MemoryStore, the dict is shared with a MemorySink """ - def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): + def __init__(self, stix_data=None, allow_custom=True, version=None, _store=False): super(MemorySource, self).__init__() self._data = {} + self.allow_custom = allow_custom if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom, version=version) + _add(self, stix_data, version=version) def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. @@ -260,6 +246,7 @@ class MemorySource(DataSource): is returned in the same form as it as added """ + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] def query(self, query=None, _composite_filters=None): @@ -301,8 +288,12 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path, allow_custom=False, version=None): - file_path = os.path.abspath(file_path) - stix_data = json.load(open(file_path, "r")) - _add(self, stix_data, allow_custom=allow_custom, version=version) + def load_from_file(self, file_path, version=None): + stix_data = json.load(open(os.path.abspath(file_path), "r")) + + if stix_data["type"] == "bundle": + for stix_obj in stix_data["objects"]: + _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=stix_data["spec_version"])) + else: + _add(self, stix_data=parse(stix_obj, allow_custom=self.allow_custom, version=version)) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 8eb5069..2d54725 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,6 +1,7 @@ """ Python STIX 2.x TAXIICollectionStore """ +from requests.exceptions import HTTPError from stix2.base import _STIXBase from stix2.core import Bundle, parse @@ -18,11 +19,23 @@ class TAXIICollectionStore(DataStore): Args: collection (taxii2.Collection): TAXII Collection instance + allow_custom (bool): whether to allow custom STIX content to be + pushed/retrieved. Defaults to True for TAXIICollectionSource + side(retrieving data) and False for TAXIICollectionSink + side(pushing data). However, when parameter is supplied, it will + be applied to both TAXIICollectionSource/Sink. + """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=None): + if allow_custom is None: + allow_custom_source = True + allow_custom_sink = False + else: + allow_custom_sink = allow_custom_source = allow_custom + super(TAXIICollectionStore, self).__init__( - source=TAXIICollectionSource(collection), - sink=TAXIICollectionSink(collection) + source=TAXIICollectionSource(collection, allow_custom=allow_custom_source), + sink=TAXIICollectionSink(collection, allow_custom=allow_custom_sink) ) @@ -32,48 +45,49 @@ class TAXIICollectionSink(DataSink): Args: collection (taxii2.Collection): TAXII2 Collection instance + allow_custom (bool): Whether to allow custom STIX content to be + added to the TAXIICollectionSink. Default: False """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=False): super(TAXIICollectionSink, self).__init__() self.collection = collection + self.allow_custom = allow_custom - def add(self, stix_data, allow_custom=False, version=None): + def add(self, stix_data, version=None): """Add/push STIX content to TAXII Collection endpoint Args: stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content in a STIX object (or Bundle), STIX onject dict (or Bundle dict), or a STIX 2.0 json encoded string, or list of any of the following - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. """ if isinstance(stix_data, _STIXBase): # adding python STIX object - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) elif isinstance(stix_data, dict): # adding python dict (of either Bundle or STIX obj) if stix_data["type"] == "bundle": bundle = stix_data else: - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj, allow_custom=allow_custom, version=version) + self.add(obj, version=version) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom, version=version) + stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: - bundle = dict(Bundle(stix_data, allow_custom=allow_custom)) + bundle = dict(Bundle(stix_data, allow_custom=self.allow_custom)) else: raise TypeError("stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle") @@ -87,13 +101,16 @@ class TAXIICollectionSource(DataSource): Args: collection (taxii2.Collection): TAXII Collection instance + allow_custom (bool): Whether to allow custom STIX content to be + added to the FileSystemSink. Default: True """ - def __init__(self, collection): + def __init__(self, collection, allow_custom=True): super(TAXIICollectionSource, self).__init__() self.collection = collection + self.allow_custom = allow_custom - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote STIX Collection endpoint. @@ -101,8 +118,6 @@ class TAXIICollectionSource(DataSource): stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -121,12 +136,16 @@ class TAXIICollectionSource(DataSource): # dont extract TAXII filters from query (to send to TAXII endpoint) # as directly retrieveing a STIX object by ID - stix_objs = self.collection.get_object(stix_id)["objects"] + try: + stix_objs = self.collection.get_object(stix_id)["objects"] + stix_obj = list(apply_common_filters(stix_objs, query)) - stix_obj = list(apply_common_filters(stix_objs, query)) + except HTTPError: + # if resource not found or access is denied from TAXII server, return None + stix_obj = [] if len(stix_obj): - stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) + stix_obj = parse(stix_obj[0], allow_custom=self.allow_custom, version=version) if stix_obj.id != stix_id: # check - was added to handle erroneous TAXII servers stix_obj = None @@ -135,7 +154,7 @@ class TAXIICollectionSource(DataSource): return stix_obj - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it @@ -143,8 +162,6 @@ class TAXIICollectionSource(DataSource): stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -158,17 +175,17 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters) + all_data = self.query(query=query, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json - all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] + all_data = [parse(stix_obj, allow_custom=self.allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + def query(self, query=None, version=None, _composite_filters=None): """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -179,8 +196,6 @@ class TAXIICollectionSource(DataSource): query (list): list of filters to search on _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. @@ -209,16 +224,21 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters)["objects"] + try: + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] - # deduplicate data (before filtering as reduces wasted filtering) - all_data = deduplicate(all_data) + # deduplicate data (before filtering as reduces wasted filtering) + all_data = deduplicate(all_data) - # apply local (CompositeDataSource, TAXIICollectionSource and query filters) - all_data = list(apply_common_filters(all_data, query)) + # apply local (CompositeDataSource, TAXIICollectionSource and query filters) + all_data = list(apply_common_filters(all_data, query)) + + except HTTPError: + # if resources not found or access is denied from TAXII server, return empty list + all_data = [] # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 68fc185..020fee5 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -364,7 +364,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id, allow_custom=True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -376,9 +376,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): allow_custom=True) bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle, allow_custom=True) + fs_store.add(bundle) - camp_r = fs_store.get(camp.id, allow_custom=True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -391,9 +391,9 @@ def test_filesystem_custom_object(fs_store): pass newobj = NewObj(property1='something') - fs_store.add(newobj, allow_custom=True) + fs_store.add(newobj) - newobj_r = fs_store.get(newobj.id, allow_custom=True) + newobj_r = fs_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index a7d88a8..ad78611 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -203,63 +203,12 @@ def test_memory_store_save_load_file(mem_store): shutil.rmtree(os.path.dirname(filename)) -def test_memory_store_add_stix_object_str(mem_store): - # add stix object string - camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" - camp_name = "Aurelius" - camp_alias = "Purple Robes" - camp = """{ - "name": "%s", - "type": "campaign", - "objective": "German and French Intelligence Services", - "aliases": ["%s"], - "id": "%s", - "created": "2017-05-31T21:31:53.197755Z" - }""" % (camp_name, camp_alias, camp_id) - - mem_store.add(camp) - - camp_r = mem_store.get(camp_id) - assert camp_r["id"] == camp_id - assert camp_r["name"] == camp_name - assert camp_alias in camp_r["aliases"] - - -def test_memory_store_add_stix_bundle_str(mem_store): - # add stix bundle string - camp_id = "campaign--133111b6-1112-4fb0-111b-b111107ca70a" - camp_name = "Atilla" - camp_alias = "Huns" - bund = """{ - "type": "bundle", - "id": "bundle--112211b6-1112-4fb0-111b-b111107ca70a", - "spec_version": "2.0", - "objects": [ - { - "name": "%s", - "type": "campaign", - "objective": "Bulgarian, Albanian and Romanian Intelligence Services", - "aliases": ["%s"], - "id": "%s", - "created": "2017-05-31T21:31:53.197755Z" - } - ] - }""" % (camp_name, camp_alias, camp_id) - - mem_store.add(bund) - - camp_r = mem_store.get(camp_id) - assert camp_r["id"] == camp_id - assert camp_r["name"] == camp_name - assert camp_alias in camp_r["aliases"] - - def test_memory_store_add_invalid_object(mem_store): ind = ('indicator', IND1) # tuple isn't valid with pytest.raises(TypeError) as excinfo: mem_store.add(ind) - assert 'stix_data must be' in str(excinfo.value) - assert 'a STIX object' in str(excinfo.value) + assert 'stix_data expected to be' in str(excinfo.value) + assert 'a python-stix2 object' in str(excinfo.value) assert 'JSON formatted STIX' in str(excinfo.value) assert 'JSON formatted STIX bundle' in str(excinfo.value) diff --git a/stix2/test/test_versioning.py b/stix2/test/test_versioning.py index 8695a30..233587e 100644 --- a/stix2/test/test_versioning.py +++ b/stix2/test/test_versioning.py @@ -88,11 +88,15 @@ def test_versioning_error_bad_modified_value(): assert excinfo.value.cls == stix2.Campaign assert excinfo.value.prop_name == "modified" - assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \ + "It cannot be equal, as according to STIX 2 specification, objects that are different " \ + "but have the same id and modified timestamp do not have defined consumer behavior." msg = "Invalid value for {0} '{1}': {2}" msg = msg.format(stix2.Campaign.__name__, "modified", - "The new modified datetime cannot be before the current modified datatime.") + "The new modified datetime cannot be before than or equal to the current modified datetime." + "It cannot be equal, as according to STIX 2 specification, objects that are different " + "but have the same id and modified timestamp do not have defined consumer behavior.") assert str(excinfo.value) == msg @@ -153,7 +157,9 @@ def test_versioning_error_dict_bad_modified_value(): assert excinfo.value.cls == dict assert excinfo.value.prop_name == "modified" - assert excinfo.value.reason == "The new modified datetime cannot be before the current modified datatime." + assert excinfo.value.reason == "The new modified datetime cannot be before than or equal to the current modified datetime." \ + "It cannot be equal, as according to STIX 2 specification, objects that are different " \ + "but have the same id and modified timestamp do not have defined consumer behavior." def test_versioning_error_dict_no_modified_value(): @@ -206,3 +212,33 @@ def test_revoke_invalid_cls(): stix2.utils.revoke(campaign_v1) assert 'cannot revoke object of this type' in str(excinfo.value) + + +def test_remove_custom_stix_property(): + mal = stix2.Malware(name="ColePowers", + labels=["rootkit"], + x_custom="armada", + allow_custom=True) + + mal_nc = stix2.utils.remove_custom_stix(mal) + + assert "x_custom" not in mal_nc + assert stix2.utils.parse_into_datetime(mal["modified"], precision="millisecond") < stix2.utils.parse_into_datetime(mal_nc["modified"], + precision="millisecond") + + +def test_remove_custom_stix_object(): + @stix2.CustomObject("x-animal", [ + ("species", stix2.properties.StringProperty(required=True)), + ("animal_class", stix2.properties.StringProperty()), + ]) + class Animal(object): + def __init__(self, animal_class=None, **kwargs): + if animal_class and animal_class not in ["mammal", "bird"]: + raise ValueError("Not a recognized class of animal") + + animal = Animal(species="lion", animal_class="mammal") + + nc = stix2.utils.remove_custom_stix(animal) + + assert nc is None diff --git a/stix2/utils.py b/stix2/utils.py index 541e6d8..73337d0 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -1,5 +1,4 @@ """Utility functions and classes for the stix2 library.""" - from collections import Mapping import copy import datetime as dt @@ -16,6 +15,9 @@ from .exceptions import (InvalidValueError, RevokeError, # timestamps in a single object, the timestamps will vary by a few microseconds. NOW = object() +# STIX object properties that cannot be modified +STIX_UNMOD_PROPERTIES = ["created", "created_by_ref", "id", "type"] + class STIXdatetime(dt.datetime): def __new__(cls, *args, **kwargs): @@ -215,7 +217,7 @@ def new_version(data, **kwargs): properties_to_change = kwargs.keys() # Make sure certain properties aren't trying to change - for prop in ["created", "created_by_ref", "id", "type"]: + for prop in STIX_UNMOD_PROPERTIES: if prop in properties_to_change: unchangable_properties.append(prop) if unchangable_properties: @@ -227,8 +229,11 @@ def new_version(data, **kwargs): elif 'modified' in data: old_modified_property = parse_into_datetime(data.get('modified'), precision='millisecond') new_modified_property = parse_into_datetime(kwargs['modified'], precision='millisecond') - if new_modified_property < old_modified_property: - raise InvalidValueError(cls, 'modified', "The new modified datetime cannot be before the current modified datatime.") + if new_modified_property <= old_modified_property: + raise InvalidValueError(cls, 'modified', + "The new modified datetime cannot be before than or equal to the current modified datetime." + "It cannot be equal, as according to STIX 2 specification, objects that are different " + "but have the same id and modified timestamp do not have defined consumer behavior.") new_obj_inner.update(kwargs) # Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass return cls(**{k: v for k, v in new_obj_inner.items() if v is not None}) @@ -257,5 +262,68 @@ def get_class_hierarchy_names(obj): return names +def remove_custom_stix(stix_obj): + """remove any custom STIX objects or properties + + Warning: This function is a best effort utility, in that + it will remove custom objects and properties based on the + type names; i.e. if "x-" prefixes object types, and "x_" + prefixes property types. According to the STIX2 spec, + those naming conventions are a SHOULDs not MUSTs, meaning + that valid custom STIX content may ignore those conventions + and in effect render this utility function invalid when used + on that STIX content. + + Args: + stix_obj (dict OR python-stix obj): a single python-stix object + or dict of a STIX object + + Returns: + A new version of the object with any custom content removed + """ + + if stix_obj["type"].startswith("x-"): + # if entire object is custom, discard + return None + + custom_props = [] + for prop in stix_obj.items(): + if prop[0].startswith("x_"): + # for every custom property, record it and set value to None + # (so we can pass it to new_version() and it will be dropped) + custom_props.append((prop[0], None)) + + if custom_props: + # obtain set of object properties that can be transferred + # to a new object version. This is 1)custom props with their + # values set to None, and 2)any properties left that are not + # unmodifiable STIX properties or the "modified" property + + # set of properties that are not supplied to new_version() + # to be used for updating properties. This includes unmodifiable + # properties (properties that new_version() just re-uses from the + # existing STIX object) and the "modified" property. We dont supply the + # "modified" property so that new_version() creates a new datetime + # value for this property + non_supplied_props = STIX_UNMOD_PROPERTIES + ["modified"] + + props = [(prop, stix_obj[prop]) for prop in stix_obj if prop not in non_supplied_props] + + # add to set the custom properties we want to get rid of (with their value=None) + props.extend(custom_props) + + new_obj = new_version(stix_obj, **(dict(props))) + + while parse_into_datetime(new_obj["modified"]) == parse_into_datetime(stix_obj["modified"]): + # Prevents bug when fast computation allows multiple STIX object + # versions to be created in single unit of time + new_obj = new_version(stix_obj, **(dict(props))) + + return new_obj + + else: + return stix_obj + + def get_type_from_id(stix_id): return stix_id.split('--', 1)[0]