diff --git a/.travis.yml b/.travis.yml index aba764d..0d5a046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ language: python cache: pip python: - "2.7" - - "3.3" - "3.4" - "3.5" - "3.5-dev" @@ -16,6 +15,6 @@ install: - pip install codecov script: - tox - - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]]; then pre-commit run --all-files; fi + - pre-commit run --all-files after_success: - codecov diff --git a/CHANGELOG b/CHANGELOG index 650d372..c5ffabf 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,15 @@ CHANGELOG ========= +0.4.0 - 2017-11-13 + +* Adds `creator_of` function to easily get the Identity that created an object, + and a `serialize` function for fast serialization without sorting properties. +* Fixes bugs with DataStores and with adding custom STIX content to Bundles. +* Supports filtering on any property, not just common properties. +* Includes internal changes to make it easier to support multiple versions of + the STIX specification. + 0.3.0 - 2017-10-06 * Adds data stores, object factory, and the environment layer. diff --git a/docs/api/stix2.common.rst b/docs/api/stix2.common.rst deleted file mode 100644 index 9e90b03..0000000 --- a/docs/api/stix2.common.rst +++ /dev/null @@ -1,5 +0,0 @@ -common -============ - -.. automodule:: stix2.common - :members: \ No newline at end of file diff --git a/docs/api/stix2.observables.rst b/docs/api/stix2.observables.rst deleted file mode 100644 index 2d19996..0000000 --- a/docs/api/stix2.observables.rst +++ /dev/null @@ -1,5 +0,0 @@ -observables -================= - -.. automodule:: stix2.observables - :members: \ No newline at end of file diff --git a/docs/api/stix2.sdo.rst b/docs/api/stix2.sdo.rst deleted file mode 100644 index 9448ff4..0000000 --- a/docs/api/stix2.sdo.rst +++ /dev/null @@ -1,5 +0,0 @@ -sdo -========= - -.. automodule:: stix2.sdo - :members: \ No newline at end of file diff --git a/docs/api/stix2.sro.rst b/docs/api/stix2.sro.rst deleted file mode 100644 index a63af76..0000000 --- a/docs/api/stix2.sro.rst +++ /dev/null @@ -1,5 +0,0 @@ -sro -========= - -.. automodule:: stix2.sro - :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.common.rst b/docs/api/stix2.v20.common.rst new file mode 100644 index 0000000..0c7a296 --- /dev/null +++ b/docs/api/stix2.v20.common.rst @@ -0,0 +1,5 @@ +common +================ + +.. automodule:: stix2.v20.common + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.observables.rst b/docs/api/stix2.v20.observables.rst new file mode 100644 index 0000000..d31f75f --- /dev/null +++ b/docs/api/stix2.v20.observables.rst @@ -0,0 +1,5 @@ +observables +===================== + +.. automodule:: stix2.v20.observables + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.sdo.rst b/docs/api/stix2.v20.sdo.rst new file mode 100644 index 0000000..c4c97f8 --- /dev/null +++ b/docs/api/stix2.v20.sdo.rst @@ -0,0 +1,5 @@ +sdo +============= + +.. automodule:: stix2.v20.sdo + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.sro.rst b/docs/api/stix2.v20.sro.rst new file mode 100644 index 0000000..379ed18 --- /dev/null +++ b/docs/api/stix2.v20.sro.rst @@ -0,0 +1,5 @@ +sro +============= + +.. automodule:: stix2.v20.sro + :members: \ No newline at end of file diff --git a/docs/api_ref.rst b/docs/api_ref.rst index 11bf278..bb60081 100644 --- a/docs/api_ref.rst +++ b/docs/api_ref.rst @@ -4,4 +4,9 @@ API Reference This section of documentation contains information on all of the classes and functions in the ``stix2`` API, as given by the package's docstrings. +.. note:: + All the classes and functions detailed in the pages below are importable + directly from `stix2`. See also: + :ref:`How imports will work `. + .. automodule:: stix2 diff --git a/docs/conf.py b/docs/conf.py index 6155be6..c337e79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,8 +28,8 @@ project = 'stix2' copyright = '2017, OASIS Open' author = 'OASIS Open' -version = '0.3.0' -release = '0.3.0' +version = '0.4.0' +release = '0.4.0' language = None exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store', 'guide/.ipynb_checkpoints'] diff --git a/docs/guide/creating.ipynb b/docs/guide/creating.ipynb index 6cfea7f..5836d1d 100644 --- a/docs/guide/creating.ipynb +++ b/docs/guide/creating.ipynb @@ -381,7 +381,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To update the properties of an object, see the **Versioning** section." + "To update the properties of an object, see the [Versioning](versioning.ipynb) section." ] }, { @@ -500,7 +500,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided." + "As with indicators, the ``type``, ``id``, ``created``, and ``modified`` properties will be set automatically if not provided. For Malware objects, the ``labels`` and ``name`` properties must be provided.\n", + "\n", + "You can see the full list of SDO classes [here](../api/stix2.v20.sdo.rst)." ] }, { @@ -869,9 +871,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 2254fa8..7f30401 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": { "collapsed": true, "nbsphinx": "hidden" @@ -99,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -174,13 +174,13 @@ ".highlight .vi { color: #19177C } /* Name.Variable.Instance */\n", ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
-       "    "x_foo": "bar",\n",
        "    "type": "identity",\n",
-       "    "id": "identity--8d7f0697-e589-4e3b-aa57-cae798d2d138",\n",
-       "    "created": "2017-09-26T21:02:19.465Z",\n",
-       "    "modified": "2017-09-26T21:02:19.465Z",\n",
+       "    "id": "identity--00c5743f-2d5e-4d66-88f1-1842584f4519",\n",
+       "    "created": "2017-11-09T16:17:44.596Z",\n",
+       "    "modified": "2017-11-09T16:17:44.596Z",\n",
        "    "name": "John Smith",\n",
-       "    "identity_class": "individual"\n",
+       "    "identity_class": "individual",\n",
+       "    "x_foo": "bar"\n",
        "}\n",
        "
\n" ], @@ -188,12 +188,14 @@ "" ] }, - "execution_count": 5, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "from stix2 import Identity\n", + "\n", "identity = Identity(name=\"John Smith\",\n", " identity_class=\"individual\",\n", " custom_properties={\n", @@ -317,7 +319,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to ``parse()``:" + "Likewise, when parsing STIX content with custom properties, pass ``allow_custom=True`` to [parse()](../api/stix2.core.rst#stix2.core.parse):" ] }, { @@ -355,7 +357,7 @@ "source": [ "### Custom STIX Object Types\n", "\n", - "To create a custom STIX object type, define a class with the ``@CustomObject`` decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n", + "To create a custom STIX object type, define a class with the @[CustomObject](../api/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject) decorator. It takes the type name and a list of property tuples, each tuple consisting of the property name and a property instance. Any special validation of the properties can be added by supplying an ``__init__`` function.\n", "\n", "Let's say zoo animals have become a serious cyber threat and we want to model them in STIX using a custom object type. Let's use a ``species`` property to store the kind of animal, and make that property required. We also want a property to store the class of animal, such as \"mammal\" or \"bird\" but only want to allow specific values in it. We can add some logic to validate this property in ``__init__``." ] @@ -586,7 +588,7 @@ "source": [ "### Custom Cyber Observable Types\n", "\n", - "Similar to custom STIX object types, use a decorator to create custom Cyber Observable types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary." + "Similar to custom STIX object types, use a decorator to create [custom Cyber Observable](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable) types. Just as before, ``__init__()`` can hold additional validation, but it is not necessary." ] }, { @@ -752,7 +754,7 @@ "source": [ "### Custom Cyber Observable Extensions\n", "\n", - "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the ``@CustomExtension`` decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:" + "Finally, custom extensions to existing Cyber Observable types can also be created. Just use the @[CustomExtension](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) decorator. Note that you must provide the Cyber Observable class to which the extension applies. Again, any extra validation of the properties can be implemented by providing an ``__init__()`` but it is not required. Let's say we want to make an extension to the ``File`` Cyber Observable Object:" ] }, { @@ -916,9 +918,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/datastore.ipynb b/docs/guide/datastore.ipynb index f6d360b..24a2b4f 100644 --- a/docs/guide/datastore.ipynb +++ b/docs/guide/datastore.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 40, "metadata": { "collapsed": true, "nbsphinx": "hidden" @@ -58,9 +58,9 @@ "source": [ "# DataStore API\n", "\n", - "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of DataStore, DataSource and DataSink constructs: a DataSource for pulling STIX2 content, a DataSink for pushing STIX2 content, and a DataStore for pulling/pushing.\n", + "CTI Python STIX2 features a new interface for pulling and pushing STIX2 content. The new interface consists of [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) constructs: a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) for pulling STIX2 content, a [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) for pushing STIX2 content, and a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) for both pulling and pushing.\n", "\n", - "The DataStore, DataSource, DataSink (referred to as \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then sublcassed into real DataStore suite(s). CTI Python STIX2 provides for the DataStore suites of **FileSystem**, **Memory**, and **TAXII**. Users are also encrouraged subclassing the base Data suite and creating their own custom DataStore suites." + "The DataStore, [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource), [DataSink](../api/stix2.sources.rst#stix2.sources.DataSink) (collectively referred to as the \"DataStore suite\") APIs are not referenced directly by a user but are used as base classes, which are then sublcassed by real DataStore suites. CTI Python STIX2 provides the DataStore suites of [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst). Users are also encouraged to subclass the base classes and create their own custom DataStore suites." ] }, { @@ -69,13 +69,13 @@ "source": [ "## CompositeDataSource\n", "\n", - "**CompositeDataSource** is an available controller that can be used as a single interface to a set of defined DataSources. The purpose of this controller is allow for the grouping of **DataSources** and making get/query calls to a set of DataSources in one API call. **CompositeDataSource** can be used to organize/group **DataSources**, federate get()/all_versions()/query() calls, and reduce user code.\n", + "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is an available controller that can be used as a single interface to a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). The purpose of this controller is allow for the grouping of [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) and making `get()`/`query()` calls to a set of DataSources in one API call. [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can be used to organize/group [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource), federate `get()`/`all_versions()`/`query()` calls, and reduce user code.\n", "\n", - "**CompositeDataSource** is just a wrapper around a set of defined **DataSources** (e.g. FileSystemSource) that federates get()/all_versions()/query() calls individually to each of the attached **DataSources** , collects the results from each **DataSource** and returns them.\n", + "[CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) is just a wrapper around a set of defined [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) (e.g. [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource)) that federates `get()`/`all_versions()`/`query()` calls individually to each of the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) , collects the results from each [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) and returns them.\n", "\n", - "Filters can be attached to **CompositeDataSources** just as they can be done to **DataStores** and **DataSources**. When get()/all_versions()/query() calls are made to the **CompositeDataSource**, it will pass along any query filters from the call and any of its own filters to the attached **DataSources**. To which, those attached **DataSources** may have their own attached filters as well. The effect is that all the filters are eventually combined when the get()/all_versions()/query() call is actually executed within a **DataSource**. \n", + "Filters can be attached to [CompositeDataSources](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) just as they can be done to [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore) and [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). When `get()`/`all_versions()`/`query()` calls are made to the [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource), it will pass along any query filters from the call and any of its own filters to the attached [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource). In addition, those [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) may have their own attached filters as well. The effect is that all the filters are eventually combined when the `get()`/`all_versions()`/`query()` call is actually executed within a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource). \n", "\n", - "A **CompositeDataSource** can also be attached to a **CompositeDataSource** for multiple layers of grouped **DataSources**.\n", + "A [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) can also be attached to a [CompositeDataSource](../api/stix2.sources.rst#stix2.sources.CompositeDataSource) for multiple layers of grouped [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource).\n", "\n", "\n", "### CompositeDataSource API\n", @@ -161,37 +161,18 @@ "source": [ "## Filters\n", "\n", - "The CTI Python STIX2 **DataStore** suites - **FileSystem**, **Memory** and **TAXII** - all use the **Filters** module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to **query()**, and/or attached to a **DataStore** so that every future query placed to that **DataStore** is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from **DataStores**.\n", + "The CTI Python STIX2 DataStore suites - [FileSystem](../api/sources/stix2.sources.filesystem.rst), [Memory](../api/sources/stix2.sources.memory.rst), and [TAXII](../api/sources/stix2.sources.taxii.rst) - all use the [Filters](../api/sources/stix2.sources.filters.rst) module to allow for the querying of STIX content. The basic functionality is that filters can be created and supplied everytime to calls to `query()`, and/or attached to a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) so that every future query placed to that [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) is evaluated against the attached filters, supplemented with any further filters supplied with the query call. Attached filters can also be removed from [DataStores](../api/stix2.sources.rst#stix2.sources.DataStore).\n", "\n", - "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). Currently, CTI Python STIX2 supports **ONLY** STIX 2 object common properties and TAXII2 Filtering parameters for fields to filter on:\n", + "Filters are very simple, as they consist of a field name, comparison operator and an object property value (i.e. value to compare to). All properties of STIX2 objects can be filtered on. In addition, TAXII2 Filtering parameters for fields can also be used in filters.\n", "\n", - "Fields\n", - "\n", - "(STIX2 Object Common Properties)\n", - "\n", - "* created\n", - "* created_by_ref\n", - "* external_references.source_name\n", - "* external_references.description\n", - "* external_references.url\n", - "* external_references.external_id\n", - "* granular_markings.marking_ref\n", - "* granular_markings.selectors\n", - "* id\n", - "* labels\n", - "* modified\n", - "* object_marking_refs\n", - "* revoked\n", - "* type\n", - "\n", - "(TAXII2 filter fields)\n", + "TAXII2 filter fields:\n", "\n", "* added_after\n", "* match[id]\n", "* match[type]\n", "* match[version]\n", "\n", - "Supported operators on above properties:\n", + "Supported operators:\n", "\n", "* =\n", "* !=\n", @@ -201,7 +182,7 @@ "* ```>=```\n", "* <=\n", "\n", - "Value types of the common property values must be one of these (python) types:\n", + "Value types of the property values must be one of these (Python) types:\n", "\n", "* bool\n", "* dict\n", @@ -245,7 +226,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For Filters to be applied to a query, they must be either supplied with the query call or attached a **DataStore**, more specifically to **DataSource** whether that **DataSource** is a part of a **DataStore** or stands by itself. " + "For Filters to be applied to a query, they must be either supplied with the query call or attached a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore), more specifically to a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) whether that [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource) is a part of a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) or stands by itself. " ] }, { @@ -281,25 +262,296 @@ "# attach multiple filters to a MemoryStore\n", "mem.source.filters.update([f1,f2])" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## De-Referencing Relationships\n", + "\n", + "Given a STIX object, there are several ways to find other STIX objects related to it. To illustrate this, let's first create a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) and add some objects and relationships." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from stix2 import Campaign, Identity, Indicator, Malware, Relationship\n", + "\n", + "mem = MemoryStore()\n", + "cam = Campaign(name='Charge', description='Attack!')\n", + "idy = Identity(name='John Doe', identity_class=\"individual\")\n", + "ind = Indicator(labels=['malicious-activity'], pattern=\"[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']\")\n", + "mal = Malware(labels=['ransomware'], name=\"Cryptolocker\", created_by_ref=idy)\n", + "rel1 = Relationship(ind, 'indicates', mal,)\n", + "rel2 = Relationship(mal, 'targets', idy)\n", + "rel3 = Relationship(cam, 'uses', mal)\n", + "mem.add([cam, idy, ind, mal, rel1, rel2, rel3])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If a STIX object has a `created_by_ref` property, you can use the [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of) method to retrieve the [Identity](../api/stix2.v20.sdo.rst#stix2.v20.sdo.Identity) object that created it." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "identity",\n",
+       "    "id": "identity--be3baac0-9aba-48a8-81e4-4408b1c379a8",\n",
+       "    "created": "2017-11-21T22:14:45.213Z",\n",
+       "    "modified": "2017-11-21T22:14:45.213Z",\n",
+       "    "name": "John Doe",\n",
+       "    "identity_class": "individual"\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(mem.creator_of(mal))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) method to retrieve all the relationship objects that reference a STIX object." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rels = mem.relationships(mal)\n", + "len(rels)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can limit it to only specific relationship types:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mem.relationships(mal, relationship_type='indicates')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can limit it to only relationships where the given object is the source:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Relationship(type='relationship', id='relationship--7eb7f5cd-8bf2-4f7c-8756-84c0b5693b9a', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'targets', source_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4', target_ref='identity--be3baac0-9aba-48a8-81e4-4408b1c379a8')]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mem.relationships(mal, source_only=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And you can limit it to only relationships where the given object is the target:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Relationship(type='relationship', id='relationship--bd6fd399-c907-4feb-b1da-b90f15942f1d', created='2017-11-21T22:14:45.214Z', modified='2017-11-21T22:14:45.214Z', relationship_type=u'indicates', source_ref='indicator--5ee33ff0-c50d-456b-a8dd-b5d1b69a66e8', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4'),\n", + " Relationship(type='relationship', id='relationship--3c759d40-c92a-430e-aab6-77d5c5763302', created='2017-11-21T22:14:45.215Z', modified='2017-11-21T22:14:45.215Z', relationship_type=u'uses', source_ref='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', target_ref='malware--66c0bc78-4e27-4d80-a565-a07e6eb6fba4')]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mem.relationships(mal, target_only=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, you can retrieve all STIX objects related to a given STIX object using [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to). This calls [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) but then performs the extra step of getting the objects that these Relationships point to. [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to) takes all the same arguments that [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) does." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Campaign(type='campaign', id='campaign--82ab7aa4-d13b-4e99-8a09-ebcba30668a7', created='2017-11-21T22:14:45.213Z', modified='2017-11-21T22:14:45.213Z', name=u'Charge', description=u'Attack!')]" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mem.related_to(mal, target_only=True, relationship_type='uses')" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/environment.ipynb b/docs/guide/environment.ipynb index a9184a1..0cb5796 100644 --- a/docs/guide/environment.ipynb +++ b/docs/guide/environment.ipynb @@ -58,11 +58,11 @@ "source": [ "## Using Environments\n", "\n", - "An ``Environment`` object makes it easier to use STIX 2 content as part of a larger application or ecosystem. It allows you to abstract away the nasty details of sending and receiving STIX data, and to create STIX objects with default values for common properties.\n", + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) object makes it easier to use STIX 2 content as part of a larger application or ecosystem. It allows you to abstract away the nasty details of sending and receiving STIX data, and to create STIX objects with default values for common properties.\n", "\n", "### Storing and Retrieving STIX Content\n", "\n", - "An ``Environment`` can be set up with a ``DataStore`` if you want to store and retrieve STIX content from the same place. " + "An [Environment](../api/stix2.environment.rst#stix2.environment.Environment) can be set up with a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore) if you want to store and retrieve STIX content from the same place. " ] }, { @@ -82,7 +82,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If desired, you can instead set up an ``Environment`` with different data sources and sinks. In the following example we set up an environment that retrieves objects from memory and a directory on the filesystem, and stores objects in a different directory on the filesystem." + "If desired, you can instead set up an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with different data sources and sinks. In the following example we set up an environment that retrieves objects from [memory](../api/sources/stix2.sources.memory.rst) and a directory on the [filesystem](../api/sources/stix2.sources.filesystem.rst), and stores objects in a different directory on the filesystem." ] }, { @@ -105,13 +105,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once you have an ``Environment`` you can store some STIX content in its DataSinks with ``add()``:" + "Once you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) you can store some STIX content in its [DataSinks](../api/stix2.sources.rst#stix2.sources.DataSink) with [add()](../api/stix2.environment.rst#stix2.environment.Environment.add):" ] }, { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import Indicator\n", @@ -126,7 +128,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You can retrieve STIX objects from the DataSources in the Environment with ``get()``, ``query()``, and ``all_versions()``, just as you would for a DataSource." + "You can retrieve STIX objects from the [DataSources](../api/stix2.sources.rst#stix2.sources.DataSource) in the [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with [get()](../api/stix2.environment.rst#stix2.environment.Environment.get), [query()](../api/stix2.environment.rst#stix2.environment.Environment.query), [all_versions()](../api/stix2.environment.rst#stix2.environment.Environment.all_versions), [creator_of()](../api/stix2.sources.rst#stix2.sources.DataSource.creator_of), [related_to()](../api/stix2.sources.rst#stix2.sources.DataSource.related_to), and [relationships()](../api/stix2.sources.rst#stix2.sources.DataSource.relationships) just as you would for a [DataSource](../api/stix2.sources.rst#stix2.sources.DataSource)." ] }, { @@ -237,7 +239,7 @@ "source": [ "### Creating STIX Objects With Defaults\n", "\n", - "To create STIX objects with default values for certain properties, use an ``ObjectFactory``. For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization." + "To create STIX objects with default values for certain properties, use an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory). For instance, say we want all objects we create to have a ``created_by_ref`` property pointing to the ``Identity`` object representing our organization." ] }, { @@ -259,7 +261,7 @@ "collapsed": true }, "source": [ - "Once you've set up the Object Factory, use its ``create()`` method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object." + "Once you've set up the [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory), use its [create()](../api/stix2.environment.rst#stix2.environment.ObjectFactory.create) method, passing in the class for the type of object you wish to create, followed by the other properties and their values for the object." ] }, { @@ -374,14 +376,14 @@ "collapsed": true }, "source": [ - "All objects we create with that ``ObjectFactory`` will automatically get the default value for ``created_by_ref``. These are the properties for which defaults can be set:\n", + "All objects we create with that [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory) will automatically get the default value for ``created_by_ref``. These are the properties for which defaults can be set:\n", "\n", "- ``created_by_ref``\n", "- ``created``\n", "- ``external_references``\n", "- ``object_marking_refs``\n", "\n", - "These defaults can be bypassed. For example, say you have an ``Environment`` with multiple default values but want to create an object with a different value for ``created_by_ref``, or none at all." + "These defaults can be bypassed. For example, say you have an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with multiple default values but want to create an object with a different value for ``created_by_ref``, or none at all." ] }, { @@ -607,7 +609,7 @@ "collapsed": true }, "source": [ - "For the full power of the Environment layer, create an Environment with both a DataStore/Source/Sink and an Object Factory:" + "For the full power of the Environment layer, create an [Environment](../api/stix2.environment.rst#stix2.environment.Environment) with both a [DataStore](../api/stix2.sources.rst#stix2.sources.DataStore)/[Source](../api/stix2.sources.rst#stix2.sources.DataSource)/[Sink](../api/stix2.sources.rst#stix2.sources.DataSink) and an [ObjectFactory](../api/stix2.environment.rst#stix2.environment.ObjectFactory):" ] }, { @@ -723,9 +725,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/filesystem.ipynb b/docs/guide/filesystem.ipynb index c2e21b4..4b5bd6f 100644 --- a/docs/guide/filesystem.ipynb +++ b/docs/guide/filesystem.ipynb @@ -58,7 +58,7 @@ "source": [ "## FileSystem \n", "\n", - "The FileSystem suite contains **FileSystemStore **, **FileSystemSource** and **FileSystemSink**. Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", + "The FileSystem suite contains [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore), [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink). Under the hood, all FileSystem objects point to a file directory (on disk) that contains STIX2 content. \n", "\n", "The directory and file structure of the intended STIX2 content should be:\n", "\n", @@ -82,7 +82,7 @@ " /STIX2 Domain Object type\n", "```\n", "\n", - "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are json files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", + "Essentially a master STIX2 content directory where each subdirectory aligns to a STIX2 domain object type (i.e. \"attack-pattern\", \"campaign\", \"malware\" etc..). Within each STIX2 domain object subdirectory are JSON files that are STIX2 domain objects of the specified type. The name of the json files correspond to the ID of the STIX2 domain object found within that file. A real example of the FileSystem directory structure:\n", "\n", "```\n", "stix2_content/\n", @@ -107,15 +107,15 @@ " /vulnerability\n", "```\n", "\n", - "**FileSystemStore** is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As **FileSystemStore** is just a wrapper around a paired **FileSystemSource** and **FileSystemSink** that point the same file directory.\n", + "[FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is intended for use cases where STIX2 content is retrieved and pushed to the same file directory. As [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) is just a wrapper around a paired [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) that point the same file directory.\n", "\n", - "Use cases where STIX2 content will only be retrieved or pushed, then a **FileSystemSource** and **FileSystemSink** can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", + "Use cases where STIX2 content will only be retrieved or pushed, then a [FileSystemSource](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource) and [FileSystemSink](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink) can be used individually. Or for the use case where STIX2 content will be retrieved from one distinct file directory and pushed to another.\n", "\n", "### FileSystem API\n", "\n", - "A note on **get()**, **all_versions()**, and **query()**. The format of the STIX2 content targeted by the FileSystem suite is json files. When STIX2 content (in json) is retrieved by the **FileSystemStore** from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", + "A note on [get()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.get), [all_versions()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.all_versions), and [query()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSource.query). The format of the STIX2 content targeted by the FileSystem suite is JSON files. When STIX2 content (in JSON) is retrieved by the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore) from disk, the content will attempt to be parsed into full-featured python STIX2 objects and returned as such. \n", "\n", - "A note on **add()**. When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: python STIX objects, python dicts (of valid STIX objects or Bundles), json-encoded strings (of valid STIX objects or Bundles), or a (python)list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX json object (in a STIX Bundle) and written to disk. \n", + "A note on [add()](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemSink.add): When STIX content is added (pushed) to the file system, the STIX content can be supplied in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. Any of the previous STIX content forms will be converted to a STIX JSON object (in a STIX Bundle) and written to disk. \n", "\n", "### FileSystem Examples\n", "\n", @@ -532,21 +532,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/docs/guide/markings.ipynb b/docs/guide/markings.ipynb index cb8f762..fcacf47 100644 --- a/docs/guide/markings.ipynb +++ b/docs/guide/markings.ipynb @@ -146,14 +146,14 @@ ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n", ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--409a0b15-1108-4251-8aee-a08995976561",\n",
-       "    "created": "2017-10-04T14:42:54.685Z",\n",
-       "    "modified": "2017-10-04T14:42:54.685Z",\n",
+       "    "id": "indicator--65ff0082-bb92-4812-9b74-b144b858297f",\n",
+       "    "created": "2017-11-13T14:42:14.641Z",\n",
+       "    "modified": "2017-11-13T14:42:14.641Z",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2017-11-13T14:42:14.641818Z",\n",
        "    "labels": [\n",
        "        "malicious-activity"\n",
        "    ],\n",
-       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-10-04T14:42:54.685184Z",\n",
        "    "object_marking_refs": [\n",
        "        "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"\n",
        "    ]\n",
@@ -187,7 +187,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -263,8 +263,8 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "marking-definition",\n",
-       "    "id": "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53",\n",
-       "    "created": "2017-10-04T14:43:04.090873Z",\n",
+       "    "id": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
+       "    "created": "2017-11-13T14:43:30.558058Z",\n",
        "    "definition_type": "statement",\n",
        "    "definition": {\n",
        "        "statement": "Copyright 2017, Example Corp"\n",
@@ -276,7 +276,7 @@
        ""
       ]
      },
-     "execution_count": 4,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -523,7 +523,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -599,9 +599,9 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "malware",\n",
-       "    "id": "malware--9f8970eb-b398-41b6-b8c8-8a607ad3a2c5",\n",
-       "    "created": "2017-10-04T14:43:26.129Z",\n",
-       "    "modified": "2017-10-04T14:43:26.129Z",\n",
+       "    "id": "malware--f7128008-f6ab-4d43-a8a2-a681651268f8",\n",
+       "    "created": "2017-11-13T14:43:34.857Z",\n",
+       "    "modified": "2017-11-13T14:43:34.857Z",\n",
        "    "name": "Poison Ivy",\n",
        "    "description": "A ransomware related to ...",\n",
        "    "labels": [\n",
@@ -609,7 +609,7 @@
        "    ],\n",
        "    "granular_markings": [\n",
        "        {\n",
-       "            "marking_ref": "marking-definition--030bb5c6-c5eb-4e9c-8e7a-b9aab08ded53",\n",
+       "            "marking_ref": "marking-definition--d16f0975-c5dd-4b25-a41d-af4afcc5da92",\n",
        "            "selectors": [\n",
        "                "description"\n",
        "            ]\n",
@@ -628,7 +628,7 @@
        ""
       ]
      },
-     "execution_count": 7,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -696,7 +696,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Both object markings and granular markings can also be added to STIX objects which have already been created.\n",
+    "[Several functions](../api/stix2.markings.rst) exist to support working with data markings.\n",
+    "\n",
+    "Both object markings and granular markings can be added to STIX objects which have already been created.\n",
     "\n",
     "**Note**: Doing so will create a new version of the object (note the updated ``modified`` time)."
    ]
@@ -1041,7 +1043,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "STIX objects can also be cleared of all markings:"
+    "STIX objects can also be cleared of all markings with [clear_markings()](../api/stix2.markings.rst#stix2.markings.clear_markings):"
    ]
   },
   {
@@ -1188,12 +1190,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To get a list of the granular markings on an object, pass the object and a list of selectors to ``get_markings``:"
+    "To get a list of the granular markings on an object, pass the object and a list of selectors to [get_markings()](../api/stix2.markings.rst#stix2.markings.get_markings):"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -1202,20 +1204,22 @@
        "['marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9']"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "malware.get_markings('name')"
+    "from stix2 import get_markings\n",
+    "\n",
+    "get_markings(malware, 'name')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can also call ``get_markings()`` as a method on the STIX object."
+    "You can also call [get_markings()](../api/stix2.markings.rst#stix2.markings.get_markings) as a method on the STIX object."
    ]
   },
   {
@@ -1310,9 +1314,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
diff --git a/docs/guide/memory.ipynb b/docs/guide/memory.ipynb
index 570425e..75c0475 100644
--- a/docs/guide/memory.ipynb
+++ b/docs/guide/memory.ipynb
@@ -58,14 +58,14 @@
    "source": [
     "## Memory\n",
     "\n",
-    "The Memory suite consists of **MemoryStore**, **MemorySource**, and **MemorySink**. Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the **MemoryStore** is a just a wrapper around a paired **MemorySource** and **MemorySink**; as there is quite limited uses for just a **MemorySource** or a **MemorySink**, it is recommended to always use **MemoryStore**. The **MemoryStore** is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is ecompassed within the **FileSystemStore**. However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. **MemoryStore.save_to_file()** allows for saving all the STIX content that is in memory to a json file. **MemoryStore.load_from_file()** allows for loading STIX content from a json-formatted file. \n",
+    "The Memory suite consists of [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore), [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource), and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink). Under the hood, the Memory suite points to an in-memory dictionary. Similarly, the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is a just a wrapper around a paired [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) and [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink); as there is quite limited uses for just a [MemorySource](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySource) or a [MemorySink](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemorySink), it is recommended to always use [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). The [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) is intended for retrieving/searching and pushing STIX content to memory. It is important to note that all STIX content in memory is not backed up on the file system (disk), as that functionality is encompassed within the [FileSystemStore](../api/sources/stix2.sources.filesystem.rst#stix2.sources.filesystem.FileSystemStore). However, the Memory suite does provide some utility methods for saving and loading STIX content to disk. [MemoryStore.save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file) allows for saving all the STIX content that is in memory to a json file. [MemoryStore.load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) allows for loading STIX content from a JSON-formatted file. \n",
     "\n",
     "\n",
     "### Memory API\n",
     "\n",
-    "A note on **load_from_file()** and **save()**. These methods both add STIX content to an internal dictionary (maintained by MemoryStore). STIX content that is to be added can be in the following forms: python STIX objects, python dicts (of valid STIX objects or Bundles), json-encoded strings (of valid STIX objects or Bundles), or a (python)list of any of the previously listed types. **MemoryStore** actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as , is what it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dicts (of STIX objects) are stored as python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the **MemoryStore**. Also, for **load_from_file()**, STIX content is assumed to  be in json form within the file, individually or in a Bundle. \n",
+    "A note on [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file) and [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). These methods both add STIX content to an internal dictionary (maintained by [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore)). STIX content that is to be added can be in the following forms: Python STIX objects, Python dictionaries (of valid STIX objects or Bundles), JSON-encoded strings (of valid STIX objects or Bundles), or a (Python) list of any of the previously listed types. [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) actually stores STIX content either as python STIX objects or as python dictionaries, reducing and converting any of the aforementioned types to one of those; and whatever form the STIX object is stored as, is how it will be returned as when queried or retrieved. Python STIX objects, and json-encoded strings (of STIX content) are stored as python STIX objects. Python dictionaries (of STIX objects) are stored as Python dictionaries. This is done, as can be efficiently supported, in order to return STIX content in the form it was added to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). Also, for [load_from_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.load_from_file), STIX content is assumed to be in JSON form within the file, individually or in a Bundle. \n",
     "\n",
-    "A note on **save_to_file()**. This method dumps all STIX content that is in MemoryStore to the specified file. The file format will be json, and the STIX content will be within a STIX Bundle. ntoe, the the output form will be a json STIX Bundle regardless of the form that the individual STIX objects are stored(i.e. supplied) to the MemoryStore. \n",
+    "A note on [save_to_file()](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore.save_to_file). This method dumps all STIX content that is in [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore) to the specified file. The file format will be JSON, and the STIX content will be within a STIX Bundle. Note also that the the output form will be a JSON STIX Bundle regardless of the form that the individual STIX objects are stored (i.e. supplied) to the [MemoryStore](../api/sources/stix2.sources.memory.rst#stix2.sources.memory.MemoryStore). \n",
     "\n",
     "### Memory Examples\n",
     "\n",
@@ -288,21 +288,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.2"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
   }
  },
  "nbformat": 4,
diff --git a/docs/guide/parsing.ipynb b/docs/guide/parsing.ipynb
index e991e0c..d24f994 100644
--- a/docs/guide/parsing.ipynb
+++ b/docs/guide/parsing.ipynb
@@ -63,7 +63,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Parsing STIX content is as easy as calling the `parse()` function on a JSON string. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well."
+    "Parsing STIX content is as easy as calling the [parse()](../api/stix2.core.rst#stix2.core.parse) function on a JSON string. It will automatically determine the type of the object. The STIX objects within `bundle` objects, and the cyber observables contained within `observed-data` objects will be parsed as well."
    ]
   },
   {
@@ -109,9 +109,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
diff --git a/docs/guide/serializing.ipynb b/docs/guide/serializing.ipynb
index 8fcc19d..4499677 100644
--- a/docs/guide/serializing.ipynb
+++ b/docs/guide/serializing.ipynb
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {
     "collapsed": true,
     "nbsphinx": "hidden"
@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -144,15 +144,15 @@
        ".highlight .vm { color: #19177C } /* Name.Variable.Magic */\n",
        ".highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
{\n",
        "    "type": "indicator",\n",
-       "    "id": "indicator--81b0644d-5e9d-48fb-bb83-aabe77918305",\n",
-       "    "created": "2017-09-26T23:38:55.476Z",\n",
-       "    "modified": "2017-09-26T23:38:55.476Z",\n",
-       "    "labels": [\n",
-       "        "malicious-activity"\n",
-       "    ],\n",
+       "    "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674",\n",
+       "    "created": "2017-11-09T19:21:06.285Z",\n",
+       "    "modified": "2017-11-09T19:21:06.285Z",\n",
        "    "name": "File hash for malware variant",\n",
        "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
-       "    "valid_from": "2017-09-26T23:38:55.476436Z"\n",
+       "    "valid_from": "2017-11-09T19:21:06.285451Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
        "}\n",
        "
\n" ], @@ -160,7 +160,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -174,13 +174,112 @@ "\n", "print(str(indicator))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the string representation can be slow, as it sorts properties to be in a more readable order. If you need performance and don't care about the human-readability of the output, use the object's serialize() function:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
{"valid_from": "2017-11-09T19:21:06.285451Z", "name": "File hash for malware variant", "created": "2017-11-09T19:21:06.285Z", "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']", "labels": ["malicious-activity"], "modified": "2017-11-09T19:21:06.285Z", "type": "indicator", "id": "indicator--5eac4517-6539-4e48-ab51-7d499f599674"}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(indicator.serialize())" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/taxii.ipynb b/docs/guide/taxii.ipynb index 016f9d8..2f8905b 100644 --- a/docs/guide/taxii.ipynb +++ b/docs/guide/taxii.ipynb @@ -58,9 +58,9 @@ "source": [ "## TAXIICollection\n", "\n", - "The TAXIICollection suite contains **TAXIICollectionStore**, **TAXIICollectionSource**, and **TAXIICollectionSink**. **TAXIICollectionStore** for pushing and retrieving STIX content to local/remote TAXII Collection(s). **TAXIICollectionSource** for retrieving STIX content to local/remote TAXII Collection(s). **TAXIICollectionSink** for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be binded to a Collection from the taxii2client library (taxii2client.Collection), where all **TAXIICollection** API calls will be executed through that Collection instance.\n", + "The TAXIICollection suite contains [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore), [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource), and [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink). [TAXIICollectionStore](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionStore) for pushing and retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSource](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSource) for retrieving STIX content to local/remote TAXII Collection(s). [TAXIICollectionSink](../api/sources/stix2.sources.taxii.rst#stix2.sources.taxii.TAXIICollectionSink) for pushing STIX content to local/remote TAXII Collection(s). Each of the interfaces is designed to be bound to a Collection from the [taxii2client](https://github.com/oasis-open/cti-taxii-client) library (taxii2client.Collection), where all [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API calls will be executed through that Collection instance.\n", "\n", - "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the **TAXIICollection** suite supports searching on all STIX2 common object properties (see **Filters** documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. **TAXIICollection** will seperate any supplied queries into TAXII supported filters and non-supported filters. During a **TAXIICollection** API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the **TAXIICollection** API call. \n", + "A note on TAXII2 searching/filtering of STIX content. TAXII2 server implementations natively support searching on the STIX2 object properties: id, type and version; API requests made to TAXII2 can contain filter arguments for those 3 properties. However, the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) suite supports searching on all STIX2 common object properties (see [Filters](../api/sources/stix2.sources.filters.rst) documentation for full listing). This works simply by augmenting the filtering that is done remotely at the TAXII2 server instance. [TAXIICollection](../api/sources/stix2.sources.taxii.rst) will seperate any supplied queries into TAXII supported filters and non-supported filters. During a [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call, TAXII2 supported filters get inserted into the TAXII2 server request (to be evaluated at the server). The rest of the filters are kept locally and then applied to the STIX2 content that is returned from the TAXII2 server, before being returned from the [TAXIICollection](../api/sources/stix2.sources.taxii.rst) API call. \n", "\n", "### TAXIICollection API\n", "\n", @@ -265,9 +265,9 @@ ], "metadata": { "kernelspec": { - "display_name": "cti-python-stix2", + "display_name": "Python 2", "language": "python", - "name": "cti-python-stix2" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index f98d7b5..6ceef99 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -1,5 +1,57 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# Delete this cell to re-enable tracebacks\n", + "import sys\n", + "ipython = get_ipython()\n", + "\n", + "def hide_traceback(exc_tuple=None, filename=None, tb_offset=None,\n", + " exception_only=False, running_compiled_code=False):\n", + " etype, value, tb = sys.exc_info()\n", + " return ipython._showtraceback(etype, value, ipython.InteractiveTB.get_exception_only(etype, value))\n", + "\n", + "ipython.showtraceback = hide_traceback" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true, + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# JSON output syntax highlighting\n", + "from __future__ import print_function\n", + "from pygments import highlight\n", + "from pygments.lexers import JsonLexer\n", + "from pygments.formatters import HtmlFormatter\n", + "from IPython.display import HTML\n", + "\n", + "original_print = print\n", + "\n", + "def json_print(inpt):\n", + " string = str(inpt)\n", + " if string[0] == '{':\n", + " formatter = HtmlFormatter()\n", + " return HTML('{}'.format(\n", + " formatter.get_style_defs('.highlight'),\n", + " highlight(string, JsonLexer(), formatter)))\n", + " else:\n", + " original_print(inpt)\n", + "\n", + "print = json_print" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -10,13 +62,15 @@ "\n", "Imports can be used in different ways depending on the use case and support levels.\n", "\n", - "People who want to (in general) support the latest version of STIX 2.X without making changes, implicitly using the latest version" + "People who want to support the latest version of STIX 2.X without having to make changes, can implicitly use the latest version:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", @@ -34,7 +88,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import Indicator\n", @@ -46,13 +102,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "People who want to use an explicit version" + "People who want to use an explicit version:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2.v20\n", @@ -70,7 +128,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2.v20 import Indicator\n", @@ -88,7 +148,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2.v20 as stix2\n", @@ -108,13 +170,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", "\n", "stix2.v20.Indicator()\n", - "\n", "stix2.v21.Indicator()" ] }, @@ -128,7 +191,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2 import v20, v21\n", @@ -147,7 +212,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from stix2.v20 import Indicator as Indicator_v20\n", @@ -164,14 +231,108 @@ "### How parsing will work\n", "If the ``version`` positional argument is not provided. The data will be parsed using the latest version of STIX 2.X supported by the `stix2` library.\n", "\n", - "You can lock your `parse()` method to a specific STIX version by" + "You can lock your [parse()](../api/stix2.core.rst#stix2.core.parse) method to a specific STIX version by:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
{\n",
+       "    "type": "indicator",\n",
+       "    "id": "indicator--dbcbd659-c927-4f9a-994f-0a2632274394",\n",
+       "    "created": "2017-09-26T23:33:39.829Z",\n",
+       "    "modified": "2017-09-26T23:33:39.829Z",\n",
+       "    "name": "File hash for malware variant",\n",
+       "    "pattern": "[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']",\n",
+       "    "valid_from": "2017-09-26T23:33:39.829952Z",\n",
+       "    "labels": [\n",
+       "        "malicious-activity"\n",
+       "    ]\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from stix2 import parse\n", "\n", @@ -201,17 +362,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### How will custom work\n", + "### How will custom content work\n", "\n", - "CustomObject, CustomObservable, CustomMarking and CustomExtension must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", + "[CustomObject](../api/stix2.v20.sdo.rst#stix2.v20.sdo.CustomObject), [CustomObservable](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomObservable), [CustomMarking](../api/stix2.v20.common.rst#stix2.v20.common.CustomMarking) and [CustomExtension](../api/stix2.v20.observables.rst#stix2.v20.observables.CustomExtension) must be registered explicitly by STIX version. This is a design decision since properties or requirements may change as the STIX Technical Specification advances.\n", "\n", - "You can perform this by," + "You can perform this by:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import stix2\n", @@ -231,7 +394,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } diff --git a/docs/guide/versioning.ipynb b/docs/guide/versioning.ipynb index 30ceb69..fb3b866 100644 --- a/docs/guide/versioning.ipynb +++ b/docs/guide/versioning.ipynb @@ -182,7 +182,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The modified time will be updated to the current time unless you provide a specific value as a keyword argument. Note that you can’t change the type, id, or created properties." + "The modified time will be updated to the current time unless you provide a specific value as a keyword argument. Note that you can’t change the `type`, `id`, or `created` properties." ] }, { @@ -322,9 +322,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { diff --git a/requirements.txt b/requirements.txt index 93328de..0de31a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bumpversion -nbsphinx +nbsphinx>=0.2.15 pre-commit pytest pytest-cov diff --git a/setup.cfg b/setup.cfg index 5edf45c..76162b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.0 +current_version = 0.4.0 commit = True tag = True diff --git a/setup.py b/setup.py index 3ed7ba2..a5ada02 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/stix2/__init__.py b/stix2/__init__.py index 6fe2a79..87dfda1 100644 --- a/stix2/__init__.py +++ b/stix2/__init__.py @@ -3,17 +3,17 @@ .. autosummary:: :toctree: api - common + v21.common core environment exceptions markings - observables + v21.observables patterns properties - sdo + v21.sdo sources - sro + v21.sro utils """ diff --git a/stix2/base.py b/stix2/base.py index b0cf6ff..76b07b8 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -153,15 +153,7 @@ class _STIXBase(collections.Mapping): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - properties = self.object_properties() - - def sort_by(element): - return find_property_index(self, properties, element) - - # separators kwarg -> don't include spaces after commas. - return json.dumps(self, indent=4, cls=STIXJSONEncoder, - item_sort_key=sort_by, - separators=(",", ": ")) + return self.serialize(pretty=True) def __repr__(self): props = [(k, self[k]) for k in self.object_properties() if self.get(k)] @@ -185,6 +177,38 @@ class _STIXBase(collections.Mapping): def revoke(self): return _revoke(self) + def serialize(self, pretty=False, **kwargs): + """ + Serialize a STIX object. + + Args: + pretty (bool): If True, output properties following the STIX specs + formatting. This includes indentation. Refer to notes for more + details. + **kwargs: The arguments for a json.dumps() call. + + Returns: + dict: The serialized JSON object. + + Note: + The argument ``pretty=True`` will output the STIX object following + spec order. Using this argument greatly impacts object serialization + performance. If your use case is centered across machine-to-machine + operation it is recommended to set ``pretty=False``. + + When ``pretty=True`` the following key-value pairs will be added or + overridden: indent=4, separators=(",", ": "), item_sort_key=sort_by. + """ + if pretty: + properties = self.object_properties() + + def sort_by(element): + return find_property_index(self, properties, element) + + kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by}) + + return json.dumps(self, cls=STIXJSONEncoder, **kwargs) + class _Observable(_STIXBase): diff --git a/stix2/environment.py b/stix2/environment.py index 4919335..55130d3 100644 --- a/stix2/environment.py +++ b/stix2/environment.py @@ -76,7 +76,7 @@ class ObjectFactory(object): class Environment(object): - """ + """Abstract away some of the nasty details of working with STIX content. Args: factory (ObjectFactory, optional): Factory for creating objects with common @@ -105,30 +105,13 @@ class Environment(object): return self.factory.create(*args, **kwargs) create.__doc__ = ObjectFactory.create.__doc__ - def get(self, *args, **kwargs): - try: - return self.source.get(*args, **kwargs) - except AttributeError: - raise AttributeError('Environment has no data source to query') - get.__doc__ = DataStore.get.__doc__ - - def all_versions(self, *args, **kwargs): - """Retrieve all versions of a single STIX object by ID. - """ - try: - return self.source.all_versions(*args, **kwargs) - except AttributeError: - raise AttributeError('Environment has no data source to query') - all_versions.__doc__ = DataStore.all_versions.__doc__ - - def query(self, *args, **kwargs): - """Retrieve STIX objects matching a set of filters. - """ - try: - return self.source.query(*args, **kwargs) - except AttributeError: - raise AttributeError('Environment has no data source to query') - query.__doc__ = DataStore.query.__doc__ + get = DataStore.__dict__['get'] + all_versions = DataStore.__dict__['all_versions'] + query = DataStore.__dict__['query'] + creator_of = DataStore.__dict__['creator_of'] + relationships = DataStore.__dict__['relationships'] + related_to = DataStore.__dict__['related_to'] + add = DataStore.__dict__['add'] def add_filters(self, *args, **kwargs): try: @@ -142,13 +125,6 @@ class Environment(object): except AttributeError: raise AttributeError('Environment has no data source') - def add(self, *args, **kwargs): - try: - return self.sink.add(*args, **kwargs) - except AttributeError: - raise AttributeError('Environment has no data sink to put objects in') - add.__doc__ = DataStore.add.__doc__ - def parse(self, *args, **kwargs): return _parse(*args, **kwargs) parse.__doc__ = _parse.__doc__ diff --git a/stix2/markings/__init__.py b/stix2/markings/__init__.py index 4038a5e..c8dbdbc 100644 --- a/stix2/markings/__init__.py +++ b/stix2/markings/__init__.py @@ -1,9 +1,14 @@ """ -Functions and classes for working with STIX 2 Data Markings. +Functions for working with STIX 2 Data Markings. -These high level functions will operate on both object level markings and +These high level functions will operate on both object-level markings and granular markings unless otherwise noted in each of the functions. +Note: + These functions are also available as methods on SDOs, SROs, and Marking + Definitions. The corresponding methods on those classes are identical to + these functions except that the `obj` parameter is omitted. + .. autosummary:: :toctree: markings @@ -20,7 +25,7 @@ from stix2.markings import granular_markings, object_markings def get_markings(obj, selectors=None, inherited=False, descendants=False): """ - Get all markings associated to the field(s). + Get all markings associated to the field(s) specified by selectors. Args: obj: An SDO or SRO object. @@ -57,15 +62,15 @@ def get_markings(obj, selectors=None, inherited=False, descendants=False): def set_markings(obj, marking, selectors=None): """ - Removes all markings associated with selectors and appends a new granular + Remove all markings associated with selectors and appends a new granular marking. Refer to `clear_markings` and `add_markings` for details. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Returns: A new version of the given SDO or SRO with specified markings removed @@ -84,14 +89,14 @@ def set_markings(obj, marking, selectors=None): def remove_markings(obj, marking, selectors=None): """ - Removes granular_marking from the granular_markings collection. + Remove a marking from this object. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -114,14 +119,14 @@ def remove_markings(obj, marking, selectors=None): def add_markings(obj, marking, selectors=None): """ - Appends a granular_marking to the granular_markings collection. + Append a marking to this object. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -142,7 +147,7 @@ def add_markings(obj, marking, selectors=None): def clear_markings(obj, selectors=None): """ - Removes all granular_marking associated with the selectors. + Remove all markings associated with the selectors. Args: obj: An SDO or SRO object. @@ -170,14 +175,14 @@ def clear_markings(obj, selectors=None): def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ - Checks if field(s) is marked by any marking or by specific marking(s). + Check if field(s) is marked by any marking or by specific marking(s). Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the field(s) appear(s). marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the field(s) appear(s). inherited: If True, include object level markings and granular markings inherited to determine if the properties is/are marked. descendants: If True, include granular markings applied to any children diff --git a/stix2/markings/granular_markings.py b/stix2/markings/granular_markings.py index 3fe3a48..be5d258 100644 --- a/stix2/markings/granular_markings.py +++ b/stix2/markings/granular_markings.py @@ -8,7 +8,7 @@ from stix2.utils import new_version def get_markings(obj, selectors, inherited=False, descendants=False): """ - Get all markings associated to with the properties. + Get all granular markings associated to with the properties. Args: obj: An SDO or SRO object. @@ -50,8 +50,8 @@ def get_markings(obj, selectors, inherited=False, descendants=False): def set_markings(obj, marking, selectors): """ - Removes all markings associated with selectors and appends a new granular - marking. Refer to `clear_markings` and `add_markings` for details. + Remove all granular markings associated with selectors and append a new + granular marking. Refer to `clear_markings` and `add_markings` for details. Args: obj: An SDO or SRO object. @@ -71,14 +71,14 @@ def set_markings(obj, marking, selectors): def remove_markings(obj, marking, selectors): """ - Removes granular_marking from the granular_markings collection. + Remove a granular marking from the granular_markings collection. Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -123,14 +123,14 @@ def remove_markings(obj, marking, selectors): def add_markings(obj, marking, selectors): """ - Appends a granular_marking to the granular_markings collection. + Append a granular marking to the granular_markings collection. Args: obj: An SDO or SRO object. - selectors: list of type string, selectors must be relative to the TLO - in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: list of type string, selectors must be relative to the TLO + in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. @@ -157,7 +157,7 @@ def add_markings(obj, marking, selectors): def clear_markings(obj, selectors): """ - Removes all granular_markings associated with the selectors. + Remove all granular markings associated with the selectors. Args: obj: An SDO or SRO object. @@ -214,14 +214,14 @@ def clear_markings(obj, selectors): def is_marked(obj, marking=None, selectors=None, inherited=False, descendants=False): """ - Checks if field is marked by any marking or by specific marking(s). + Check if field is marked by any marking or by specific marking(s). Args: obj: An SDO or SRO object. - selectors: string or list of selectors strings relative to the SDO or - SRO in which the properties appear. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. + selectors: string or list of selectors strings relative to the SDO or + SRO in which the properties appear. inherited: If True, return markings inherited from the given selector. descendants: If True, return granular markings applied to any children of the given selector. diff --git a/stix2/markings/object_markings.py b/stix2/markings/object_markings.py index cb78294..c0375c3 100644 --- a/stix2/markings/object_markings.py +++ b/stix2/markings/object_markings.py @@ -23,7 +23,7 @@ def get_markings(obj): def add_markings(obj, marking): """ - Appends an object level marking to the object_marking_refs collection. + Append an object level marking to the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -42,7 +42,7 @@ def add_markings(obj, marking): def remove_markings(obj, marking): """ - Removes object level marking from the object_marking_refs collection. + Remove an object level marking from the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -76,7 +76,7 @@ def remove_markings(obj, marking): def set_markings(obj, marking): """ - Removes all object level markings and appends new object level markings to + Remove all object level markings and append new object level markings to the collection. Refer to `clear_markings` and `add_markings` for details. Args: @@ -94,7 +94,7 @@ def set_markings(obj, marking): def clear_markings(obj): """ - Removes all object level markings from the object_marking_refs collection. + Remove all object level markings from the object_marking_refs collection. Args: obj: A SDO or SRO object. @@ -108,7 +108,7 @@ def clear_markings(obj): def is_marked(obj, marking=None): """ - Checks if SDO or SRO is marked by any marking or by specific marking(s). + Check if SDO or SRO is marked by any marking or by specific marking(s). Args: obj: A SDO or SRO object. diff --git a/stix2/markings/utils.py b/stix2/markings/utils.py index 3024fe8..429311b 100644 --- a/stix2/markings/utils.py +++ b/stix2/markings/utils.py @@ -9,7 +9,7 @@ from stix2 import exceptions def _evaluate_expression(obj, selector): - """Walks an SDO or SRO generating selectors to match against ``selector``. + """Walk an SDO or SRO generating selectors to match against ``selector``. If a match is found and the the value of this property is present in the objects. Matching value of the property will be returned. @@ -32,7 +32,7 @@ def _evaluate_expression(obj, selector): def _validate_selector(obj, selector): - """Internal method to evaluate each selector.""" + """Evaluate each selector against an object.""" results = list(_evaluate_expression(obj, selector)) if len(results) >= 1: @@ -132,7 +132,7 @@ def compress_markings(granular_markings): def expand_markings(granular_markings): - """Expands granular markings list. + """Expand granular markings list. If there is more than one selector per granular marking. It will be expanded using the same marking_ref. @@ -187,7 +187,7 @@ def expand_markings(granular_markings): def build_granular_marking(granular_marking): - """Returns a dictionary with the required structure for a granular marking. + """Return a dictionary with the required structure for a granular marking. """ return {"granular_markings": expand_markings(granular_marking)} diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 1fe9391..adc6def 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -11,8 +11,12 @@ | """ +from abc import ABCMeta, abstractmethod import uuid +from six import with_metaclass + +from stix2.sources.filters import Filter from stix2.utils import deduplicate @@ -21,94 +25,170 @@ def make_id(): class DataStore(object): - """An implementer will create a concrete subclass from - this class for the specific DataStore. + """An implementer can subclass to create custom behavior from + this class for the specific DataStores. Args: source (DataSource): An existing DataSource to use as this DataStore's DataSource component - sink (DataSink): An existing DataSink to use as this DataStore's DataSink component Attributes: id (str): A unique UUIDv4 to identify this DataStore. - source (DataSource): An object that implements DataSource class. - sink (DataSink): An object that implements DataSink class. """ def __init__(self, source=None, sink=None): + super(DataStore, self).__init__() self.id = make_id() self.source = source self.sink = sink - def get(self, stix_id, allow_custom=False): + def get(self, *args, **kwargs): """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the single most recent version of the STIX object specified by the "id". """ - return self.source.get(stix_id, allow_custom=allow_custom) + try: + return self.source.get(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) - def all_versions(self, stix_id, allow_custom=False): + def all_versions(self, *args, **kwargs): """Retrieve all versions of a single STIX object by ID. - Implement: Translate all_versions() call to the appropriate DataSource call + Translate all_versions() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id, allow_custom=allow_custom) + try: + return self.source.all_versions(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) - def query(self, query=None, allow_custom=False): + def query(self, *args, **kwargs): """Retrieve STIX objects matching a set of filters. - Implement: Specific data source API calls, processing, - functionality required for retrieving query from the data source. + Translate query() call to the appropriate DataSource call. Args: query (list): a list of filters (which collectively are the query) to conduct search on. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.query(query=query) + try: + return self.source.query(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + def creator_of(self, *args, **kwargs): + """Retrieve the Identity refered to by the object's `created_by_ref`. - Translates add() to the appropriate DataSink call. + Translate creator_of() call to the appropriate DataSource call. + + Args: + obj: The STIX object whose `created_by_ref` property will be looked + up. + + Returns: + The STIX object's creator, or None, if the object contains no + `created_by_ref` property or the object's creator cannot be found. + + """ + try: + return self.source.creator_of(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) + + def relationships(self, *args, **kwargs): + """Retrieve Relationships involving the given STIX object. + + Translate relationships() call to the appropriate DataSource call. + + Only one of `source_only` and `target_only` may be `True`. + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + relationships will be looked up. + relationship_type (str): Only retrieve Relationships of this type. + If None, all relationships will be returned, regardless of type. + source_only (bool): Only retrieve Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only retrieve Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of Relationship objects involving the given STIX object. + + """ + try: + return self.source.relationships(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) + + def related_to(self, *args, **kwargs): + """Retrieve STIX Objects that have a Relationship involving the given + STIX object. + + Translate related_to() call to the appropriate DataSource call. + + Only one of `source_only` and `target_only` may be `True`. + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + related objects will be looked up. + relationship_type (str): Only retrieve objects related by this + Relationships type. If None, all related objects will be + returned, regardless of type. + source_only (bool): Only examine Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only examine Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of STIX objects related to the given STIX object. + + """ + try: + return self.source.related_to(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data source to query' % self.__class__.__name__) + + def add(self, *args, **kwargs): + """Method for storing STIX objects. + + Define custom behavior before storing STIX objects using the associated + DataSink. Translates add() to the appropriate DataSink call. Args: stix_objs (list): a list of STIX objects - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + """ - return self.sink.add(stix_objs, allow_custom=allow_custom) + try: + return self.sink.add(*args, **kwargs) + except AttributeError: + raise AttributeError('%s has no data sink to put objects in' % self.__class__.__name__) -class DataSink(object): +class DataSink(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSink. @@ -117,10 +197,12 @@ class DataSink(object): """ def __init__(self): + super(DataSink, self).__init__() self.id = make_id() - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + @abstractmethod + def add(self, stix_objs): + """Method for storing STIX objects. Implement: Specific data sink API calls, processing, functionality required for adding data to the sink @@ -128,28 +210,26 @@ class DataSink(object): Args: stix_objs (list): a list of STIX objects (where each object is a STIX object) - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. """ - raise NotImplementedError() -class DataSource(object): +class DataSource(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSource. Attributes: id (str): A unique UUIDv4 to identify this DataSource. - - _filters (set): A collection of filters attached to this DataSource. + filters (set): A collection of filters attached to this DataSource. """ def __init__(self): + super(DataSource, self).__init__() self.id = make_id() self.filters = set() - def get(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def get(self, stix_id): """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -158,21 +238,17 @@ class DataSource(object): stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the STIX object """ - raise NotImplementedError() - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def all_versions(self, stix_id): """ - Implement: Similar to get() except returns list of all object versions of - the specified "id". In addition, implement the specific data + Implement: Similar to get() except returns list of all object versions + of the specified "id". In addition, implement the specific data source API calls, processing, functionality required for retrieving data from the data source. @@ -180,35 +256,128 @@ class DataSource(object): stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - raise NotImplementedError() - def query(self, query=None, _composite_filters=None, allow_custom=False): + @abstractmethod + def query(self, query=None): """ - Implement:Implement the specific data source API calls, processing, + Implement: The specific data source API calls, processing, functionality required for retrieving query from the data source Args: query (list): a list of filters (which collectively are the query) - to conduct search on - _composite_filters (set): a set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to conduct search on. Returns: stix_objs (list): a list of STIX objects """ - raise NotImplementedError() + + def creator_of(self, obj): + """Retrieve the Identity refered to by the object's `created_by_ref`. + + Args: + obj: The STIX object whose `created_by_ref` property will be looked + up. + + Returns: + The STIX object's creator, or None, if the object contains no + `created_by_ref` property or the object's creator cannot be found. + + """ + creator_id = obj.get('created_by_ref', '') + if creator_id: + return self.get(creator_id) + else: + return None + + def relationships(self, obj, relationship_type=None, source_only=False, target_only=False): + """Retrieve Relationships involving the given STIX object. + + Only one of `source_only` and `target_only` may be `True`. + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + relationships will be looked up. + relationship_type (str): Only retrieve Relationships of this type. + If None, all relationships will be returned, regardless of type. + source_only (bool): Only retrieve Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only retrieve Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of Relationship objects involving the given STIX object. + + """ + results = [] + filters = [Filter('type', '=', 'relationship')] + + try: + obj_id = obj['id'] + except KeyError: + raise ValueError("STIX object has no 'id' property") + except TypeError: + # Assume `obj` is an ID string + obj_id = obj + + if relationship_type: + filters.append(Filter('relationship_type', '=', relationship_type)) + + if source_only and target_only: + raise ValueError("Search either source only or target only, but not both") + + if not target_only: + results.extend(self.query(filters + [Filter('source_ref', '=', obj_id)])) + if not source_only: + results.extend(self.query(filters + [Filter('target_ref', '=', obj_id)])) + + return results + + def related_to(self, obj, relationship_type=None, source_only=False, target_only=False): + """Retrieve STIX Objects that have a Relationship involving the given + STIX object. + + Only one of `source_only` and `target_only` may be `True`. + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + related objects will be looked up. + relationship_type (str): Only retrieve objects related by this + Relationships type. If None, all related objects will be + returned, regardless of type. + source_only (bool): Only examine Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only examine Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of STIX objects related to the given STIX object. + + """ + results = [] + rels = self.relationships(obj, relationship_type, source_only, target_only) + + try: + obj_id = obj['id'] + except TypeError: + # Assume `obj` is an ID string + obj_id = obj + + # Get all unique ids from the relationships except that of the object + ids = set() + for r in rels: + ids.update((r.source_ref, r.target_ref)) + ids.remove(obj_id) + + for i in ids: + results.append(self.get(i)) + + return results class CompositeDataSource(DataSource): @@ -224,7 +393,7 @@ class CompositeDataSource(DataSource): Attributes: - data_sources (dict): A dictionary of DataSource objects; to be + data_sources (list): A dictionary of DataSource objects; to be controlled and used by the Data Source Controller object. """ @@ -237,7 +406,7 @@ class CompositeDataSource(DataSource): super(CompositeDataSource, self).__init__() self.data_sources = [] - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object by STIX ID Federated retrieve method, iterates through all DataSources @@ -253,9 +422,7 @@ class CompositeDataSource(DataSource): stix_id (str): the id of the STIX object to retrieve. _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached - to another parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to another parent CompositeDataSource), not user supplied. Returns: stix_obj: the STIX object to be returned. @@ -273,7 +440,7 @@ class CompositeDataSource(DataSource): # for every configured Data Source, call its retrieve handler for ds in self.data_sources: - data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.get(stix_id=stix_id, _composite_filters=all_filters) if data: all_data.append(data) @@ -288,22 +455,20 @@ class CompositeDataSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects by STIX ID + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve all versions of a STIX object by STIX ID. - Federated all_versions retrieve method - iterates through all DataSources - defined in "data_sources" + Federated all_versions retrieve method - iterates through all + DataSources defined in "data_sources". A composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + each configured data source, pushing filtering to them to handle. Args: - stix_id (str): id of the STIX objects to retrieve + stix_id (str): id of the STIX objects to retrieve. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects that have the specified id @@ -322,7 +487,7 @@ class CompositeDataSource(DataSource): # retrieve STIX objects from all configured data sources for ds in self.data_sources: - data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -332,19 +497,17 @@ class CompositeDataSource(DataSource): return all_data - def query(self, query=None, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects that match query + def query(self, query=None, _composite_filters=None): + """Retrieve STIX objects that match a query. Federate the query to all DataSources attached to the Composite Data Source. Args: - query (list): list of filters to search on + query (list): list of filters to search on. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects to be returned @@ -354,7 +517,7 @@ class CompositeDataSource(DataSource): raise AttributeError('CompositeDataSource has no data sources') if not query: - # dont mess with the query (i.e. convert to a set, as thats done + # don't mess with the query (i.e. convert to a set, as that's done # within the specific DataSources that are called) query = [] @@ -369,7 +532,7 @@ class CompositeDataSource(DataSource): # federate query to all attached data sources, # pass composite filters to id for ds in self.data_sources: - data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.query(query=query, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 @@ -379,6 +542,80 @@ class CompositeDataSource(DataSource): return all_data + def relationships(self, *args, **kwargs): + """Retrieve Relationships involving the given STIX object. + + Only one of `source_only` and `target_only` may be `True`. + + Federated relationships retrieve method - iterates through all + DataSources defined in "data_sources". + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + relationships will be looked up. + relationship_type (str): Only retrieve Relationships of this type. + If None, all relationships will be returned, regardless of type. + source_only (bool): Only retrieve Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only retrieve Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of Relationship objects involving the given STIX object. + + """ + if not self.has_data_sources(): + raise AttributeError('CompositeDataSource has no data sources') + + results = [] + for ds in self.data_sources: + results.extend(ds.relationships(*args, **kwargs)) + + # remove exact duplicates (where duplicates are STIX 2.0 + # objects with the same 'id' and 'modified' values) + if len(results) > 0: + results = deduplicate(results) + + return results + + def related_to(self, *args, **kwargs): + """Retrieve STIX Objects that have a Relationship involving the given + STIX object. + + Only one of `source_only` and `target_only` may be `True`. + + Federated related objects method - iterates through all + DataSources defined in "data_sources". + + Args: + obj (STIX object OR dict OR str): The STIX object (or its ID) whose + related objects will be looked up. + relationship_type (str): Only retrieve objects related by this + Relationships type. If None, all related objects will be + returned, regardless of type. + source_only (bool): Only examine Relationships for which this + object is the source_ref. Default: False. + target_only (bool): Only examine Relationships for which this + object is the target_ref. Default: False. + + Returns: + (list): List of STIX objects related to the given STIX object. + + """ + if not self.has_data_sources(): + raise AttributeError('CompositeDataSource has no data sources') + + results = [] + for ds in self.data_sources: + results.extend(ds.related_to(*args, **kwargs)) + + # remove exact duplicates (where duplicates are STIX 2.0 + # objects with the same 'id' and 'modified' values) + if len(results) > 0: + results = deduplicate(results) + + return results + def add_data_source(self, data_source): """Attach a DataSource to CompositeDataSource instance diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index eb83d8c..e92c525 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -26,14 +26,15 @@ class FileSystemStore(DataStore): Default: False. Attributes: - source (FileSystemSource): FuleSystemSource + source (FileSystemSource): FileSystemSource sink (FileSystemSink): FileSystemSink """ def __init__(self, stix_dir, bundlify=False): - super(FileSystemStore, self).__init__() - self.source = FileSystemSource(stix_dir=stix_dir) - self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + super(FileSystemStore, self).__init__( + source=FileSystemSource(stix_dir=stix_dir), + sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + ) class FileSystemSink(DataSink): @@ -99,11 +100,11 @@ class FileSystemSink(DataSink): self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): - stix_data = parse(stix_data, allow_custom, version) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # extract STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: # adding json-formatted STIX self._check_path_and_write(stix_data) @@ -111,12 +112,12 @@ class FileSystemSink(DataSink): elif isinstance(stix_data, Bundle): # recursively add individual STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects for stix_obj in stix_data: - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), " @@ -146,7 +147,7 @@ class FileSystemSource(DataSource): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: @@ -166,8 +167,7 @@ class FileSystemSource(DataSource): """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version) + all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -176,7 +176,7 @@ class FileSystemSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions @@ -197,10 +197,9 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version)] + return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -305,7 +304,7 @@ class FileSystemSource(DataSource): all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/sources/filters.py b/stix2/sources/filters.py index 5772112..5af48cd 100644 --- a/stix2/sources/filters.py +++ b/stix2/sources/filters.py @@ -10,6 +10,11 @@ FILTER_OPS = ['=', '!=', 'in', '>', '<', '>=', '<='] """Supported filter value types""" FILTER_VALUE_TYPES = [bool, dict, float, int, list, str, tuple] +try: + FILTER_VALUE_TYPES.append(unicode) +except NameError: + # Python 3 doesn't need to worry about unicode + pass def _check_filter_components(prop, op, value): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2d1705d..308d0d0 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,7 +24,7 @@ from stix2.sources import DataSink, DataSource, DataStore from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None, allow_custom=False): +def _add(store, stix_data=None, allow_custom=False, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. @@ -34,6 +34,8 @@ def _add(store, stix_data=None, allow_custom=False): stix_data (list OR dict OR STIX object): STIX objects to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -44,25 +46,25 @@ def _add(store, stix_data=None, allow_custom=False): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # recurse on each STIX object in bundle for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: - _add(store, stix_data) + _add(store, stix_data, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") @@ -81,6 +83,8 @@ class MemoryStore(DataStore): stix_data (list OR dict OR STIX object): STIX content to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Attributes: _data (dict): the in-memory dict that holds STIX objects @@ -88,17 +92,18 @@ class MemoryStore(DataStore): sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False): - super(MemoryStore, self).__init__() + def __init__(self, stix_data=None, allow_custom=False, version=None): self._data = {} if stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) - self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) + super(MemoryStore, self).__init__( + source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True), + sink=MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + ) - def save_to_file(self, file_path, allow_custom=False): + def save_to_file(self, *args, **kwargs): """Write SITX objects from in-memory dictionary to JSON file, as a STIX Bundle. @@ -108,9 +113,9 @@ class MemoryStore(DataStore): not. Default: False. """ - return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) + return self.sink.save_to_file(*args, **kwargs) - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, *args, **kwargs): """Load STIX data from JSON file. File format is expected to be a single JSON @@ -120,9 +125,11 @@ class MemoryStore(DataStore): file_path (str): file path to load STIX data from allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ - return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) + return self.source.load_from_file(*args, **kwargs) class MemorySink(DataSink): @@ -146,17 +153,17 @@ class MemorySink(DataSink): a MemorySource """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySink, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def add(self, stix_data, allow_custom=False): - _add(self, stix_data, allow_custom=allow_custom) + def add(self, stix_data, allow_custom=False, version=None): + _add(self, stix_data, allow_custom=allow_custom, version=version) add.__doc__ = _add.__doc__ def save_to_file(self, file_path, allow_custom=False): @@ -164,7 +171,7 @@ class MemorySink(DataSink): if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "w") as f: - f.write(str(Bundle(self._data.values(), allow_custom=allow_custom))) + f.write(str(Bundle(list(self._data.values()), allow_custom=allow_custom))) save_to_file.__doc__ = MemoryStore.save_to_file.__doc__ @@ -190,24 +197,22 @@ class MemorySource(DataSource): a MemorySink """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySource, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (dict OR STIX object): STIX object that has the supplied @@ -227,7 +232,7 @@ class MemorySource(DataSource): # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, _composite_filters=_composite_filters) if all_data: # reduce to most recent version @@ -237,7 +242,7 @@ class MemorySource(DataSource): else: return None - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, _composite_filters=None): """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it Note: Since Memory sources/sinks don't handle multiple versions of a @@ -245,10 +250,8 @@ class MemorySource(DataSource): Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that has the supplied ID. As the @@ -257,9 +260,9 @@ class MemorySource(DataSource): is returned in the same form as it as added """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -268,10 +271,8 @@ class MemorySource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that matches the supplied @@ -284,7 +285,7 @@ class MemorySource(DataSource): query = set() else: if not isinstance(query, list): - # make sure dont make set from a Filter object, + # make sure don't make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) query = [query] query = set(query) @@ -300,8 +301,8 @@ class MemorySource(DataSource): return all_data - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, file_path, allow_custom=False, version=None): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 414e27f..8eb5069 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,5 +1,5 @@ """ -Python STIX 2.x TaxiiCollectionStore +Python STIX 2.x TAXIICollectionStore """ from stix2.base import _STIXBase @@ -20,9 +20,10 @@ class TAXIICollectionStore(DataStore): collection (taxii2.Collection): TAXII Collection instance """ def __init__(self, collection): - super(TAXIICollectionStore, self).__init__() - self.source = TAXIICollectionSource(collection) - self.sink = TAXIICollectionSink(collection) + super(TAXIICollectionStore, self).__init__( + source=TAXIICollectionSource(collection), + sink=TAXIICollectionSink(collection) + ) class TAXIICollectionSink(DataSink): @@ -37,7 +38,7 @@ class TAXIICollectionSink(DataSink): super(TAXIICollectionSink, self).__init__() self.collection = collection - def add(self, stix_data, allow_custom=False): + def add(self, stix_data, allow_custom=False, version=None): """Add/push STIX content to TAXII Collection endpoint Args: @@ -46,6 +47,8 @@ class TAXIICollectionSink(DataSink): json encoded string, or list of any of the following allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -62,11 +65,11 @@ class TAXIICollectionSink(DataSink): elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj, allow_custom=allow_custom) + self.add(obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: @@ -90,16 +93,18 @@ class TAXIICollectionSource(DataSource): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote STIX Collection endpoint. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -121,7 +126,7 @@ class TAXIICollectionSource(DataSource): stix_obj = list(apply_common_filters(stix_objs, query)) if len(stix_obj): - stix_obj = parse(stix_obj[0], allow_custom=allow_custom) + stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) if stix_obj.id != stix_id: # check - was added to handle erroneous TAXII servers stix_obj = None @@ -130,16 +135,18 @@ class TAXIICollectionSource(DataSource): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (see query() as all_versions() is just a wrapper) @@ -151,17 +158,17 @@ class TAXIICollectionSource(DataSource): Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json - all_data = [parse(stix_obj) for stix_obj in all_data] + all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -170,10 +177,12 @@ class TAXIICollectionSource(DataSource): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): list of STIX objects that matches the supplied @@ -200,7 +209,7 @@ class TAXIICollectionSource(DataSource): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) @@ -209,7 +218,7 @@ class TAXIICollectionSource(DataSource): all_data = list(apply_common_filters(all_data, query)) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/test/constants.py b/stix2/test/constants.py index 7ced397..8038dd6 100644 --- a/stix2/test/constants.py +++ b/stix2/test/constants.py @@ -31,6 +31,18 @@ MARKING_IDS = [ "marking-definition--68520ae2-fefe-43a9-84ee-2c2a934d2c7d", "marking-definition--2802dfb1-1019-40a8-8848-68d0ec0e417f", ] +RELATIONSHIP_IDS = [ + 'relationship--06520621-5352-4e6a-b976-e8fa3d437ffd', + 'relationship--181c9c09-43e6-45dd-9374-3bec192f05ef', + 'relationship--a0cbb21c-8daf-4a7f-96aa-7155a4ef8f70' +] + +# All required args for a Campaign instance +CAMPAIGN_KWARGS = dict( + name="Green Group Attacks Against Finance", + description="Campaign by Green Group against a series of targets in the financial services sector.", +) + # All required args for a Campaign instance, plus some optional args CAMPAIGN_MORE_KWARGS = dict( diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b1cffd0..8b14172 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -1,8 +1,9 @@ +import json + import pytest import stix2 - EXPECTED_BUNDLE = """{ "type": "bundle", "id": "bundle--00000000-0000-0000-0000-000000000004", @@ -41,6 +42,44 @@ EXPECTED_BUNDLE = """{ ] }""" +EXPECTED_BUNDLE_DICT = { + "type": "bundle", + "id": "bundle--00000000-0000-0000-0000-000000000004", + "spec_version": "2.0", + "objects": [ + { + "type": "indicator", + "id": "indicator--00000000-0000-0000-0000-000000000001", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "valid_from": "2017-01-01T12:34:56Z", + "labels": [ + "malicious-activity" + ] + }, + { + "type": "malware", + "id": "malware--00000000-0000-0000-0000-000000000002", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Cryptolocker", + "labels": [ + "ransomware" + ] + }, + { + "type": "relationship", + "id": "relationship--00000000-0000-0000-0000-000000000003", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "relationship_type": "indicates", + "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" + } + ] +} + def test_empty_bundle(): bundle = stix2.Bundle() @@ -82,10 +121,17 @@ def test_bundle_with_wrong_spec_version(): assert str(excinfo.value) == "Invalid value for Bundle 'spec_version': must equal '2.0'." -def test_create_bundle(indicator, malware, relationship): +def test_create_bundle1(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE + + +def test_create_bundle2(indicator, malware, relationship): + bundle = stix2.Bundle(objects=[indicator, malware, relationship]) + + assert json.loads(bundle.serialize()) == EXPECTED_BUNDLE_DICT def test_create_bundle_with_positional_args(indicator, malware, relationship): diff --git a/stix2/test/test_custom.py b/stix2/test/test_custom.py index 92d5d4c..7c1832b 100644 --- a/stix2/test/test_custom.py +++ b/stix2/test/test_custom.py @@ -94,6 +94,28 @@ def test_custom_property_in_bundled_object(): assert '"x_foo": "bar"' in str(bundle) +def test_custom_marking_no_init_1(): + @stix2.CustomMarking('x-new-obj', [ + ('property1', stix2.properties.StringProperty(required=True)), + ]) + class NewObj(): + pass + + no = NewObj(property1='something') + assert no.property1 == 'something' + + +def test_custom_marking_no_init_2(): + @stix2.CustomMarking('x-new-obj2', [ + ('property1', stix2.properties.StringProperty(required=True)), + ]) + class NewObj2(object): + pass + + no2 = NewObj2(property1='something') + assert no2.property1 == 'something' + + @stix2.sdo.CustomObject('x-new-type', [ ('property1', stix2.properties.StringProperty(required=True)), ('property2', stix2.properties.IntegerProperty()), @@ -102,6 +124,15 @@ class NewType(object): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") + if "property3" in kwargs and not isinstance(kwargs.get("property3"), int): + raise TypeError("Must be integer!") + + +def test_custom_object_raises_exception(): + with pytest.raises(TypeError) as excinfo: + NewType(property1='something', property3='something', allow_custom=True) + + assert str(excinfo.value) == "Must be integer!" def test_custom_object_type(): @@ -117,7 +148,7 @@ def test_custom_object_type(): assert "'property2' is too small." in str(excinfo.value) -def test_custom_object_no_init(): +def test_custom_object_no_init_1(): @stix2.sdo.CustomObject('x-new-obj', [ ('property1', stix2.properties.StringProperty(required=True)), ]) @@ -127,6 +158,8 @@ def test_custom_object_no_init(): no = NewObj(property1='something') assert no.property1 == 'something' + +def test_custom_object_no_init_2(): @stix2.sdo.CustomObject('x-new-obj2', [ ('property1', stix2.properties.StringProperty(required=True)), ]) @@ -170,23 +203,36 @@ class NewObservable(): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") + if "property3" in kwargs and not isinstance(kwargs.get("property3"), int): + raise TypeError("Must be integer!") -def test_custom_observable_object(): +def test_custom_observable_object_1(): no = NewObservable(property1='something') assert no.property1 == 'something' + +def test_custom_observable_object_2(): with pytest.raises(stix2.exceptions.MissingPropertiesError) as excinfo: NewObservable(property2=42) assert excinfo.value.properties == ['property1'] assert "No values for required properties" in str(excinfo.value) + +def test_custom_observable_object_3(): with pytest.raises(ValueError) as excinfo: NewObservable(property1='something', property2=4) assert "'property2' is too small." in str(excinfo.value) -def test_custom_observable_object_no_init(): +def test_custom_observable_raises_exception(): + with pytest.raises(TypeError) as excinfo: + NewObservable(property1='something', property3='something', allow_custom=True) + + assert str(excinfo.value) == "Must be integer!" + + +def test_custom_observable_object_no_init_1(): @stix2.observables.CustomObservable('x-new-observable', [ ('property1', stix2.properties.StringProperty()), ]) @@ -196,6 +242,8 @@ def test_custom_observable_object_no_init(): no = NewObs(property1='something') assert no.property1 == 'something' + +def test_custom_observable_object_no_init_2(): @stix2.observables.CustomObservable('x-new-obs2', [ ('property1', stix2.properties.StringProperty()), ]) @@ -354,6 +402,15 @@ class NewExtension(): def __init__(self, property2=None, **kwargs): if property2 and property2 < 10: raise ValueError("'property2' is too small.") + if "property3" in kwargs and not isinstance(kwargs.get("property3"), int): + raise TypeError("Must be integer!") + + +def test_custom_extension_raises_exception(): + with pytest.raises(TypeError) as excinfo: + NewExtension(property1='something', property3='something', allow_custom=True) + + assert str(excinfo.value) == "Must be integer!" def test_custom_extension(): @@ -433,7 +490,7 @@ def test_custom_extension_empty_properties(): assert "'properties' must be a dict!" in str(excinfo.value) -def test_custom_extension_no_init(): +def test_custom_extension_no_init_1(): @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-extension', { 'property1': stix2.properties.StringProperty(required=True), }) @@ -443,6 +500,8 @@ def test_custom_extension_no_init(): ne = NewExt(property1="foobar") assert ne.property1 == "foobar" + +def test_custom_extension_no_init_2(): @stix2.observables.CustomExtension(stix2.DomainName, 'x-new-ext2', { 'property1': stix2.properties.StringProperty(required=True), }) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 3327ca9..d7f238a 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,9 +1,9 @@ import pytest from taxii2client import Collection -from stix2 import Filter, MemorySource -from stix2.sources import (CompositeDataSource, DataSink, DataSource, - DataStore, make_id, taxii) +from stix2 import Filter, MemorySink, MemorySource +from stix2.sources import (CompositeDataSource, DataSink, DataSource, make_id, + taxii) from stix2.sources.filters import apply_common_filters from stix2.utils import deduplicate @@ -20,11 +20,6 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) -@pytest.fixture -def ds(): - return DataSource() - - IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -127,21 +122,11 @@ STIX_OBJS1 = [IND1, IND2, IND3, IND4, IND5] def test_ds_abstract_class_smoke(): - ds1 = DataSource() - ds2 = DataSink() - ds3 = DataStore(source=ds1, sink=ds2) + with pytest.raises(TypeError): + DataSource() - with pytest.raises(NotImplementedError): - ds3.add(None) - - with pytest.raises(NotImplementedError): - ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") - - with pytest.raises(NotImplementedError): - ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") - - with pytest.raises(NotImplementedError): - ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + with pytest.raises(TypeError): + DataSink() def test_ds_taxii(collection): @@ -177,7 +162,8 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -def test_add_get_remove_filter(ds): +def test_add_get_remove_filter(): + ds = taxii.TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ @@ -226,7 +212,7 @@ def test_add_get_remove_filter(ds): ds.filters.update(valid_filters) -def test_apply_common_filters(ds): +def test_apply_common_filters(): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -374,35 +360,35 @@ def test_apply_common_filters(ds): assert len(resp) == 0 -def test_filters0(ds): +def test_filters0(): # "Return any object modified before 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters1(ds): +def test_filters1(): # "Return any object modified after 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters2(ds): +def test_filters2(): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters3(ds): +def test_filters3(): # "Return any object modified before or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters4(ds): +def test_filters4(): # Assert invalid Filter cannot be created with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") @@ -410,21 +396,21 @@ def test_filters4(ds): "for specified property: 'modified'") -def test_filters5(ds): +def test_filters5(): # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters6(ds): +def test_filters6(): # Test filtering on non-common property resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters7(ds): +def test_filters7(): # Test filtering on embedded property stix_objects = list(STIX_OBJS2) + [{ "type": "observed-data", @@ -463,7 +449,7 @@ def test_filters7(ds): assert len(resp) == 1 -def test_deduplicate(ds): +def test_deduplicate(): unique = deduplicate(STIX_OBJS1) # Only 3 objects are unique @@ -483,14 +469,14 @@ def test_deduplicate(ds): def test_add_remove_composite_datasource(): cds = CompositeDataSource() - ds1 = DataSource() - ds2 = DataSource() - ds3 = DataSink() + ds1 = MemorySource() + ds2 = MemorySource() + ds3 = MemorySink() with pytest.raises(TypeError) as excinfo: cds.add_data_sources([ds1, ds2, ds1, ds3]) assert str(excinfo.value) == ("DataSource (to be added) is not of type " - "stix2.DataSource. DataSource type is ''") + "stix2.DataSource. DataSource type is ''") cds.add_data_sources([ds1, ds2, ds1]) @@ -506,29 +492,66 @@ def test_composite_datasource_operations(): objects=STIX_OBJS1, spec_version="2.0", type="bundle") - cds = CompositeDataSource() - ds1 = MemorySource(stix_data=BUNDLE1) - ds2 = MemorySource(stix_data=STIX_OBJS2) + cds1 = CompositeDataSource() + ds1_1 = MemorySource(stix_data=BUNDLE1) + ds1_2 = MemorySource(stix_data=STIX_OBJS2) - cds.add_data_sources([ds1, ds2]) + cds2 = CompositeDataSource() + ds2_1 = MemorySource(stix_data=BUNDLE1) + ds2_2 = MemorySource(stix_data=STIX_OBJS2) - indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + cds1.add_data_sources([ds1_1, ds1_2]) + cds2.add_data_sources([ds2_1, ds2_2]) + + indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") # In STIX_OBJS2 changed the 'modified' property to a later time... assert len(indicators) == 2 - indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + cds1.add_data_sources([cds2]) + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" - query = [ + query1 = [ Filter("type", "=", "indicator") ] - results = cds.query(query) + query2 = [ + Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") + ] + + cds1.filters.update(query2) + + results = cds1.query(query1) # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" + + # There is only one indicator with different ID. Since we use the same data + # when deduplicated, only two indicators (one with different modified). + results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(results) == 2 + + # Since we have filters already associated with our CompositeSource providing + # nothing returns the same as cds1.query(query1) (the associated query is query2) + results = cds1.query([]) + assert len(results) == 3 + + +def test_composite_datastore_no_datasource(): + cds = CompositeDataSource() + + with pytest.raises(AttributeError) as excinfo: + cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert 'CompositeDataSource has no data source' in str(excinfo.value) diff --git a/stix2/test/test_environment.py b/stix2/test/test_environment.py index c669a33..84ca803 100644 --- a/stix2/test/test_environment.py +++ b/stix2/test/test_environment.py @@ -2,8 +2,22 @@ import pytest import stix2 -from .constants import (FAKE_TIME, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, - INDICATOR_KWARGS, MALWARE_ID) +from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, FAKE_TIME, IDENTITY_ID, + IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, + MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS) + + +@pytest.fixture +def ds(): + cam = stix2.Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) + idy = stix2.Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) + ind = stix2.Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) + mal = stix2.Malware(id=MALWARE_ID, **MALWARE_KWARGS) + rel1 = stix2.Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) + rel2 = stix2.Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) + rel3 = stix2.Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) + stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] + yield stix2.MemoryStore(stix_objs) def test_object_factory_created_by_ref_str(): @@ -150,6 +164,14 @@ def test_environment_no_datastore(): env.query(INDICATOR_ID) assert 'Environment has no data source' in str(excinfo.value) + with pytest.raises(AttributeError) as excinfo: + env.relationships(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + env.related_to(INDICATOR_ID) + assert 'Environment has no data source' in str(excinfo.value) + def test_environment_add_filters(): env = stix2.Environment(factory=stix2.ObjectFactory()) @@ -186,7 +208,7 @@ def test_parse_malware(): assert mal.name == "Cryptolocker" -def test_created_by(): +def test_creator_of(): identity = stix2.Identity(**IDENTITY_KWARGS) factory = stix2.ObjectFactory(created_by_ref=identity.id) env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) @@ -197,7 +219,7 @@ def test_created_by(): assert creator is identity -def test_created_by_no_datasource(): +def test_creator_of_no_datasource(): identity = stix2.Identity(**IDENTITY_KWARGS) factory = stix2.ObjectFactory(created_by_ref=identity.id) env = stix2.Environment(factory=factory) @@ -208,7 +230,7 @@ def test_created_by_no_datasource(): assert 'Environment has no data source' in str(excinfo.value) -def test_created_by_not_found(): +def test_creator_of_not_found(): identity = stix2.Identity(**IDENTITY_KWARGS) factory = stix2.ObjectFactory(created_by_ref=identity.id) env = stix2.Environment(store=stix2.MemoryStore(), factory=factory) @@ -216,3 +238,113 @@ def test_created_by_not_found(): ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) creator = env.creator_of(ind) assert creator is None + + +def test_creator_of_no_created_by_ref(): + env = stix2.Environment(store=stix2.MemoryStore()) + ind = env.create(stix2.Indicator, **INDICATOR_KWARGS) + creator = env.creator_of(ind) + assert creator is None + + +def test_relationships(ds): + env = stix2.Environment(store=ds) + mal = env.get(MALWARE_ID) + resp = env.relationships(mal) + + assert len(resp) == 3 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_no_id(ds): + env = stix2.Environment(store=ds) + mal = { + "type": "malware", + "name": "some variant" + } + with pytest.raises(ValueError) as excinfo: + env.relationships(mal) + assert "object has no 'id' property" in str(excinfo.value) + + +def test_relationships_by_type(ds): + env = stix2.Environment(store=ds) + mal = env.get(MALWARE_ID) + resp = env.relationships(mal, relationship_type='indicates') + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[0] + + +def test_relationships_by_source(ds): + env = stix2.Environment(store=ds) + resp = env.relationships(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[1] + + +def test_relationships_by_target(ds): + env = stix2.Environment(store=ds) + resp = env.relationships(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_type(ds): + env = stix2.Environment(store=ds) + resp = env.relationships(MALWARE_ID, relationship_type='uses', target_only=True) + + assert len(resp) == 1 + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_source(ds): + env = stix2.Environment(store=ds) + with pytest.raises(ValueError) as excinfo: + env.relationships(MALWARE_ID, target_only=True, source_only=True) + + assert 'not both' in str(excinfo.value) + + +def test_related_to(ds): + env = stix2.Environment(store=ds) + mal = env.get(MALWARE_ID) + resp = env.related_to(mal) + + assert len(resp) == 3 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_no_id(ds): + env = stix2.Environment(store=ds) + mal = { + "type": "malware", + "name": "some variant" + } + with pytest.raises(ValueError) as excinfo: + env.related_to(mal) + assert "object has no 'id' property" in str(excinfo.value) + + +def test_related_to_by_source(ds): + env = stix2.Environment(store=ds) + resp = env.related_to(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert resp[0]['id'] == IDENTITY_ID + + +def test_related_to_by_target(ds): + env = stix2.Environment(store=ds) + resp = env.related_to(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 7aaa3f5..68fc185 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -4,7 +4,12 @@ import shutil import pytest from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, - FileSystemSource, FileSystemStore, Filter, properties) + FileSystemSource, FileSystemStore, Filter, Identity, + Indicator, Malware, Relationship, properties) + +from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, + IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, + MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS) FS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stix2_data") @@ -40,6 +45,25 @@ def fs_sink(): shutil.rmtree(os.path.join(FS_PATH, "campaign"), True) +@pytest.fixture(scope='module') +def rel_fs_store(): + cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) + idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) + ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) + mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) + rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) + rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) + rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) + stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] + fs = FileSystemStore(FS_PATH) + for o in stix_objs: + fs.add(o) + yield fs + + for o in stix_objs: + os.remove(os.path.join(FS_PATH, o.type, o.id + '.json')) + + def test_filesystem_source_nonexistent_folder(): with pytest.raises(ValueError) as excinfo: FileSystemSource('nonexistent-folder') @@ -340,7 +364,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -352,9 +376,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): allow_custom=True) bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle, True) + fs_store.add(bundle, allow_custom=True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -367,11 +391,83 @@ def test_filesystem_custom_object(fs_store): pass newobj = NewObj(property1='something') - fs_store.add(newobj, True) + fs_store.add(newobj, allow_custom=True) - newobj_r = fs_store.get(newobj.id, True) + newobj_r = fs_store.get(newobj.id, allow_custom=True) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' # remove dir shutil.rmtree(os.path.join(FS_PATH, "x-new-obj"), True) + + +def test_relationships(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.relationships(mal) + + assert len(resp) == 3 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_type(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.relationships(mal, relationship_type='indicates') + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[0] + + +def test_relationships_by_source(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[1] + + +def test_relationships_by_target(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_type(rel_fs_store): + resp = rel_fs_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True) + + assert len(resp) == 1 + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_source(rel_fs_store): + with pytest.raises(ValueError) as excinfo: + rel_fs_store.relationships(MALWARE_ID, target_only=True, source_only=True) + + assert 'not both' in str(excinfo.value) + + +def test_related_to(rel_fs_store): + mal = rel_fs_store.get(MALWARE_ID) + resp = rel_fs_store.related_to(mal) + + assert len(resp) == 3 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_source(rel_fs_store): + resp = rel_fs_store.related_to(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_target(rel_fs_store): + resp = rel_fs_store.related_to(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py index a9415fe..8e3dd42 100644 --- a/stix2/test/test_identity.py +++ b/stix2/test/test_identity.py @@ -62,4 +62,15 @@ def test_parse_no_type(): "identity_class": "individual" }""") + +def test_identity_with_custom(): + identity = stix2.Identity( + name="John Smith", + identity_class="individual", + custom_properties={'x_foo': 'bar'} + ) + + assert identity.x_foo == "bar" + assert "x_foo" in identity.object_properties() + # TODO: Add other examples diff --git a/stix2/test/test_markings.py b/stix2/test/test_markings.py index 456bf92..d2271f0 100644 --- a/stix2/test/test_markings.py +++ b/stix2/test/test_markings.py @@ -187,7 +187,8 @@ def test_parse_marking_definition(data): ]) class NewMarking(object): def __init__(self, property2=None, **kwargs): - return + if "property3" in kwargs and not isinstance(kwargs.get("property3"), int): + raise TypeError("Must be integer!") def test_registered_custom_marking(): @@ -208,6 +209,13 @@ def test_registered_custom_marking(): assert marking_def.definition_type == "x-new-marking-type" +def test_registered_custom_marking_raises_exception(): + with pytest.raises(TypeError) as excinfo: + NewMarking(property1='something', property3='something', allow_custom=True) + + assert str(excinfo.value) == "Must be integer!" + + def test_not_registered_marking_raises_exception(): with pytest.raises(ValueError) as excinfo: # Used custom object on purpose to demonstrate a not-registered marking diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index 0603bf7..a7d88a8 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -1,9 +1,17 @@ +import os +import shutil + import pytest -from stix2 import (Bundle, Campaign, CustomObject, Filter, MemorySource, - MemoryStore, properties) +from stix2 import (Bundle, Campaign, CustomObject, Filter, Identity, Indicator, + Malware, MemorySource, MemoryStore, Relationship, + properties) from stix2.sources import make_id +from .constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, + IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, + MALWARE_ID, MALWARE_KWARGS, RELATIONSHIP_IDS) + IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -115,6 +123,19 @@ def mem_source(): yield MemorySource(STIX_OBJS1) +@pytest.fixture +def rel_mem_store(): + cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) + idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) + ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) + mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) + rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) + rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) + rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) + stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] + yield MemoryStore(stix_objs) + + def test_memory_source_get(mem_source): resp = mem_source.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") assert resp["id"] == "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" @@ -166,6 +187,22 @@ def test_memory_store_query_multiple_filters(mem_store): assert len(resp) == 1 +def test_memory_store_save_load_file(mem_store): + filename = 'memory_test/mem_store.json' + mem_store.save_to_file(filename) + contents = open(os.path.abspath(filename)).read() + + assert '"id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f",' in contents + assert '"id": "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f",' in contents + + mem_store2 = MemoryStore() + mem_store2.load_from_file(filename) + assert mem_store2.get("indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f") + assert mem_store2.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + shutil.rmtree(os.path.dirname(filename)) + + def test_memory_store_add_stix_object_str(mem_store): # add stix object string camp_id = "campaign--111111b6-1112-4fb0-111b-b111107ca70a" @@ -235,7 +272,7 @@ def test_memory_store_object_with_custom_property(mem_store): mem_store.add(camp, True) - camp_r = mem_store.get(camp.id, True) + camp_r = mem_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -249,7 +286,7 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store): bundle = Bundle(camp, allow_custom=True) mem_store.add(bundle, True) - bundle_r = mem_store.get(bundle.id, True) + bundle_r = mem_store.get(bundle.id) camp_r = bundle_r['objects'][0] assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -265,6 +302,78 @@ def test_memory_store_custom_object(mem_store): newobj = NewObj(property1='something') mem_store.add(newobj, True) - newobj_r = mem_store.get(newobj.id, True) + newobj_r = mem_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' + + +def test_relationships(rel_mem_store): + mal = rel_mem_store.get(MALWARE_ID) + resp = rel_mem_store.relationships(mal) + + assert len(resp) == 3 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[1] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_type(rel_mem_store): + mal = rel_mem_store.get(MALWARE_ID) + resp = rel_mem_store.relationships(mal, relationship_type='indicates') + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[0] + + +def test_relationships_by_source(rel_mem_store): + resp = rel_mem_store.relationships(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert resp[0]['id'] == RELATIONSHIP_IDS[1] + + +def test_relationships_by_target(rel_mem_store): + resp = rel_mem_store.relationships(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == RELATIONSHIP_IDS[0] for x in resp) + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_type(rel_mem_store): + resp = rel_mem_store.relationships(MALWARE_ID, relationship_type='uses', target_only=True) + + assert len(resp) == 1 + assert any(x['id'] == RELATIONSHIP_IDS[2] for x in resp) + + +def test_relationships_by_target_and_source(rel_mem_store): + with pytest.raises(ValueError) as excinfo: + rel_mem_store.relationships(MALWARE_ID, target_only=True, source_only=True) + + assert 'not both' in str(excinfo.value) + + +def test_related_to(rel_mem_store): + mal = rel_mem_store.get(MALWARE_ID) + resp = rel_mem_store.related_to(mal) + + assert len(resp) == 3 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_source(rel_mem_store): + resp = rel_mem_store.related_to(MALWARE_ID, source_only=True) + + assert len(resp) == 1 + assert any(x['id'] == IDENTITY_ID for x in resp) + + +def test_related_to_by_target(rel_mem_store): + resp = rel_mem_store.related_to(MALWARE_ID, target_only=True) + + assert len(resp) == 2 + assert any(x['id'] == CAMPAIGN_ID for x in resp) + assert any(x['id'] == INDICATOR_ID for x in resp) diff --git a/stix2/test/test_utils.py b/stix2/test/test_utils.py index c73bcd2..cbe5b0f 100644 --- a/stix2/test/test_utils.py +++ b/stix2/test/test_utils.py @@ -74,3 +74,11 @@ def test_get_dict(data): def test_get_dict_invalid(data): with pytest.raises(ValueError): stix2.utils.get_dict(data) + + +@pytest.mark.parametrize('stix_id, typ', [ + ('malware--d69c8146-ab35-4d50-8382-6fc80e641d43', 'malware'), + ('intrusion-set--899ce53f-13a0-479b-a0e4-67d46e241542', 'intrusion-set') +]) +def test_get_type_from_id(stix_id, typ): + assert stix2.utils.get_type_from_id(stix_id) == typ diff --git a/stix2/utils.py b/stix2/utils.py index cb02b15..6169dce 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -259,3 +259,7 @@ def get_class_hierarchy_names(obj): for cls in obj.__class__.__mro__: names.append(cls.__name__) return names + + +def get_type_from_id(stix_id): + return stix_id.split('--', 1)[0] diff --git a/stix2/v20/common.py b/stix2/v20/common.py index 51ded05..11cbec7 100644 --- a/stix2/v20/common.py +++ b/stix2/v20/common.py @@ -150,7 +150,14 @@ def CustomMarking(type='x-custom-marking', properties=None): def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) - cls.__init__(self, **kwargs) + try: + cls.__init__(self, **kwargs) + except (AttributeError, TypeError) as e: + # Don't accidentally catch errors raised in a custom __init__() + if ("has no attribute '__init__'" in str(e) or + str(e) == "object.__init__() takes no parameters"): + return + raise e _register_marking(_Custom) return _Custom diff --git a/stix2/v21/common.py b/stix2/v21/common.py index 7a7feb0..4087189 100644 --- a/stix2/v21/common.py +++ b/stix2/v21/common.py @@ -169,7 +169,6 @@ def CustomMarking(type='x-custom-marking', properties=None): def custom_builder(cls): class _Custom(cls, _STIXBase): - _type = type _properties = OrderedDict() @@ -180,7 +179,14 @@ def CustomMarking(type='x-custom-marking', properties=None): def __init__(self, **kwargs): _STIXBase.__init__(self, **kwargs) - cls.__init__(self, **kwargs) + try: + cls.__init__(self, **kwargs) + except (AttributeError, TypeError) as e: + # Don't accidentally catch errors raised in a custom __init__() + if ("has no attribute '__init__'" in str(e) or + str(e) == "object.__init__() takes no parameters"): + return + raise e _register_marking(_Custom) return _Custom diff --git a/stix2/version.py b/stix2/version.py index 493f741..6a9beea 100644 --- a/stix2/version.py +++ b/stix2/version.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/tox.ini b/tox.ini index fe4fb01..bfc8c1b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,py35,py36,pycodestyle,isort-check +envlist = py27,py34,py35,py36,pycodestyle,isort-check [testenv] deps = @@ -36,7 +36,6 @@ commands = [travis] python = 2.7: py27, pycodestyle - 3.3: py33, pycodestyle 3.4: py34, pycodestyle 3.5: py35, pycodestyle 3.6: py36, pycodestyle