From c4668f5dc1f2f15d4d8b3a388f973a3780a7e6e1 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Wed, 7 Nov 2018 10:10:06 -0500 Subject: [PATCH] Revert "Multi version filesystem store" --- docs/Makefile | 2 +- .../datastore/stix2.datastore.filesystem.rst | 2 +- .../api/datastore/stix2.datastore.filters.rst | 2 +- docs/api/datastore/stix2.datastore.memory.rst | 2 +- docs/api/datastore/stix2.datastore.taxii.rst | 2 +- .../stix2.markings.granular_markings.rst | 2 +- .../stix2.markings.object_markings.rst | 2 +- docs/api/markings/stix2.markings.utils.rst | 2 +- docs/api/stix2.core.rst | 2 +- docs/api/stix2.datastore.rst | 2 +- docs/api/stix2.environment.rst | 2 +- docs/api/stix2.exceptions.rst | 2 +- docs/api/stix2.markings.rst | 2 +- docs/api/stix2.patterns.rst | 2 +- docs/api/stix2.properties.rst | 2 +- docs/api/stix2.utils.rst | 2 +- docs/api/stix2.v20.common.rst | 2 +- docs/api/stix2.v20.observables.rst | 2 +- docs/api/stix2.v20.sdo.rst | 2 +- docs/api/stix2.v20.sro.rst | 2 +- docs/api/stix2.workbench.rst | 2 +- stix2/datastore/filesystem.py | 510 ++++------------- stix2/datastore/memory.py | 23 +- ...0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json} | 0 ...0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json} | 0 ...774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json} | 0 ...7e150503-88e7-4861-866b-ff1ac82c4475.json} | 0 ...ae676644-d2d2-41b7-af7e-9bed1b55898c.json} | 0 ...b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json} | 0 ...95ddb356-7ba0-4bd9-a889-247262b8946f.json} | 2 +- ...d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json} | 2 +- ...c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json} | 2 +- .../20181101232448446000.json | 11 - ...a653431d-6a5e-4600-8ad3-609b5af57064.json} | 2 +- ...f3bdec95-3d62-42d9-a840-29630f6cdc1a.json} | 2 +- ...6b616fc1-1505-48e3-8b2c-0d19337bff38.json} | 2 +- .../20181101232448456000.json | 27 - .../20181101232448457000.json | 27 - ...92ec0cbd-2c30-44a2-b270-73f4ec949841.json} | 2 +- ...96b08451-b27a-4ff6-893f-790e26393a8e.json} | 0 ...b42378e0-f147-496f-992a-26a49705395b.json} | 0 ...-fa42a846-8d90-4e51-bc29-71d5b4802168.json | 2 +- ...0d4a7788-7f3b-4df8-a498-31a38003c883.json} | 2 +- ...0e55ee98-0c6d-43d4-b424-b18a0036b227.json} | 2 +- ...1e91cd45-a725-4965-abe3-700694374432.json} | 2 +- ...3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json} | 2 +- ...3a3ed0b2-0c38-441f-ac40-53b873e545d1.json} | 2 +- ...592d0c31-e61f-495e-a60e-70d7be59a719.json} | 2 +- ...70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json} | 2 +- ...8797579b-e3be-4209-a71b-255a4d08243d.json} | 2 +- ...03342581-f790-4f03-ba41-e82e67392e23.json} | 0 ...242f3da3-4425-4d11-8f5c-b842886da966.json} | 0 stix2/test/test_datastore_filesystem.py | 541 ++---------------- stix2/utils.py | 19 - 54 files changed, 211 insertions(+), 1021 deletions(-) rename stix2/test/stix2_data/attack-pattern/{attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json => attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json => attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json => attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json => attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json => attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json => attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/course-of-action/{course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json => course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/course-of-action/{course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json => course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/identity/{identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json => identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json} (99%) mode change 100644 => 100755 delete mode 100644 stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20181101232448446000.json rename stix2/test/stix2_data/intrusion-set/{intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json => intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/intrusion-set/{intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json => intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/malware/{malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json => malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json} (99%) mode change 100644 => 100755 delete mode 100644 stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json delete mode 100644 stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json rename stix2/test/stix2_data/malware/{malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json => malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/malware/{malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json => malware--96b08451-b27a-4ff6-893f-790e26393a8e.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/malware/{malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json => malware--b42378e0-f147-496f-992a-26a49705395b.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json => relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json => relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json => relationship--1e91cd45-a725-4965-abe3-700694374432.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json => relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json => relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json => relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json => relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/relationship/{relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json => relationship--8797579b-e3be-4209-a71b-255a4d08243d.json} (99%) mode change 100644 => 100755 rename stix2/test/stix2_data/tool/{tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json => tool--03342581-f790-4f03-ba41-e82e67392e23.json} (100%) mode change 100644 => 100755 rename stix2/test/stix2_data/tool/{tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json => tool--242f3da3-4425-4d11-8f5c-b842886da966.json} (100%) mode change 100644 => 100755 diff --git a/docs/Makefile b/docs/Makefile index e1582e8..0c4ce84 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.filesystem.rst b/docs/api/datastore/stix2.datastore.filesystem.rst index 2b0d2ee..665df66 100644 --- a/docs/api/datastore/stix2.datastore.filesystem.rst +++ b/docs/api/datastore/stix2.datastore.filesystem.rst @@ -2,4 +2,4 @@ filesystem ========================== .. automodule:: stix2.datastore.filesystem - :members: + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.filters.rst b/docs/api/datastore/stix2.datastore.filters.rst index 3c9f0b1..b556754 100644 --- a/docs/api/datastore/stix2.datastore.filters.rst +++ b/docs/api/datastore/stix2.datastore.filters.rst @@ -2,4 +2,4 @@ filters ======================= .. automodule:: stix2.datastore.filters - :members: + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.memory.rst b/docs/api/datastore/stix2.datastore.memory.rst index eda42cb..b0521c7 100644 --- a/docs/api/datastore/stix2.datastore.memory.rst +++ b/docs/api/datastore/stix2.datastore.memory.rst @@ -2,4 +2,4 @@ memory ====================== .. automodule:: stix2.datastore.memory - :members: + :members: \ No newline at end of file diff --git a/docs/api/datastore/stix2.datastore.taxii.rst b/docs/api/datastore/stix2.datastore.taxii.rst index f1c43bf..68389a0 100644 --- a/docs/api/datastore/stix2.datastore.taxii.rst +++ b/docs/api/datastore/stix2.datastore.taxii.rst @@ -2,4 +2,4 @@ taxii ===================== .. automodule:: stix2.datastore.taxii - :members: + :members: \ No newline at end of file diff --git a/docs/api/markings/stix2.markings.granular_markings.rst b/docs/api/markings/stix2.markings.granular_markings.rst index d64ebc9..b4a7160 100644 --- a/docs/api/markings/stix2.markings.granular_markings.rst +++ b/docs/api/markings/stix2.markings.granular_markings.rst @@ -2,4 +2,4 @@ granular_markings ================================ .. automodule:: stix2.markings.granular_markings - :members: + :members: \ No newline at end of file diff --git a/docs/api/markings/stix2.markings.object_markings.rst b/docs/api/markings/stix2.markings.object_markings.rst index 8e8de67..d861c87 100644 --- a/docs/api/markings/stix2.markings.object_markings.rst +++ b/docs/api/markings/stix2.markings.object_markings.rst @@ -2,4 +2,4 @@ object_markings ============================== .. automodule:: stix2.markings.object_markings - :members: + :members: \ No newline at end of file diff --git a/docs/api/markings/stix2.markings.utils.rst b/docs/api/markings/stix2.markings.utils.rst index 66793aa..ee59b6c 100644 --- a/docs/api/markings/stix2.markings.utils.rst +++ b/docs/api/markings/stix2.markings.utils.rst @@ -2,4 +2,4 @@ utils ==================== .. automodule:: stix2.markings.utils - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.core.rst b/docs/api/stix2.core.rst index dbd5256..f0e98d4 100644 --- a/docs/api/stix2.core.rst +++ b/docs/api/stix2.core.rst @@ -2,4 +2,4 @@ core ========== .. automodule:: stix2.core - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.datastore.rst b/docs/api/stix2.datastore.rst index 0d90987..4af05a9 100644 --- a/docs/api/stix2.datastore.rst +++ b/docs/api/stix2.datastore.rst @@ -2,4 +2,4 @@ datastore =============== .. automodule:: stix2.datastore - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.environment.rst b/docs/api/stix2.environment.rst index 6b44ba5..fb5c7b7 100644 --- a/docs/api/stix2.environment.rst +++ b/docs/api/stix2.environment.rst @@ -2,4 +2,4 @@ environment ================= .. automodule:: stix2.environment - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.exceptions.rst b/docs/api/stix2.exceptions.rst index ad8ddf3..a8d498e 100644 --- a/docs/api/stix2.exceptions.rst +++ b/docs/api/stix2.exceptions.rst @@ -2,4 +2,4 @@ exceptions ================ .. automodule:: stix2.exceptions - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.markings.rst b/docs/api/stix2.markings.rst index 881fda1..9819fe7 100644 --- a/docs/api/stix2.markings.rst +++ b/docs/api/stix2.markings.rst @@ -2,4 +2,4 @@ markings ============== .. automodule:: stix2.markings - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.patterns.rst b/docs/api/stix2.patterns.rst index f95ec34..ec7b42c 100644 --- a/docs/api/stix2.patterns.rst +++ b/docs/api/stix2.patterns.rst @@ -2,4 +2,4 @@ patterns ============== .. automodule:: stix2.patterns - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.properties.rst b/docs/api/stix2.properties.rst index e357ef4..c3db9ff 100644 --- a/docs/api/stix2.properties.rst +++ b/docs/api/stix2.properties.rst @@ -2,4 +2,4 @@ properties ================ .. automodule:: stix2.properties - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.utils.rst b/docs/api/stix2.utils.rst index 49a1e16..4091fb5 100644 --- a/docs/api/stix2.utils.rst +++ b/docs/api/stix2.utils.rst @@ -2,4 +2,4 @@ utils =========== .. automodule:: stix2.utils - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.common.rst b/docs/api/stix2.v20.common.rst index 8cec059..0c7a296 100644 --- a/docs/api/stix2.v20.common.rst +++ b/docs/api/stix2.v20.common.rst @@ -2,4 +2,4 @@ common ================ .. automodule:: stix2.v20.common - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.observables.rst b/docs/api/stix2.v20.observables.rst index 4d9803a..d31f75f 100644 --- a/docs/api/stix2.v20.observables.rst +++ b/docs/api/stix2.v20.observables.rst @@ -2,4 +2,4 @@ observables ===================== .. automodule:: stix2.v20.observables - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.sdo.rst b/docs/api/stix2.v20.sdo.rst index a115d5b..c4c97f8 100644 --- a/docs/api/stix2.v20.sdo.rst +++ b/docs/api/stix2.v20.sdo.rst @@ -2,4 +2,4 @@ sdo ============= .. automodule:: stix2.v20.sdo - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.v20.sro.rst b/docs/api/stix2.v20.sro.rst index 397cf29..379ed18 100644 --- a/docs/api/stix2.v20.sro.rst +++ b/docs/api/stix2.v20.sro.rst @@ -2,4 +2,4 @@ sro ============= .. automodule:: stix2.v20.sro - :members: + :members: \ No newline at end of file diff --git a/docs/api/stix2.workbench.rst b/docs/api/stix2.workbench.rst index 8fa2544..19345f0 100644 --- a/docs/api/stix2.workbench.rst +++ b/docs/api/stix2.workbench.rst @@ -2,4 +2,4 @@ workbench =============== .. automodule:: stix2.workbench - :members: + :members: \ No newline at end of file diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index 95c9050..c13b02c 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -3,362 +3,13 @@ Python STIX 2.0 FileSystem Source/Sink """ -import errno import json import os -import stat -import pytz -import six - -from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, FilterSet, apply_common_filters -from stix2.utils import get_type_from_id, is_marking - - -def _timestamp2filename(timestamp): - """ - Encapsulates a way to create unique filenames based on an object's - "modified" property value. This should not include an extension. - - :param timestamp: A timestamp, as a datetime.datetime object. - """ - # Different times will only produce different file names if all timestamps - # are in the same time zone! So if timestamp is timezone-aware convert - # to UTC just to be safe. If naive, just use as-is. - if timestamp.tzinfo is not None: - timestamp = timestamp.astimezone(pytz.utc) - - return timestamp.strftime("%Y%m%d%H%M%S%f") - - -class AuthSet(object): - """ - Represents either a whitelist or blacklist of values, where/what we - must/must not search to find objects which match a query. (Maybe "AuthSet" - isn't the right name, but determining authorization is a typical context in - which black/white lists are used.) - - The set may be empty. For a whitelist, this means you mustn't search - anywhere, which means the query was impossible to match, so you can skip - searching altogether. For a blacklist, this means nothing is excluded - and you must search everywhere. - """ - - BLACK = 0 - WHITE = 1 - - def __init__(self, allowed, prohibited): - """ - Initialize this AuthSet from the given sets of allowed and/or - prohibited values. The type of set (black or white) is determined - from the allowed and/or prohibited values given. - - :param allowed: A set of allowed values (or None if no allow filters - were found in the query) - :param prohibited: A set of prohibited values (not None) - """ - if allowed is None: - self.__values = prohibited - self.__type = AuthSet.BLACK - - else: - # There was at least one allow filter, so create a whitelist. But - # any matching prohibited values create a combination of conditions - # which can never match. So exclude those. - self.__values = allowed - prohibited - self.__type = AuthSet.WHITE - - @property - def values(self): - """ - Get the values in this white/blacklist, as a set. - """ - return self.__values - - @property - def auth_type(self): - """ - Get the type of set: AuthSet.WHITE or AuthSet.BLACK. - """ - return self.__type - - def __repr__(self): - return "{}list: {}".format( - "white" if self.auth_type == AuthSet.WHITE else "black", - self.values - ) - - -# A fixed, reusable AuthSet which accepts anything. It came in handy. -_AUTHSET_ANY = AuthSet(None, set()) - - -def _update_allow(allow_set, value): - """ - Updates the given set of "allow" values. The first time an update to the - set occurs, the value(s) are added. Thereafter, since all filters are - implicitly AND'd, the given values are intersected with the existing allow - set, which may remove values. At the end, it may even wind up empty. - - :param allow_set: The allow set, or None - :param value: The value(s) to add (single value, or iterable of values) - :return: The updated allow set (not None) - """ - adding_seq = hasattr(value, "__iter__") and \ - not isinstance(value, six.string_types) - - if allow_set is None: - allow_set = set() - if adding_seq: - allow_set.update(value) - else: - allow_set.add(value) - - else: - # strangely, the "&=" operator requires a set on the RHS - # whereas the method allows any iterable. - if adding_seq: - allow_set.intersection_update(value) - else: - allow_set.intersection_update({value}) - - return allow_set - - -def _find_search_optimizations(filters): - """ - Searches through all the filters, and creates white/blacklists of types and - IDs, which can be used to optimize the filesystem search. - - :param filters: An iterable of filter objects representing a query - :return: A 2-tuple of AuthSet objects: the first is for object types, and - the second is for object IDs. - """ - - # The basic approach to this is to determine what is allowed and - # prohibited, independently, and then combine them to create the final - # white/blacklists. - - allowed_types = allowed_ids = None - prohibited_types = set() - prohibited_ids = set() - - for filter_ in filters: - if filter_.property == "type": - if filter_.op in ("=", "in"): - allowed_types = _update_allow(allowed_types, filter_.value) - elif filter_.op == "!=": - prohibited_types.add(filter_.value) - - elif filter_.property == "id": - if filter_.op == "=": - # An "allow" ID filter implies a type filter too, since IDs - # contain types within them. - allowed_ids = _update_allow(allowed_ids, filter_.value) - allowed_types = _update_allow(allowed_types, - get_type_from_id(filter_.value)) - elif filter_.op == "!=": - prohibited_ids.add(filter_.value) - elif filter_.op == "in": - allowed_ids = _update_allow(allowed_ids, filter_.value) - allowed_types = _update_allow(allowed_types, ( - get_type_from_id(id_) for id_ in filter_.value - )) - - opt_types = AuthSet(allowed_types, prohibited_types) - opt_ids = AuthSet(allowed_ids, prohibited_ids) - - # If we have both type and ID whitelists, perform a type-based intersection - # on them, to further optimize. (Some of the cross-property constraints - # occur above; this is essentially a second pass which operates on the - # final whitelists, which among other things, incorporates any of the - # prohibitions found above.) - if opt_types.auth_type == AuthSet.WHITE and \ - opt_ids.auth_type == AuthSet.WHITE: - - opt_types.values.intersection_update( - get_type_from_id(id_) for id_ in opt_ids.values - ) - - opt_ids.values.intersection_update( - id_ for id_ in opt_ids.values - if get_type_from_id(id_) in opt_types.values - ) - - return opt_types, opt_ids - - -def _get_matching_dir_entries(parent_dir, auth_set, st_mode_test=None, ext=""): - """ - Search a directory (non-recursively), and find entries which match the - given criteria. - - :param parent_dir: The directory to search - :param auth_set: an AuthSet instance, which represents a black/whitelist - filter on filenames - :param st_mode_test: A callable allowing filtering based on the type of - directory entry. E.g. just get directories, or just get files. It - will be passed the st_mode field of a stat() structure and should - return True to include the file, or False to exclude it. Easy thing to - do is pass one of the stat module functions, e.g. stat.S_ISREG. If - None, don't filter based on entry type. - :param ext: Determines how names from auth_set match up to directory - entries, and allows filtering by extension. The extension is added - to auth_set values to obtain directory entries; it is removed from - directory entries to obtain auth_set values. In this way, auth_set - may be treated as having only "basenames" of the entries. Only entries - having the given extension will be included in the results. If not - empty, the extension MUST include a leading ".". The default is the - empty string, which will result in direct comparisons, and no - extension-based filtering. - :return: A list of directory entries matching the criteria. These will not - have any path info included; they will just be bare names. - :raises OSError: If there are errors accessing directory contents or - stat()'ing files - """ - - results = [] - if auth_set.auth_type == AuthSet.WHITE: - for value in auth_set.values: - try: - filename = value + ext - s = os.stat(os.path.join(parent_dir, filename)) - if not st_mode_test or st_mode_test(s.st_mode): - results.append(filename) - except OSError as e: - if e.errno != errno.ENOENT: - raise e - # else, file-not-found is ok, just skip - - else: # auth_set is a blacklist - for entry in os.listdir(parent_dir): - if ext: - auth_name, this_ext = os.path.splitext(entry) - if this_ext != ext: - continue - else: - auth_name = entry - - if auth_name in auth_set.values: - continue - - try: - s = os.stat(os.path.join(parent_dir, entry)) - if not st_mode_test or st_mode_test(s.st_mode): - results.append(entry) - except OSError as e: - if e.errno != errno.ENOENT: - raise e - # else, file-not-found is ok, just skip - - return results - - -def _check_object_from_file(query, filepath): - """ - Read a STIX object from the given file, and check it against the given - filters. - - :param query: Iterable of filters - :param filepath: Path to file to read - :return: The STIX object, as a dict, if the object passes the filters. If - not, None is returned. - :raises TypeError: If the file had invalid content - :raises IOError: If there are problems opening/reading the file - """ - try: - with open(filepath, "r") as f: - stix_obj = json.load(f) - - if stix_obj["type"] == "bundle": - stix_obj = stix_obj["objects"][0] - - # naive STIX type checking - stix_obj["type"] - stix_obj["id"] - - except (ValueError, KeyError): # likely not a JSON file - raise TypeError( - "STIX JSON object at '{0}' could either not be parsed " - "to JSON or was not valid STIX JSON".format( - filepath)) - - # check against other filters, add if match - result = next(apply_common_filters([stix_obj], query), None) - - return result - - -def _search_versioned(query, type_path, auth_ids): - """ - Searches the given directory, which contains data for STIX objects of a - particular versioned type (i.e. not markings), and return any which match - the query. - - :param query: The query to match against - :param type_path: The directory with type-specific STIX object files - :param auth_ids: Search optimization based on object ID - :return: A list of all matching objects - :raises TypeError: If any objects had invalid content - :raises IOError, OSError: If there were any problems opening/reading files - """ - results = [] - id_dirs = _get_matching_dir_entries(type_path, auth_ids, - stat.S_ISDIR) - for id_dir in id_dirs: - id_path = os.path.join(type_path, id_dir) - - # This leverages a more sophisticated function to do a simple thing: - # get all the JSON files from a directory. I guess it does give us - # file type checking, ensuring we only get regular files. - version_files = _get_matching_dir_entries(id_path, _AUTHSET_ANY, - stat.S_ISREG, ".json") - for version_file in version_files: - version_path = os.path.join(id_path, version_file) - - try: - stix_obj = _check_object_from_file(query, version_path) - if stix_obj: - results.append(stix_obj) - except IOError as e: - if e.errno != errno.ENOENT: - raise e - # else, file-not-found is ok, just skip - - return results - - -def _search_markings(query, markings_path, auth_ids): - """ - Searches the given directory, which contains markings data, and return any - which match the query. - - :param query: The query to match against - :param markings_path: The directory with STIX markings files - :param auth_ids: Search optimization based on object ID - :return: A list of all matching objects - :raises TypeError: If any objects had invalid content - :raises IOError: If there were any problems opening/reading files - """ - results = [] - id_files = _get_matching_dir_entries(markings_path, auth_ids, stat.S_ISREG, - ".json") - for id_file in id_files: - id_path = os.path.join(markings_path, id_file) - - try: - stix_obj = _check_object_from_file(query, id_path) - if stix_obj: - results.append(stix_obj) - except IOError as e: - if e.errno != errno.ENOENT: - raise e - # else, file-not-found is ok, just skip - - return results +from stix2.utils import deduplicate, get_class_hierarchy_names class FileSystemStore(DataStoreMixin): @@ -426,23 +77,15 @@ class FileSystemSink(DataSink): def _check_path_and_write(self, stix_obj): """Write the given STIX object to a file in the STIX file directory. """ - type_dir = os.path.join(self._stix_dir, stix_obj["type"]) - if is_marking(stix_obj): - filename = stix_obj["id"] - obj_dir = type_dir - else: - filename = _timestamp2filename(stix_obj["modified"]) - obj_dir = os.path.join(type_dir, stix_obj["id"]) + path = os.path.join(self._stix_dir, stix_obj["type"], stix_obj["id"] + ".json") - file_path = os.path.join(obj_dir, filename + ".json") - - if not os.path.exists(obj_dir): - os.makedirs(obj_dir) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) if self.bundlify: stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom) - with open(file_path, "w") as f: + with open(path, "w") as f: f.write(str(stix_obj)) def add(self, stix_data=None, version=None): @@ -461,18 +104,25 @@ class FileSystemSink(DataSink): the Bundle contained, but not the Bundle itself. """ - if isinstance(stix_data, Bundle): - # recursively add individual STIX objects - for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, version=version) - - elif isinstance(stix_data, _STIXBase): + if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition') + for x in get_class_hierarchy_names(stix_data)): # adding python STIX object self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) - self.add(stix_data, version=version) + if stix_data["type"] == "bundle": + # extract STIX objects + for stix_obj in stix_data.get("objects", []): + self.add(stix_obj, version=version) + else: + # adding json-formatted STIX + self._check_path_and_write(stix_data,) + + elif isinstance(stix_data, Bundle): + # recursively add individual STIX objects + for stix_obj in stix_data.get("objects", []): + self.add(stix_obj, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects @@ -526,15 +176,12 @@ class FileSystemSource(DataSource): a python STIX object and then returned """ - all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters) + query = [Filter("id", "=", stix_id)] + + all_data = self.query(query=query, version=version, _composite_filters=_composite_filters) if all_data: - if is_marking(stix_id): - # Markings are unversioned; there shouldn't be more than one - # result. - stix_obj = all_data[0] - else: - stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1] + stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] else: stix_obj = None @@ -559,8 +206,7 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - query = [Filter("id", "=", stix_id)] - return self.query(query, version=version, _composite_filters=_composite_filters) + return [self.get(stix_id=stix_id, version=version, _composite_filters=_composite_filters)] def query(self, query=None, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. @@ -593,25 +239,105 @@ class FileSystemSource(DataSource): if _composite_filters: query.add(_composite_filters) - auth_types, auth_ids = _find_search_optimizations(query) + # extract any filters that are for "type" or "id" , as we can then do + # filtering before reading in the STIX objects. A STIX 'type' filter + # can reduce the query to a single sub-directory. A STIX 'id' filter + # allows for the fast checking of the file names versus loading it. + file_filters = self._parse_file_filters(query) - type_dirs = _get_matching_dir_entries(self._stix_dir, auth_types, - stat.S_ISDIR) - for type_dir in type_dirs: - type_path = os.path.join(self._stix_dir, type_dir) + # establish which subdirectories can be avoided in query + # by decluding as many as possible. A filter with "type" as the property + # means that certain STIX object types can be ruled out, and thus + # the corresponding subdirectories as well + include_paths = [] + declude_paths = [] + if "type" in [filter.property for filter in file_filters]: + for filter in file_filters: + if filter.property == "type": + if filter.op == "=": + include_paths.append(os.path.join(self._stix_dir, filter.value)) + elif filter.op == "!=": + declude_paths.append(os.path.join(self._stix_dir, filter.value)) + else: + # have to walk entire STIX directory + include_paths.append(self._stix_dir) - if type_dir == "marking-definition": - type_results = _search_markings(query, type_path, auth_ids) + # if a user specifies a "type" filter like "type = ", + # the filter is reducing the search space to single stix object types + # (and thus single directories). This makes such a filter more powerful + # than "type != " bc the latter is substracting + # only one type of stix object type (and thus only one directory), + # As such the former type of filters are given preference over the latter; + # i.e. if both exist in a query, that latter type will be ignored + + if not include_paths: + # user has specified types that are not wanted (i.e. "!=") + # so query will look in all STIX directories that are not + # the specified type. Compile correct dir paths + for dir in os.listdir(self._stix_dir): + if os.path.abspath(os.path.join(self._stix_dir, dir)) not in declude_paths: + include_paths.append(os.path.abspath(os.path.join(self._stix_dir, dir))) + + # grab stix object ID as well - if present in filters, as + # may forgo the loading of STIX content into memory + if "id" in [filter.property for filter in file_filters]: + for filter in file_filters: + if filter.property == "id" and filter.op == "=": + id_ = filter.value + break else: - type_results = _search_versioned(query, type_path, auth_ids) + id_ = None + else: + id_ = None - all_data.extend(type_results) + # now iterate through all STIX objs + for path in include_paths: + for root, dirs, files in os.walk(path): + for file_ in files: + if not file_.endswith(".json"): + # skip non '.json' files as more likely to be random non-STIX files + continue + + if not id_ or id_ == file_.split(".")[0]: + # have to load into memory regardless to evaluate other filters + try: + stix_obj = json.load(open(os.path.join(root, file_))) + + if stix_obj["type"] == "bundle": + stix_obj = stix_obj["objects"][0] + + # naive STIX type checking + stix_obj["type"] + stix_obj["id"] + + except (ValueError, KeyError): # likely not a JSON file + raise TypeError("STIX JSON object at '{0}' could either not be parsed to " + "JSON or was not valid STIX JSON".format(os.path.join(root, file_))) + + # check against other filters, add if match + all_data.extend(apply_common_filters([stix_obj], query)) + + all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [ - parse(stix_obj_dict, allow_custom=self.allow_custom, - version=version) - for stix_obj_dict in all_data - ] + stix_objs = [parse(stix_obj_dict, allow_custom=self.allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs + + def _parse_file_filters(self, query): + """Extract STIX common filters. + + Possibly speeds up querying STIX objects from the file system. + + Extracts filters that are for the "id" and "type" property of + a STIX object. As the file directory is organized by STIX + object type with filenames that are equivalent to the STIX + object ID, these filters can be used first to reduce the + search space of a FileSystemStore (or FileSystemSink). + + """ + file_filters = [] + for filter_ in query: + if filter_.property == "id" or filter_.property == "type": + file_filters.append(filter_) + return file_filters diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index 3e065ad..4b82c34 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -10,7 +10,6 @@ from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import FilterSet, apply_common_filters -from stix2.utils import is_marking def _add(store, stix_data=None, allow_custom=True, version=None): @@ -44,7 +43,7 @@ def _add(store, stix_data=None, allow_custom=True, version=None): # Map ID directly to the object, if it is a marking. Otherwise, # map to a family, so we can track multiple versions. - if is_marking(stix_obj): + if _is_marking(stix_obj): store._data[stix_obj.id] = stix_obj else: @@ -57,6 +56,22 @@ def _add(store, stix_data=None, allow_custom=True, version=None): obj_family.add(stix_obj) +def _is_marking(obj_or_id): + """Determines whether the given object or object ID is/is for a marking + definition. + + :param obj_or_id: A STIX object or object ID as a string. + :return: True if a marking definition, False otherwise. + """ + + if isinstance(obj_or_id, _STIXBase): + id_ = obj_or_id.id + else: + id_ = obj_or_id + + return id_.startswith("marking-definition--") + + class _ObjectFamily(object): """ An internal implementation detail of memory sources/sinks/stores. @@ -240,7 +255,7 @@ class MemorySource(DataSource): """ stix_obj = None - if is_marking(stix_id): + if _is_marking(stix_id): stix_obj = self._data.get(stix_id) else: object_family = self._data.get(stix_id) @@ -276,7 +291,7 @@ class MemorySource(DataSource): """ results = [] stix_objs_to_filter = None - if is_marking(stix_id): + if _is_marking(stix_id): stix_obj = self._data.get(stix_id) if stix_obj: stix_objs_to_filter = [stix_obj] diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json b/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json rename to stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json index 541ede1..bf14aa7 --- a/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json @@ -13,4 +13,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json rename to stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json index 669aae5..cb9cfe2 --- a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json @@ -6,4 +6,4 @@ "modified": "2017-05-31T21:30:41.022744Z", "name": "Data from Network Shared Drive Mitigation", "type": "course-of-action" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json rename to stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json index d110a09..77d4464 --- a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json +++ b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json @@ -12,4 +12,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20181101232448446000.json b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20181101232448446000.json deleted file mode 100644 index 7528b44..0000000 --- a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20181101232448446000.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "identity", - "id": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-06-01T00:00:00.000Z", - "modified": "2018-11-01T23:24:48.446Z", - "name": "The MITRE Corporation", - "identity_class": "organization", - "labels": [ - "version two" - ] -} diff --git a/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json b/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json rename to stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json index 648ed94..10ef3a5 --- a/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json +++ b/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json @@ -51,4 +51,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json b/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json rename to stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json index bf3daa6..84b75b1 --- a/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json +++ b/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json @@ -41,4 +41,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json rename to stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json index c60200b..669f00c --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json @@ -31,4 +31,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json deleted file mode 100644 index af47f27..0000000 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448456000.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.456Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version two" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] -} diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json deleted file mode 100644 index 446fb26..0000000 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20181101232448457000.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": "malware", - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "created": "2017-05-31T21:32:58.226Z", - "modified": "2018-11-01T23:24:48.457Z", - "name": "Rover", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", - "labels": [ - "version three" - ], - "external_references": [ - { - "source_name": "mitre-attack", - "url": "https://attack.mitre.org/wiki/Software/S0090", - "external_id": "S0090" - }, - { - "source_name": "Palo Alto Rover", - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" - } - ], - "object_marking_refs": [ - "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ] -} diff --git a/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json b/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json rename to stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json index 50c8a5d..9eaf8ad --- a/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json +++ b/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json @@ -31,4 +31,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json b/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json rename to stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json diff --git a/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json b/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json rename to stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json diff --git a/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json b/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json index be93c62..bcae183 100755 --- a/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json +++ b/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json @@ -13,4 +13,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json b/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json rename to stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json index 0f4a32a..ac59925 --- a/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json +++ b/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json b/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json rename to stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json index e5e1e87..ee97edf --- a/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json +++ b/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json b/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json rename to stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json index 9651425..ff0d8cc --- a/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json +++ b/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json b/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json rename to stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json index 7e355fc..36d1482 --- a/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json +++ b/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json b/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json rename to stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json index f537309..888cc3b --- a/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json +++ b/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json b/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json rename to stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json index 47008f0..d9078d1 --- a/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json +++ b/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json b/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json rename to stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json index d697277..ef1c4b2 --- a/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json +++ b/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json b/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json old mode 100644 new mode 100755 similarity index 99% rename from stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json rename to stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json index d7f2ff7..1f20179 --- a/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json +++ b/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json @@ -17,4 +17,4 @@ ], "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json b/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json rename to stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json diff --git a/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json b/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json old mode 100644 new mode 100755 similarity index 100% rename from stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json rename to stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py index 0f0ca0a..17661d0 100644 --- a/stix2/test/test_datastore_filesystem.py +++ b/stix2/test/test_datastore_filesystem.py @@ -1,20 +1,12 @@ -import datetime -import errno import json import os import shutil -import stat import pytest -import pytz from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, FileSystemSource, FileSystemStore, Filter, Identity, - Indicator, Malware, MarkingDefinition, Relationship, - TLPMarking, parse, properties) -from stix2.datastore.filesystem import (AuthSet, _find_search_optimizations, - _get_matching_dir_entries, - _timestamp2filename) + Indicator, Malware, Relationship, properties) from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS, @@ -105,20 +97,7 @@ def rel_fs_store(): yield fs for o in stix_objs: - filepath = os.path.join(FS_PATH, o.type, o.id, - _timestamp2filename(o.modified) + '.json') - - # Some test-scoped fixtures (e.g. fs_store) delete all campaigns, so by - # the time this module-scoped fixture tears itself down, it may find - # its campaigns already gone, which causes not-found errors. - try: - os.remove(filepath) - except OSError as e: - # 3 is the ERROR_PATH_NOT_FOUND windows error code. Which has an - # errno symbolic value, but not the windows meaning... - if e.errno in (errno.ENOENT, 3): - continue - raise e + os.remove(os.path.join(FS_PATH, o.type, o.id + '.json')) def test_filesystem_source_nonexistent_folder(): @@ -153,32 +132,27 @@ def test_filesystem_source_bad_stix_file(fs_source, bad_stix_files): assert "could either not be parsed to JSON or was not valid STIX JSON" in str(e) -def test_filesystem_source_get_object(fs_source): - # get (latest) object +def test_filesytem_source_get_object(fs_source): + # get object mal = fs_source.get("malware--6b616fc1-1505-48e3-8b2c-0d19337bff38") assert mal.id == "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38" assert mal.name == "Rover" - assert mal.modified == datetime.datetime(2018, 11, 1, 23, 24, 48, 457000, - pytz.utc) -def test_filesystem_source_get_nonexistent_object(fs_source): +def test_filesytem_source_get_nonexistent_object(fs_source): ind = fs_source.get("indicator--6b616fc1-1505-48e3-8b2c-0d19337bff38") assert ind is None -def test_filesystem_source_all_versions(fs_source): - ids = fs_source.all_versions( - "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" - ) - assert len(ids) == 2 - assert all(id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" - for id_ in ids) - assert all(id_.name == "The MITRE Corporation" for id_ in ids) - assert all(id_.type == "identity" for id_ in ids) +def test_filesytem_source_all_versions(fs_source): + # all versions - (currently not a true all versions call as FileSystem cant have multiple versions) + id_ = fs_source.get("identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5") + assert id_.id == "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" + assert id_.name == "The MITRE Corporation" + assert id_.type == "identity" -def test_filesystem_source_query_single(fs_source): +def test_filesytem_source_query_single(fs_source): # query2 is_2 = fs_source.query([Filter("external_references.external_id", '=', "T1027")]) assert len(is_2) == 1 @@ -208,16 +182,14 @@ def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): fs_sink.add(camp1) - filepath = os.path.join(FS_PATH, "campaign", camp1.id, - _timestamp2filename(camp1.modified) + ".json") - assert os.path.exists(filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) camp1_r = fs_source.get(camp1.id) assert camp1_r.id == camp1.id assert camp1_r.name == "Hannibal" assert "War Elephant" in camp1_r.aliases - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): @@ -228,30 +200,19 @@ def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): "objective": "German and French Intelligence Services", "aliases": ["Purple Robes"], "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "created": "2017-05-31T21:31:53.197755Z", - "modified": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z" } fs_sink.add(camp2) - # Need to get the exact "modified" timestamp which would have been - # in effect at the time the object was saved to the sink, which determines - # the filename it would have been saved as. It may not be exactly the same - # as what's in the dict, since the parsing process can enforce a precision - # constraint (e.g. truncate to milliseconds), which results in a slightly - # different name. - camp2obj = parse(camp2) - filepath = os.path.join(FS_PATH, "campaign", camp2obj["id"], - _timestamp2filename(camp2obj["modified"]) + ".json") - - assert os.path.exists(filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) camp2_r = fs_source.get(camp2["id"]) assert camp2_r.id == camp2["id"] assert camp2_r.name == camp2["name"] assert "Purple Robes" in camp2_r.aliases - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): @@ -267,74 +228,53 @@ def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): "objective": "Bulgarian, Albanian and Romanian Intelligence Services", "aliases": ["Huns"], "id": "campaign--b8f86161-ccae-49de-973a-4ca320c62478", - "created": "2017-05-31T21:31:53.197755Z", - "modified": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z" } ] } fs_sink.add(bund) - camp_obj = parse(bund["objects"][0]) - filepath = os.path.join(FS_PATH, "campaign", camp_obj["id"], - _timestamp2filename(camp_obj["modified"]) + ".json") - - assert os.path.exists(filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) camp3_r = fs_source.get(bund["objects"][0]["id"]) assert camp3_r.id == bund["objects"][0]["id"] assert camp3_r.name == bund["objects"][0]["name"] assert "Huns" in camp3_r.aliases - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): # add json-encoded stix obj camp4 = '{"type": "campaign", "id":"campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d",'\ - ' "created":"2017-05-31T21:31:53.197755Z",'\ - ' "modified":"2017-05-31T21:31:53.197755Z",'\ - ' "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' fs_sink.add(camp4) - camp4obj = parse(camp4) - filepath = os.path.join(FS_PATH, "campaign", - "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d", - _timestamp2filename(camp4obj["modified"]) + ".json") - - assert os.path.exists(filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d" + ".json")) camp4_r = fs_source.get("campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d") assert camp4_r.id == "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d" assert camp4_r.name == "Ghengis Khan" - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): # add json-encoded stix bundle bund2 = '{"type": "bundle", "id": "bundle--3d267103-8475-4d8f-b321-35ec6eccfa37",' \ ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",' \ - ' "created":"2017-05-31T21:31:53.197755Z",'\ - ' "modified":"2017-05-31T21:31:53.197755Z",'\ - ' "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' + ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' fs_sink.add(bund2) - bund2obj = parse(bund2) - camp_obj = bund2obj["objects"][0] - - filepath = os.path.join(FS_PATH, "campaign", - "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b", - _timestamp2filename(camp_obj["modified"]) + ".json") - - assert os.path.exists(filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b" + ".json")) camp5_r = fs_source.get("campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b") assert camp5_r.id == "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b" assert camp5_r.name == "Spartacus" - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) def test_filesystem_sink_add_objects_list(fs_sink, fs_source): @@ -349,23 +289,13 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source): "objective": "Central and Eastern Europe military commands and departments", "aliases": ["The Frenchmen"], "id": "campaign--122818b6-1112-4fb0-b11b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z", - "modified": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z" } fs_sink.add([camp6, camp7]) - camp7obj = parse(camp7) - - camp6filepath = os.path.join(FS_PATH, "campaign", camp6.id, - _timestamp2filename(camp6["modified"]) + - ".json") - camp7filepath = os.path.join( - FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a", - _timestamp2filename(camp7obj["modified"]) + ".json") - - assert os.path.exists(camp6filepath) - assert os.path.exists(camp7filepath) + assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) + assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a" + ".json")) camp6_r = fs_source.get(camp6.id) assert camp6_r.id == camp6.id @@ -376,24 +306,8 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source): assert "The Frenchmen" in camp7_r.aliases # remove all added objects - os.remove(camp6filepath) - os.remove(camp7filepath) - - -def test_filesystem_sink_marking(fs_sink): - marking = MarkingDefinition( - definition_type="tlp", - definition=TLPMarking(tlp="green") - ) - - fs_sink.add(marking) - marking_filepath = os.path.join( - FS_PATH, "marking-definition", marking["id"] + ".json" - ) - - assert os.path.exists(marking_filepath) - - os.remove(marking_filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) + os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) def test_filesystem_store_get_stored_as_bundle(fs_store): @@ -409,9 +323,8 @@ def test_filesystem_store_get_stored_as_object(fs_store): def test_filesystem_store_all_versions(fs_store): - rels = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1") - assert len(rels) == 1 - rel = rels[0] + # all versions() - (note at this time, all_versions() is still not applicable to FileSystem, as only one version is ever stored) + rel = fs_store.all_versions("relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1")[0] assert rel.id == "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1" assert rel.type == "relationship" @@ -434,7 +347,7 @@ def test_filesystem_store_query_single_filter(fs_store): def test_filesystem_store_empty_query(fs_store): results = fs_store.query() # returns all - assert len(results) == 29 + assert len(results) == 26 assert "tool--242f3da3-4425-4d11-8f5c-b842886da966" in [obj.id for obj in results] assert "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" in [obj.id for obj in results] @@ -448,7 +361,7 @@ def test_filesystem_store_query_multiple_filters(fs_store): def test_filesystem_store_query_dont_include_type_folder(fs_store): results = fs_store.query(Filter("type", "!=", "tool")) - assert len(results) == 27 + assert len(results) == 24 def test_filesystem_store_add(fs_store): @@ -462,11 +375,8 @@ def test_filesystem_store_add(fs_store): assert camp1_r.id == camp1.id assert camp1_r.name == camp1.name - filepath = os.path.join(FS_PATH, "campaign", camp1_r.id, - _timestamp2filename(camp1_r.modified) + ".json") - # remove - os.remove(filepath) + os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) def test_filesystem_store_add_as_bundle(): @@ -477,10 +387,7 @@ def test_filesystem_store_add_as_bundle(): aliases=["Ragnar"]) fs_store.add(camp1) - filepath = os.path.join(FS_PATH, "campaign", camp1.id, - _timestamp2filename(camp1.modified) + ".json") - - with open(filepath) as bundle_file: + with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: assert '"type": "bundle"' in bundle_file.read() camp1_r = fs_store.get(camp1.id) @@ -505,26 +412,6 @@ def test_filesystem_store_add_invalid_object(fs_store): assert 'JSON formatted STIX bundle' in str(excinfo.value) -def test_filesystem_store_add_marking(fs_store): - marking = MarkingDefinition( - definition_type="tlp", - definition=TLPMarking(tlp="green") - ) - - fs_store.add(marking) - marking_filepath = os.path.join( - FS_PATH, "marking-definition", marking["id"] + ".json" - ) - - assert os.path.exists(marking_filepath) - - marking_r = fs_store.get(marking["id"]) - assert marking_r["id"] == marking["id"] - assert marking_r["definition"]["tlp"] == "green" - - os.remove(marking_filepath) - - def test_filesystem_object_with_custom_property(fs_store): camp = Campaign(name="Scipio Africanus", objective="Defeat the Carthaginians", @@ -640,357 +527,3 @@ def test_related_to_by_target(rel_fs_store): assert len(resp) == 2 assert any(x['id'] == CAMPAIGN_ID for x in resp) assert any(x['id'] == INDICATOR_ID for x in resp) - - -def test_auth_set_white1(): - auth_set = AuthSet({"A"}, set()) - - assert auth_set.auth_type == AuthSet.WHITE - assert auth_set.values == {"A"} - - -def test_auth_set_white2(): - auth_set = AuthSet(set(), set()) - - assert auth_set.auth_type == AuthSet.WHITE - assert len(auth_set.values) == 0 - - -def test_auth_set_white3(): - auth_set = AuthSet({"A", "B"}, {"B", "C"}) - - assert auth_set.auth_type == AuthSet.WHITE - assert auth_set.values == {"A"} - - -def test_auth_set_black1(): - auth_set = AuthSet(None, {"B", "C"}) - - assert auth_set.auth_type == AuthSet.BLACK - assert auth_set.values == {"B", "C"} - - -def test_optimize_types1(): - filters = [ - Filter("type", "=", "foo") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"foo"} - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types2(): - filters = [ - Filter("type", "=", "foo"), - Filter("type", "=", "bar") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert len(auth_types.values) == 0 - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types3(): - filters = [ - Filter("type", "in", ["A", "B", "C"]), - Filter("type", "in", ["B", "C", "D"]) - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"B", "C"} - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types4(): - filters = [ - Filter("type", "in", ["A", "B", "C"]), - Filter("type", "in", ["D", "E", "F"]) - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert len(auth_types.values) == 0 - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types5(): - filters = [ - Filter("type", "in", ["foo", "bar"]), - Filter("type", "!=", "bar") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"foo"} - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types6(): - filters = [ - Filter("type", "!=", "foo"), - Filter("type", "!=", "bar") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.BLACK - assert auth_types.values == {"foo", "bar"} - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types7(): - filters = [ - Filter("type", "=", "foo"), - Filter("type", "!=", "foo") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert len(auth_types.values) == 0 - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types8(): - filters = [] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.BLACK - assert len(auth_types.values) == 0 - assert auth_ids.auth_type == AuthSet.BLACK - assert len(auth_ids.values) == 0 - - -def test_optimize_types_ids1(): - filters = [ - Filter("type", "in", ["foo", "bar"]), - Filter("id", "=", "foo--00000000-0000-0000-0000-000000000000") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"foo"} - assert auth_ids.auth_type == AuthSet.WHITE - assert auth_ids.values == {"foo--00000000-0000-0000-0000-000000000000"} - - -def test_optimize_types_ids2(): - filters = [ - Filter("type", "=", "foo"), - Filter("id", "=", "bar--00000000-0000-0000-0000-000000000000") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert len(auth_types.values) == 0 - assert auth_ids.auth_type == AuthSet.WHITE - assert len(auth_ids.values) == 0 - - -def test_optimize_types_ids3(): - filters = [ - Filter("type", "in", ["foo", "bar"]), - Filter("id", "!=", "bar--00000000-0000-0000-0000-000000000000") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"foo", "bar"} - assert auth_ids.auth_type == AuthSet.BLACK - assert auth_ids.values == {"bar--00000000-0000-0000-0000-000000000000"} - - -def test_optimize_types_ids4(): - filters = [ - Filter("type", "in", ["A", "B", "C"]), - Filter("id", "in", [ - "B--00000000-0000-0000-0000-000000000000", - "C--00000000-0000-0000-0000-000000000000", - "D--00000000-0000-0000-0000-000000000000", - ]) - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"B", "C"} - assert auth_ids.auth_type == AuthSet.WHITE - assert auth_ids.values == { - "B--00000000-0000-0000-0000-000000000000", - "C--00000000-0000-0000-0000-000000000000" - } - - -def test_optimize_types_ids5(): - filters = [ - Filter("type", "in", ["A", "B", "C"]), - Filter("type", "!=", "C"), - Filter("id", "in", [ - "B--00000000-0000-0000-0000-000000000000", - "C--00000000-0000-0000-0000-000000000000", - "D--00000000-0000-0000-0000-000000000000" - ]), - Filter("id", "!=", "D--00000000-0000-0000-0000-000000000000") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"B"} - assert auth_ids.auth_type == AuthSet.WHITE - assert auth_ids.values == {"B--00000000-0000-0000-0000-000000000000"} - - -def test_optimize_types_ids6(): - filters = [ - Filter("id", "=", "A--00000000-0000-0000-0000-000000000000") - ] - - auth_types, auth_ids = _find_search_optimizations(filters) - - assert auth_types.auth_type == AuthSet.WHITE - assert auth_types.values == {"A"} - assert auth_ids.auth_type == AuthSet.WHITE - assert auth_ids.values == {"A--00000000-0000-0000-0000-000000000000"} - - -def test_search_auth_set_white1(): - auth_set = AuthSet( - {"attack-pattern", "doesntexist"}, - set() - ) - - results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) - assert results == ["attack-pattern"] - - results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISREG) - assert len(results) == 0 - - -def test_search_auth_set_white2(): - auth_set = AuthSet( - { - "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", - "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841" - - }, - { - "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", - "malware--96b08451-b27a-4ff6-893f-790e26393a8e", - "doesntexist" - } - ) - - results = _get_matching_dir_entries( - os.path.join(FS_PATH, "malware"), - auth_set, stat.S_ISDIR - ) - - assert results == ["malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"] - - -def test_search_auth_set_white3(): - auth_set = AuthSet({"20170531213258226477", "doesntexist"}, set()) - - results = _get_matching_dir_entries( - os.path.join(FS_PATH, "malware", - "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"), - auth_set, stat.S_ISREG, ".json" - ) - - assert results == ["20170531213258226477.json"] - - -def test_search_auth_set_black1(): - auth_set = AuthSet( - None, - {"tool--242f3da3-4425-4d11-8f5c-b842886da966", "doesntexist"} - ) - - results = _get_matching_dir_entries( - os.path.join(FS_PATH, "tool"), - auth_set, stat.S_ISDIR - ) - - assert set(results) == { - "tool--03342581-f790-4f03-ba41-e82e67392e23" - } - - -def test_search_auth_set_white_empty(): - auth_set = AuthSet( - set(), - set() - ) - - results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) - - assert len(results) == 0 - - -def test_search_auth_set_black_empty(rel_fs_store): - # Ensure rel_fs_store fixture has run so that the type directories are - # predictable (it adds "campaign"). - auth_set = AuthSet( - None, - set() - ) - - results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) - - # Should get all dirs - assert set(results) == { - "attack-pattern", - "campaign", - "course-of-action", - "identity", - "indicator", - "intrusion-set", - "malware", - "marking-definition", - "relationship", - "tool" - } - - -def test_timestamp2filename_naive(): - dt = datetime.datetime( - 2010, 6, 15, - 8, 30, 10, 1234 - ) - - filename = _timestamp2filename(dt) - assert filename == "20100615083010001234" - - -def test_timestamp2filename_tz(): - # one hour west of UTC (i.e. an hour earlier) - tz = pytz.FixedOffset(-60) - dt = datetime.datetime( - 2010, 6, 15, - 7, 30, 10, 1234, - tz - ) - - filename = _timestamp2filename(dt) - assert filename == "20100615083010001234" diff --git a/stix2/utils.py b/stix2/utils.py index 63b3d45..64c4a32 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -7,8 +7,6 @@ import json from dateutil import parser import pytz -import stix2.base - from .exceptions import (InvalidValueError, RevokeError, UnmodifiablePropertyError) @@ -366,20 +364,3 @@ def remove_custom_stix(stix_obj): def get_type_from_id(stix_id): return stix_id.split('--', 1)[0] - - -def is_marking(obj_or_id): - """Determines whether the given object or object ID is/is for a marking - definition. - - :param obj_or_id: A STIX object or object ID as a string. - :return: True if a marking definition, False otherwise. - """ - - if isinstance(obj_or_id, (stix2.base._STIXBase, dict)): - result = obj_or_id["type"] == "marking-definition" - else: - # it's a string ID - result = obj_or_id.startswith("marking-definition--") - - return result