From 9486b46f77f53defebc7b26a2e3b5ce922dafdef Mon Sep 17 00:00:00 2001 From: Michael Chisholm Date: Thu, 25 Oct 2018 21:03:27 -0400 Subject: [PATCH] Add multi-version support to the filesystem datastore. Factored out the _is_marking() function from the memory datastore module to utils so it can be reused, and changed both filesystem and memory datastore modules to import and use it. --- stix2/datastore/filesystem.py | 456 +++++++++++++++++- stix2/datastore/memory.py | 23 +- .../20170531213019735010.json} | 44 +- .../20170531213026496201.json} | 38 +- .../20170531213029458940.json} | 32 +- .../20170531213045139269.json} | 32 +- .../20170531213041022897.json} | 32 +- .../20170531213032662702.json} | 32 +- .../20170531213026495974.json} | 18 +- .../20170531213041022744.json} | 12 +- ...-c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json | 15 - .../20170601000000000000.json | 15 + .../20170531213149412497.json} | 62 +-- .../20170531213153197755.json} | 46 +- .../20170531213258226477.json} | 34 +- .../20170531213326565056.json} | 34 +- .../20170531213248482655.json} | 36 +- .../20170531213215263882.json} | 36 +- ...-fa42a846-8d90-4e51-bc29-71d5b4802168.json | 14 +- .../20170531213327182784.json} | 22 +- .../20170531213327082801.json} | 22 +- .../20170531213327018782.json} | 22 +- .../20170531213327100701.json} | 22 +- .../20170531213327143973.json} | 22 +- .../20170531213327021562.json} | 22 +- .../20170531213327044387.json} | 22 +- .../20170531213327051532.json} | 22 +- .../20170531213231601148.json} | 42 +- .../20170531213212684914.json} | 36 +- stix2/test/test_datastore_filesystem.py | 445 ++++++++++++++++- stix2/utils.py | 19 + 31 files changed, 1270 insertions(+), 459 deletions(-) rename stix2/test/stix2_data/attack-pattern/{attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json => attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json} (82%) mode change 100755 => 100644 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json => attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json} (75%) mode change 100755 => 100644 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json => attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json} (71%) mode change 100755 => 100644 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json => attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json} (74%) mode change 100755 => 100644 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json => attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json} (74%) mode change 100755 => 100644 rename stix2/test/stix2_data/attack-pattern/{attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json => attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json} (73%) mode change 100755 => 100644 rename stix2/test/stix2_data/course-of-action/{course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json => course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json} (61%) mode change 100755 => 100644 rename stix2/test/stix2_data/course-of-action/{course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json => course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json} (72%) mode change 100755 => 100644 delete mode 100755 stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json create mode 100644 stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json rename stix2/test/stix2_data/intrusion-set/{intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json => intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json} (65%) mode change 100755 => 100644 rename stix2/test/stix2_data/intrusion-set/{intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json => intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json} (74%) mode change 100755 => 100644 rename stix2/test/stix2_data/malware/{malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json => malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json} (64%) mode change 100755 => 100644 rename stix2/test/stix2_data/malware/{malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json => malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json} (63%) mode change 100755 => 100644 rename stix2/test/stix2_data/malware/{malware--96b08451-b27a-4ff6-893f-790e26393a8e.json => malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json} (65%) mode change 100755 => 100644 rename stix2/test/stix2_data/malware/{malware--b42378e0-f147-496f-992a-26a49705395b.json => malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json} (62%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json => relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json} (58%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json => relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json} (59%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--1e91cd45-a725-4965-abe3-700694374432.json => relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json} (57%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json => relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json} (59%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json => relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json} (58%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json => relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json} (57%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json => relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json} (59%) mode change 100755 => 100644 rename stix2/test/stix2_data/relationship/{relationship--8797579b-e3be-4209-a71b-255a4d08243d.json => relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json} (59%) mode change 100755 => 100644 rename stix2/test/stix2_data/tool/{tool--03342581-f790-4f03-ba41-e82e67392e23.json => tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json} (64%) mode change 100755 => 100644 rename stix2/test/stix2_data/tool/{tool--242f3da3-4425-4d11-8f5c-b842886da966.json => tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json} (58%) mode change 100755 => 100644 diff --git a/stix2/datastore/filesystem.py b/stix2/datastore/filesystem.py index c13b02c..94978b0 100644 --- a/stix2/datastore/filesystem.py +++ b/stix2/datastore/filesystem.py @@ -3,13 +3,367 @@ Python STIX 2.0 FileSystem Source/Sink """ +import errno import json import os +import stat +import six + +from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import Filter, FilterSet, apply_common_filters -from stix2.utils import deduplicate, get_class_hierarchy_names +from stix2.utils import deduplicate, is_marking + + +def _timestamp2filename(timestamp): + """ + Encapsulates a way to create unique filenames based on an object's + "modified" property value. This should not include an extension. + + :param timestamp: A timestamp, as a datetime.datetime object. + """ + # Different times will only produce different file names if all timestamps + # are in the same time zone! (Should I convert to UTC just to be safe?) + return timestamp.strftime("%Y%m%d%H%M%S%f") + + +def _type_from_id(id_): + """Extract the type from a STIX identifier""" + dd_idx = id_.find("--") + if dd_idx == -1: + raise Exception( + "Invalid ID: {}. Must have format --.".format(id_) + ) + return id_[:dd_idx] + + +class AuthSet(object): + """ + Represents either a whitelist or blacklist of values, where/what we + must/must not search to find objects which match a query. (Maybe "AuthSet" + isn't the right name, but determining authorization is a typical context in + which black/white lists are used.) + + The set may be empty. For a whitelist, this means you mustn't search + anywhere, which means the query was impossible to match, so you can skip + searching altogether. For a blacklist, this means nothing is excluded + and you must search everywhere. + """ + + BLACK = 0 + WHITE = 1 + + def __init__(self, allowed, prohibited): + """ + Initialize this AuthSet from the given sets of allowed and/or + prohibited values. The type of set (black or white) is determined + from the allowed and/or prohibited values given. + + :param allowed: A set of allowed values (or None if no allow filters + were found in the query) + :param prohibited: A set of prohibited values (not None) + """ + if allowed is None: + self.__values = prohibited + self.__type = AuthSet.BLACK + + else: + # There was at least one allow filter, so create a whitelist. But + # any matching prohibited values create a combination of conditions + # which can never match. So exclude those. + self.__values = allowed - prohibited + self.__type = AuthSet.WHITE + + @property + def values(self): + """ + Get the values in this white/blacklist, as a set. + """ + return self.__values + + @property + def auth_type(self): + """ + Get the type of set: AuthSet.WHITE or AuthSet.BLACK. + """ + return self.__type + + def __repr__(self): + return "{}list: {}".format( + "white" if self.auth_type == AuthSet.WHITE else "black", + self.values + ) + + +# A fixed, reusable AuthSet which accepts anything. It came in handy. +_AUTHSET_ANY = AuthSet(None, set()) + + +def _update_allow(allow_set, value): + """ + Updates the given set of "allow" values. The first time an update to the + set occurs, the value(s) are added. Thereafter, since all filters are + implicitly AND'd, the given values are intersected with the existing allow + set, which may remove values. At the end, it may even wind up empty. + + :param allow_set: The allow set, or None + :param value: The value(s) to add (single value, or iterable of values) + :return: The updated allow set (not None) + """ + adding_seq = hasattr(value, "__iter__") and \ + not isinstance(value, six.string_types) + + if allow_set is None: + allow_set = set() + if adding_seq: + allow_set.update(value) + else: + allow_set.add(value) + + else: + # strangely, the "&=" operator requires a set on the RHS + # whereas the method allows any iterable. + if adding_seq: + allow_set.intersection_update(value) + else: + allow_set.intersection_update({value}) + + return allow_set + + +def _find_search_optimizations(filters): + """ + Searches through all the filters, and creates white/blacklists of types and + IDs, which can be used to optimize the filesystem search. + + :param filters: An iterable of filter objects representing a query + :return: A 2-tuple of AuthSet objects: the first is for object types, and + the second is for object IDs. + """ + + # The basic approach to this is to determine what is allowed and + # prohibited, independently, and then combine them to create the final + # white/blacklists. + + allowed_types = allowed_ids = None + prohibited_types = set() + prohibited_ids = set() + + for filter_ in filters: + if filter_.property == "type": + if filter_.op in ("=", "in"): + allowed_types = _update_allow(allowed_types, filter_.value) + elif filter_.op == "!=": + prohibited_types.add(filter_.value) + + elif filter_.property == "id": + if filter_.op == "=": + # An "allow" ID filter implies a type filter too, since IDs + # contain types within them. + allowed_ids = _update_allow(allowed_ids, filter_.value) + allowed_types = _update_allow(allowed_types, + _type_from_id(filter_.value)) + elif filter_.op == "!=": + prohibited_ids.add(filter_.value) + elif filter_.op == "in": + allowed_ids = _update_allow(allowed_ids, filter_.value) + allowed_types = _update_allow(allowed_types, ( + _type_from_id(id_) for id_ in filter_.value + )) + + opt_types = AuthSet(allowed_types, prohibited_types) + opt_ids = AuthSet(allowed_ids, prohibited_ids) + + # If we have both type and ID whitelists, perform a type-based intersection + # on them, to further optimize. (Some of the cross-property constraints + # occur above; this is essentially a second pass which operates on the + # final whitelists, which among other things, incorporates any of the + # prohibitions found above.) + if opt_types.auth_type == AuthSet.WHITE and \ + opt_ids.auth_type == AuthSet.WHITE: + + opt_types.values.intersection_update( + _type_from_id(id_) for id_ in opt_ids.values + ) + + opt_ids.values.intersection_update( + id_ for id_ in opt_ids.values + if _type_from_id(id_) in opt_types.values + ) + + return opt_types, opt_ids + + +def _get_matching_dir_entries(parent_dir, auth_set, st_mode_test=None, ext=""): + """ + Search a directory (non-recursively), and find entries which match the + given criteria. + + :param parent_dir: The directory to search + :param auth_set: an AuthSet instance, which represents a black/whitelist + filter on filenames + :param st_mode_test: A callable allowing filtering based on the type of + directory entry. E.g. just get directories, or just get files. It + will be passed the st_mode field of a stat() structure and should + return True to include the file, or False to exclude it. Easy thing to + do is pass one of the stat module functions, e.g. stat.S_ISREG. If + None, don't filter based on entry type. + :param ext: Determines how names from auth_set match up to directory + entries, and allows filtering by extension. The extension is added + to auth_set values to obtain directory entries; it is removed from + directory entries to obtain auth_set values. In this way, auth_set + may be treated as having only "basenames" of the entries. Only entries + having the given extension will be included in the results. If not + empty, the extension MUST include a leading ".". The default is the + empty string, which will result in direct comparisons, and no + extension-based filtering. + :return: A list of directory entries matching the criteria. These will not + have any path info included; they will just be bare names. + :raises OSError: If there are errors accessing directory contents or + stat()'ing files + """ + + results = [] + if auth_set.auth_type == AuthSet.WHITE: + for value in auth_set.values: + try: + filename = value + ext + s = os.stat(os.path.join(parent_dir, filename)) + if not st_mode_test or st_mode_test(s.st_mode): + results.append(filename) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + # else, file-not-found is ok, just skip + + else: # auth_set is a blacklist + for entry in os.listdir(parent_dir): + if ext: + auth_name, this_ext = os.path.splitext(entry) + if this_ext != ext: + continue + else: + auth_name = entry + + if auth_name in auth_set.values: + continue + + try: + s = os.stat(os.path.join(parent_dir, entry)) + if not st_mode_test or st_mode_test(s.st_mode): + results.append(entry) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + # else, file-not-found is ok, just skip + + return results + + +def _check_object_from_file(query, filepath): + """ + Read a STIX object from the given file, and check it against the given + filters. + + :param query: Iterable of filters + :param filepath: Path to file to read + :return: The STIX object, as a dict, if the object passes the filters. If + not, None is returned. + :raises TypeError: If the file had invalid content + :raises IOError: If there are problems opening/reading the file + """ + try: + with open(filepath, "r") as f: + stix_obj = json.load(f) + + if stix_obj["type"] == "bundle": + stix_obj = stix_obj["objects"][0] + + # naive STIX type checking + stix_obj["type"] + stix_obj["id"] + + except (ValueError, KeyError): # likely not a JSON file + raise TypeError( + "STIX JSON object at '{0}' could either not be parsed " + "to JSON or was not valid STIX JSON".format( + filepath)) + + # check against other filters, add if match + result = next(apply_common_filters([stix_obj], query), None) + + return result + + +def _search_versioned(query, type_path, auth_ids): + """ + Searches the given directory, which contains data for STIX objects of a + particular versioned type (i.e. not markings), and return any which match + the query. + + :param query: The query to match against + :param type_path: The directory with type-specific STIX object files + :param auth_ids: Search optimization based on object ID + :return: A list of all matching objects + :raises TypeError: If any objects had invalid content + :raises IOError, OSError: If there were any problems opening/reading files + """ + results = [] + id_dirs = _get_matching_dir_entries(type_path, auth_ids, + stat.S_ISDIR) + for id_dir in id_dirs: + id_path = os.path.join(type_path, id_dir) + + # This leverages a more sophisticated function to do a simple thing: + # get all the JSON files from a directory. I guess it does give us + # file type checking, ensuring we only get regular files. + version_files = _get_matching_dir_entries(id_path, _AUTHSET_ANY, + stat.S_ISREG, ".json") + for version_file in version_files: + version_path = os.path.join(id_path, version_file) + + try: + stix_obj = _check_object_from_file(query, version_path) + if stix_obj: + results.append(stix_obj) + except IOError as e: + if e.errno != errno.ENOENT: + raise e + # else, file-not-found is ok, just skip + + return results + + +def _search_markings(query, markings_path, auth_ids): + """ + Searches the given directory, which contains markings data, and return any + which match the query. + + :param query: The query to match against + :param markings_path: The directory with STIX markings files + :param auth_ids: Search optimization based on object ID + :return: A list of all matching objects + :raises TypeError: If any objects had invalid content + :raises IOError: If there were any problems opening/reading files + """ + results = [] + id_files = _get_matching_dir_entries(markings_path, auth_ids, stat.S_ISREG, + ".json") + for id_file in id_files: + id_path = os.path.join(markings_path, id_file) + + try: + stix_obj = _check_object_from_file(query, id_path) + if stix_obj: + results.append(stix_obj) + except IOError as e: + if e.errno != errno.ENOENT: + raise e + # else, file-not-found is ok, just skip + + return results class FileSystemStore(DataStoreMixin): @@ -77,15 +431,23 @@ class FileSystemSink(DataSink): def _check_path_and_write(self, stix_obj): """Write the given STIX object to a file in the STIX file directory. """ - path = os.path.join(self._stix_dir, stix_obj["type"], stix_obj["id"] + ".json") + type_dir = os.path.join(self._stix_dir, stix_obj["type"]) + if is_marking(stix_obj): + filename = stix_obj.id + obj_dir = type_dir + else: + filename = _timestamp2filename(stix_obj.modified) + obj_dir = os.path.join(type_dir, stix_obj["id"]) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) + file_path = os.path.join(obj_dir, filename + ".json") + + if not os.path.exists(obj_dir): + os.makedirs(obj_dir) if self.bundlify: stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom) - with open(path, "w") as f: + with open(file_path, "w") as f: f.write(str(stix_obj)) def add(self, stix_data=None, version=None): @@ -104,25 +466,18 @@ class FileSystemSink(DataSink): the Bundle contained, but not the Bundle itself. """ - if any(x in ('STIXDomainObject', 'STIXRelationshipObject', 'MarkingDefinition') - for x in get_class_hierarchy_names(stix_data)): + if isinstance(stix_data, Bundle): + # recursively add individual STIX objects + for stix_obj in stix_data.get("objects", []): + self.add(stix_obj, version=version) + + elif isinstance(stix_data, _STIXBase): # adding python STIX object self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): stix_data = parse(stix_data, allow_custom=self.allow_custom, version=version) - if stix_data["type"] == "bundle": - # extract STIX objects - for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, version=version) - else: - # adding json-formatted STIX - self._check_path_and_write(stix_data,) - - elif isinstance(stix_data, Bundle): - # recursively add individual STIX objects - for stix_obj in stix_data.get("objects", []): - self.add(stix_obj, version=version) + self.add(stix_data, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects @@ -176,9 +531,7 @@ class FileSystemSource(DataSource): a python STIX object and then returned """ - query = [Filter("id", "=", stix_id)] - - all_data = self.query(query=query, version=version, _composite_filters=_composite_filters) + all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -206,9 +559,10 @@ class FileSystemSource(DataSource): a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, version=version, _composite_filters=_composite_filters)] + query = [Filter("id", "=", stix_id)] + return self.query(query, version=version, _composite_filters=_composite_filters) - def query(self, query=None, version=None, _composite_filters=None): + def query2(self, query=None, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -341,3 +695,57 @@ class FileSystemSource(DataSource): if filter_.property == "id" or filter_.property == "type": file_filters.append(filter_) return file_filters + + def query(self, query=None, version=None, _composite_filters=None): + """Search and retrieve STIX objects based on the complete query. + + A "complete query" includes the filters from the query, the filters + attached to this FileSystemSource, and any filters passed from a + CompositeDataSource (i.e. _composite_filters). + + Args: + query (list): list of filters to search on + _composite_filters (FilterSet): collection of filters passed from the + CompositeDataSource, not user supplied + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + (list): list of STIX objects that matches the supplied + query. The STIX objects are loaded from their json files, + parsed into a python STIX objects and then returned. + + """ + + all_data = [] + + query = FilterSet(query) + + # combine all query filters + if self.filters: + query.add(self.filters) + if _composite_filters: + query.add(_composite_filters) + + auth_types, auth_ids = _find_search_optimizations(query) + + type_dirs = _get_matching_dir_entries(self._stix_dir, auth_types, + stat.S_ISDIR) + for type_dir in type_dirs: + type_path = os.path.join(self._stix_dir, type_dir) + + if type_dir == "marking-definition": + type_results = _search_markings(query, type_path, auth_ids) + else: + type_results = _search_versioned(query, type_path, auth_ids) + + all_data.extend(type_results) + + # parse python STIX objects from the STIX object dicts + stix_objs = [ + parse(stix_obj_dict, allow_custom=self.allow_custom, + version=version) + for stix_obj_dict in all_data + ] + + return stix_objs diff --git a/stix2/datastore/memory.py b/stix2/datastore/memory.py index 4b82c34..3e065ad 100644 --- a/stix2/datastore/memory.py +++ b/stix2/datastore/memory.py @@ -10,6 +10,7 @@ from stix2.base import _STIXBase from stix2.core import Bundle, parse from stix2.datastore import DataSink, DataSource, DataStoreMixin from stix2.datastore.filters import FilterSet, apply_common_filters +from stix2.utils import is_marking def _add(store, stix_data=None, allow_custom=True, version=None): @@ -43,7 +44,7 @@ def _add(store, stix_data=None, allow_custom=True, version=None): # Map ID directly to the object, if it is a marking. Otherwise, # map to a family, so we can track multiple versions. - if _is_marking(stix_obj): + if is_marking(stix_obj): store._data[stix_obj.id] = stix_obj else: @@ -56,22 +57,6 @@ def _add(store, stix_data=None, allow_custom=True, version=None): obj_family.add(stix_obj) -def _is_marking(obj_or_id): - """Determines whether the given object or object ID is/is for a marking - definition. - - :param obj_or_id: A STIX object or object ID as a string. - :return: True if a marking definition, False otherwise. - """ - - if isinstance(obj_or_id, _STIXBase): - id_ = obj_or_id.id - else: - id_ = obj_or_id - - return id_.startswith("marking-definition--") - - class _ObjectFamily(object): """ An internal implementation detail of memory sources/sinks/stores. @@ -255,7 +240,7 @@ class MemorySource(DataSource): """ stix_obj = None - if _is_marking(stix_id): + if is_marking(stix_id): stix_obj = self._data.get(stix_id) else: object_family = self._data.get(stix_id) @@ -291,7 +276,7 @@ class MemorySource(DataSource): """ results = [] stix_objs_to_filter = None - if _is_marking(stix_id): + if is_marking(stix_id): stix_obj = self._data.get(stix_id) if stix_obj: stix_objs_to_filter = [stix_obj] diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json old mode 100755 new mode 100644 similarity index 82% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json index 47dd5f8..4da65a1 --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22/20170531213019735010.json @@ -1,42 +1,42 @@ { - "id": "bundle--f68640b4-0cdc-42ae-b176-def1754a1ea0", + "id": "bundle--f68640b4-0cdc-42ae-b176-def1754a1ea0", "objects": [ { - "created": "2017-05-31T21:30:19.73501Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Credential dumping is the process of obtaining account login and password information from the operating system and software. Credentials can be used to perform Windows Credential Editor, Mimikatz, and gsecdump. These tools are in use by both professional security testers and adversaries.\n\nPlaintext passwords can be obtained using tools such as Mimikatz to extract passwords stored by the Local Security Authority (LSA). If smart cards are used to authenticate to a domain using a personal identification number (PIN), then that PIN is also cached as a result and may be dumped.Mimikatz access the LSA Subsystem Service (LSASS) process by opening the process, locating the LSA secrets key, and decrypting the sections in memory where credential details are stored. Credential dumpers may also use methods for reflective DLL Injection to reduce potential indicators of malicious activity.\n\nNTLM hash dumpers open the Security Accounts Manager (SAM) on the local file system (%SystemRoot%/system32/config/SAM) or create a dump of the Registry SAM key to access stored account password hashes. Some hash dumpers will open the local file system as a device and parse to the SAM table to avoid file access defenses. Others will make an in-memory copy of the SAM table before reading hashes. Detection of compromised Legitimate Credentials in-use by adversaries may help as well. \n\nOn Windows 8.1 and Windows Server 2012 R2, monitor Windows Logs for LSASS.exe creation to verify that LSASS started as a protected process.\n\nMonitor processes and command-line arguments for program execution that may be indicative of credential dumping. Remote access tools may contain built-in features or incorporate existing tools like Mimikatz. PowerShell scripts also exist that contain credential dumping functionality, such as PowerSploit's Invoke-Mimikatz module,[[Citation: Powersploit]] which may require additional logging features to be configured in the operating system to collect necessary information for analysis.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: API monitoring, Process command-line parameters, Process monitoring, PowerShell logs", + "created": "2017-05-31T21:30:19.73501Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Credential dumping is the process of obtaining account login and password information from the operating system and software. Credentials can be used to perform Windows Credential Editor, Mimikatz, and gsecdump. These tools are in use by both professional security testers and adversaries.\n\nPlaintext passwords can be obtained using tools such as Mimikatz to extract passwords stored by the Local Security Authority (LSA). If smart cards are used to authenticate to a domain using a personal identification number (PIN), then that PIN is also cached as a result and may be dumped.Mimikatz access the LSA Subsystem Service (LSASS) process by opening the process, locating the LSA secrets key, and decrypting the sections in memory where credential details are stored. Credential dumpers may also use methods for reflective DLL Injection to reduce potential indicators of malicious activity.\n\nNTLM hash dumpers open the Security Accounts Manager (SAM) on the local file system (%SystemRoot%/system32/config/SAM) or create a dump of the Registry SAM key to access stored account password hashes. Some hash dumpers will open the local file system as a device and parse to the SAM table to avoid file access defenses. Others will make an in-memory copy of the SAM table before reading hashes. Detection of compromised Legitimate Credentials in-use by adversaries may help as well. \n\nOn Windows 8.1 and Windows Server 2012 R2, monitor Windows Logs for LSASS.exe creation to verify that LSASS started as a protected process.\n\nMonitor processes and command-line arguments for program execution that may be indicative of credential dumping. Remote access tools may contain built-in features or incorporate existing tools like Mimikatz. PowerShell scripts also exist that contain credential dumping functionality, such as PowerSploit's Invoke-Mimikatz module,[[Citation: Powersploit]] which may require additional logging features to be configured in the operating system to collect necessary information for analysis.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: API monitoring, Process command-line parameters, Process monitoring, PowerShell logs", "external_references": [ { - "external_id": "T1003", - "source_name": "mitre-attack", + "external_id": "T1003", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1003" - }, + }, { - "description": "Delpy, B. (2014, September 14). Mimikatz module ~ sekurlsa. Retrieved January 10, 2016.", - "source_name": "Github Mimikatz Module sekurlsa", + "description": "Delpy, B. (2014, September 14). Mimikatz module ~ sekurlsa. Retrieved January 10, 2016.", + "source_name": "Github Mimikatz Module sekurlsa", "url": "https://github.com/gentilkiwi/mimikatz/wiki/module-~-sekurlsa" - }, + }, { - "description": "PowerSploit. (n.d.). Retrieved December 4, 2014.", - "source_name": "Powersploit", + "description": "PowerSploit. (n.d.). Retrieved December 4, 2014.", + "source_name": "Powersploit", "url": "https://github.com/mattifestation/PowerSploit" } - ], - "id": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22", + ], + "id": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "credential-access" } - ], - "modified": "2017-05-31T21:30:19.73501Z", - "name": "Credential Dumping", + ], + "modified": "2017-05-31T21:30:19.73501Z", + "name": "Credential Dumping", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json old mode 100755 new mode 100644 similarity index 75% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json index 13f900f..ca50fc4 --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b/20170531213026496201.json @@ -1,37 +1,37 @@ { - "id": "bundle--b07d6fd6-7cc5-492d-a1eb-9ba956b329d5", + "id": "bundle--b07d6fd6-7cc5-492d-a1eb-9ba956b329d5", "objects": [ { - "created": "2017-05-31T21:30:26.496201Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Rootkits are programs that hide the existence of malware by intercepting and modifying operating system API calls that supply system information. Rootkits or rootkit enabling functionality may reside at the user or kernel level in the operating system or lower, to include a Hypervisor, Master Boot Record, or the Basic Input/Output System.[[Citation: Wikipedia Rootkit]]\n\nAdversaries may use rootkits to hide the presence of programs, files, network connections, services, drivers, and other system components.\n\nDetection: Some rootkit protections may be built into anti-virus or operating system software. There are dedicated rootkit detection tools that look for specific types of rootkit behavior. Monitor for the existence of unrecognized DLLs, devices, services, and changes to the MBR.[[Citation: Wikipedia Rootkit]]\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: BIOS, MBR, System calls", + "created": "2017-05-31T21:30:26.496201Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Rootkits are programs that hide the existence of malware by intercepting and modifying operating system API calls that supply system information. Rootkits or rootkit enabling functionality may reside at the user or kernel level in the operating system or lower, to include a Hypervisor, Master Boot Record, or the Basic Input/Output System.[[Citation: Wikipedia Rootkit]]\n\nAdversaries may use rootkits to hide the presence of programs, files, network connections, services, drivers, and other system components.\n\nDetection: Some rootkit protections may be built into anti-virus or operating system software. There are dedicated rootkit detection tools that look for specific types of rootkit behavior. Monitor for the existence of unrecognized DLLs, devices, services, and changes to the MBR.[[Citation: Wikipedia Rootkit]]\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: BIOS, MBR, System calls", "external_references": [ { - "external_id": "T1014", - "source_name": "mitre-attack", + "external_id": "T1014", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1014" - }, + }, { - "description": "Wikipedia. (2016, June 1). Rootkit. Retrieved June 2, 2016.", - "source_name": "Wikipedia Rootkit", + "description": "Wikipedia. (2016, June 1). Rootkit. Retrieved June 2, 2016.", + "source_name": "Wikipedia Rootkit", "url": "https://en.wikipedia.org/wiki/Rootkit" } - ], - "id": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b", + ], + "id": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "defense-evasion" } - ], - "modified": "2017-05-31T21:30:26.496201Z", - "name": "Rootkit", + ], + "modified": "2017-05-31T21:30:26.496201Z", + "name": "Rootkit", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json old mode 100755 new mode 100644 similarity index 71% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json index db57e2c..99a50cc --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9/20170531213029458940.json @@ -1,32 +1,32 @@ { - "id": "bundle--1a854c96-639e-4771-befb-e7b960a65974", + "id": "bundle--1a854c96-639e-4771-befb-e7b960a65974", "objects": [ { - "created": "2017-05-31T21:30:29.45894Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Data, such as sensitive documents, may be exfiltrated through the use of automated processing or Scripting after being gathered during Exfiltration Over Command and Control Channel and Exfiltration Over Alternative Protocol.\n\nDetection: Monitor process file access patterns and network behavior. Unrecognized processes or scripts that appear to be traversing file systems and sending network traffic may be suspicious.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process use of network", + "created": "2017-05-31T21:30:29.45894Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Data, such as sensitive documents, may be exfiltrated through the use of automated processing or Scripting after being gathered during Exfiltration Over Command and Control Channel and Exfiltration Over Alternative Protocol.\n\nDetection: Monitor process file access patterns and network behavior. Unrecognized processes or scripts that appear to be traversing file systems and sending network traffic may be suspicious.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process use of network", "external_references": [ { - "external_id": "T1020", - "source_name": "mitre-attack", + "external_id": "T1020", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1020" } - ], - "id": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9", + ], + "id": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "exfiltration" } - ], - "modified": "2017-05-31T21:30:29.45894Z", - "name": "Automated Exfiltration", + ], + "modified": "2017-05-31T21:30:29.45894Z", + "name": "Automated Exfiltration", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json old mode 100755 new mode 100644 similarity index 74% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json index d48092d..9b3179c --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475/20170531213045139269.json @@ -1,32 +1,32 @@ { - "id": "bundle--33e3e33a-38b8-4a37-9455-5b8c82d3b10a", + "id": "bundle--33e3e33a-38b8-4a37-9455-5b8c82d3b10a", "objects": [ { - "created": "2017-05-31T21:30:45.139269Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Adversaries may attempt to get a listing of network connections to or from the compromised system.\nUtilities and commands that acquire this information include netstat, \"net use,\" and \"net session\" with Net.\n\nDetection: System and network discovery techniques normally occur throughout an operation as an adversary learns the environment. Data and events should not be viewed in isolation, but as part of a chain of behavior that could lead to other activities, such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Process command-line parameters, Process monitoring", + "created": "2017-05-31T21:30:45.139269Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Adversaries may attempt to get a listing of network connections to or from the compromised system.\nUtilities and commands that acquire this information include netstat, \"net use,\" and \"net session\" with Net.\n\nDetection: System and network discovery techniques normally occur throughout an operation as an adversary learns the environment. Data and events should not be viewed in isolation, but as part of a chain of behavior that could lead to other activities, such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Process command-line parameters, Process monitoring", "external_references": [ { - "external_id": "T1049", - "source_name": "mitre-attack", + "external_id": "T1049", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1049" } - ], - "id": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475", + ], + "id": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "discovery" } - ], - "modified": "2017-05-31T21:30:45.139269Z", - "name": "Local Network Connections Discovery", + ], + "modified": "2017-05-31T21:30:45.139269Z", + "name": "Local Network Connections Discovery", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json old mode 100755 new mode 100644 similarity index 74% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json index 031419e..d80d781 --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c/20170531213041022897.json @@ -1,32 +1,32 @@ { - "id": "bundle--a87938c5-cc1e-4e06-a8a3-b10243ae397d", + "id": "bundle--a87938c5-cc1e-4e06-a8a3-b10243ae397d", "objects": [ { - "created": "2017-05-31T21:30:41.022897Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Sensitive data can be collected from remote systems via shared network drives (host shared directory, network file server, etc.) that are accessible from the current system prior to cmd may be used to gather information.\n\nDetection: Monitor processes and command-line arguments for actions that could be taken to collect files from a network share. Remote access tools with built-in features may interact directly with the Windows API to gather data. Data may also be acquired through Windows system management tools such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process command-line parameters", + "created": "2017-05-31T21:30:41.022897Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Sensitive data can be collected from remote systems via shared network drives (host shared directory, network file server, etc.) that are accessible from the current system prior to cmd may be used to gather information.\n\nDetection: Monitor processes and command-line arguments for actions that could be taken to collect files from a network share. Remote access tools with built-in features may interact directly with the Windows API to gather data. Data may also be acquired through Windows system management tools such as Windows Management Instrumentation and PowerShell.\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: File monitoring, Process monitoring, Process command-line parameters", "external_references": [ { - "external_id": "T1039", - "source_name": "mitre-attack", + "external_id": "T1039", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1039" } - ], - "id": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c", + ], + "id": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "collection" } - ], - "modified": "2017-05-31T21:30:41.022897Z", - "name": "Data from Network Shared Drive", + ], + "modified": "2017-05-31T21:30:41.022897Z", + "name": "Data from Network Shared Drive", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json b/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json old mode 100755 new mode 100644 similarity index 73% rename from stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json rename to stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json index 67c380c..0e81b3c --- a/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a.json +++ b/stix2/test/stix2_data/attack-pattern/attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a/20170531213032662702.json @@ -1,32 +1,32 @@ { - "id": "bundle--5ddaeff9-eca7-4094-9e65-4f53da21a444", + "id": "bundle--5ddaeff9-eca7-4094-9e65-4f53da21a444", "objects": [ { - "created": "2017-05-31T21:30:32.662702Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Adversaries may attempt to make an executable or file difficult to discover or analyze by encrypting, encoding, or otherwise obfuscating its contents on the system.\n\nDetection: Detection of file obfuscation is difficult unless artifacts are left behind by the obfuscation process that are uniquely detectable with a signature. If detection of the obfuscation itself is not possible, it may be possible to detect the malicious activity that caused the obfuscated file (for example, the method that was used to write, read, or modify the file on the file system).\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Network protocol analysis, Process use of network, Binary file metadata, File monitoring, Malware reverse engineering", + "created": "2017-05-31T21:30:32.662702Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Adversaries may attempt to make an executable or file difficult to discover or analyze by encrypting, encoding, or otherwise obfuscating its contents on the system.\n\nDetection: Detection of file obfuscation is difficult unless artifacts are left behind by the obfuscation process that are uniquely detectable with a signature. If detection of the obfuscation itself is not possible, it may be possible to detect the malicious activity that caused the obfuscated file (for example, the method that was used to write, read, or modify the file on the file system).\n\nPlatforms: Windows Server 2003, Windows Server 2008, Windows Server 2012, Windows XP, Windows 7, Windows 8, Windows Server 2003 R2, Windows Server 2008 R2, Windows Server 2012 R2, Windows Vista, Windows 8.1\n\nData Sources: Network protocol analysis, Process use of network, Binary file metadata, File monitoring, Malware reverse engineering", "external_references": [ { - "external_id": "T1027", - "source_name": "mitre-attack", + "external_id": "T1027", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Technique/T1027" } - ], - "id": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a", + ], + "id": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a", "kill_chain_phases": [ { - "kill_chain_name": "mitre-attack", + "kill_chain_name": "mitre-attack", "phase_name": "defense-evasion" } - ], - "modified": "2017-05-31T21:30:32.662702Z", - "name": "Obfuscated Files or Information", + ], + "modified": "2017-05-31T21:30:32.662702Z", + "name": "Obfuscated Files or Information", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "attack-pattern" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json b/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json old mode 100755 new mode 100644 similarity index 61% rename from stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json rename to stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json index bf14aa7..302d3f5 --- a/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f/20170531213026495974.json @@ -1,16 +1,16 @@ { - "id": "bundle--a42d26fe-c938-4074-a1b3-50d852e6f0bd", + "id": "bundle--a42d26fe-c938-4074-a1b3-50d852e6f0bd", "objects": [ { - "created": "2017-05-31T21:30:26.495974Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Identify potentially malicious software that may contain rootkit functionality, and audit and/or block it by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", - "id": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f", - "modified": "2017-05-31T21:30:26.495974Z", - "name": "Rootkit Mitigation", + "created": "2017-05-31T21:30:26.495974Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Identify potentially malicious software that may contain rootkit functionality, and audit and/or block it by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", + "id": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f", + "modified": "2017-05-31T21:30:26.495974Z", + "name": "Rootkit Mitigation", "type": "course-of-action" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json old mode 100755 new mode 100644 similarity index 72% rename from stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json rename to stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json index cb9cfe2..71be78d --- a/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd.json +++ b/stix2/test/stix2_data/course-of-action/course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd/20170531213041022744.json @@ -1,9 +1,9 @@ { - "created": "2017-05-31T21:30:41.022744Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", - "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", - "modified": "2017-05-31T21:30:41.022744Z", - "name": "Data from Network Shared Drive Mitigation", + "created": "2017-05-31T21:30:41.022744Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Identify unnecessary system utilities or potentially malicious software that may be used to collect data from a network share, and audit and/or block them by using whitelisting[[CiteRef::Beechey 2010]] tools, like AppLocker,[[CiteRef::Windows Commands JPCERT]][[CiteRef::NSA MS AppLocker]] or Software Restriction Policies[[CiteRef::Corio 2008]] where appropriate.[[CiteRef::TechNet Applocker vs SRP]]", + "id": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", + "modified": "2017-05-31T21:30:41.022744Z", + "name": "Data from Network Shared Drive Mitigation", "type": "course-of-action" } \ No newline at end of file diff --git a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json deleted file mode 100755 index 77d4464..0000000 --- a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "bundle--81884287-2548-47fc-a997-39489ddd5462", - "objects": [ - { - "created": "2017-06-01T00:00:00Z", - "id": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "identity_class": "organization", - "modified": "2017-06-01T00:00:00Z", - "name": "The MITRE Corporation", - "type": "identity" - } - ], - "spec_version": "2.0", - "type": "bundle" -} \ No newline at end of file diff --git a/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json new file mode 100644 index 0000000..25e9361 --- /dev/null +++ b/stix2/test/stix2_data/identity/identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5/20170601000000000000.json @@ -0,0 +1,15 @@ +{ + "id": "bundle--81884287-2548-47fc-a997-39489ddd5462", + "objects": [ + { + "created": "2017-06-01T00:00:00Z", + "id": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "identity_class": "organization", + "modified": "2017-06-01T00:00:00Z", + "name": "The MITRE Corporation", + "type": "identity" + } + ], + "spec_version": "2.0", + "type": "bundle" +} \ No newline at end of file diff --git a/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json b/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json old mode 100755 new mode 100644 similarity index 65% rename from stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json rename to stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json index 10ef3a5..72e435d --- a/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064.json +++ b/stix2/test/stix2_data/intrusion-set/intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064/20170531213149412497.json @@ -1,54 +1,54 @@ { - "id": "bundle--7790ee4c-2d57-419a-bc9d-8805b5bb4118", + "id": "bundle--7790ee4c-2d57-419a-bc9d-8805b5bb4118", "objects": [ { "aliases": [ - "Deep Panda", - "Shell Crew", - "WebMasters", - "KungFu Kittens", - "PinkPanther", + "Deep Panda", + "Shell Crew", + "WebMasters", + "KungFu Kittens", + "PinkPanther", "Black Vine" - ], - "created": "2017-05-31T21:31:49.412497Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Deep Panda is a suspected Chinese threat group known to target many industries, including government, defense, financial, and telecommunications.Deep Panda.Deep Panda also appears to be known as Black Vine based on the attribution of both group names to the Anthem intrusion.[[Citation: Symantec Black Vine]]", + ], + "created": "2017-05-31T21:31:49.412497Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Deep Panda is a suspected Chinese threat group known to target many industries, including government, defense, financial, and telecommunications.Deep Panda.Deep Panda also appears to be known as Black Vine based on the attribution of both group names to the Anthem intrusion.[[Citation: Symantec Black Vine]]", "external_references": [ { - "external_id": "G0009", - "source_name": "mitre-attack", + "external_id": "G0009", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Group/G0009" - }, + }, { - "description": "Alperovitch, D. (2014, July 7). Deep in Thought: Chinese Targeting of National Security Think Tanks. Retrieved November 12, 2014.", - "source_name": "Alperovitch 2014", + "description": "Alperovitch, D. (2014, July 7). Deep in Thought: Chinese Targeting of National Security Think Tanks. Retrieved November 12, 2014.", + "source_name": "Alperovitch 2014", "url": "http://blog.crowdstrike.com/deep-thought-chinese-targeting-national-security-think-tanks/" - }, + }, { - "description": "DiMaggio, J.. (2015, August 6). The Black Vine cyberespionage group. Retrieved January 26, 2016.", - "source_name": "Symantec Black Vine", + "description": "DiMaggio, J.. (2015, August 6). The Black Vine cyberespionage group. Retrieved January 26, 2016.", + "source_name": "Symantec Black Vine", "url": "http://www.symantec.com/content/en/us/enterprise/media/security%20response/whitepapers/the-black-vine-cyberespionage-group.pdf" - }, + }, { - "description": "RSA Incident Response. (2014, January). RSA Incident Response Emerging Threat Profile: Shell Crew. Retrieved January 14, 2016.", - "source_name": "RSA Shell Crew", + "description": "RSA Incident Response. (2014, January). RSA Incident Response Emerging Threat Profile: Shell Crew. Retrieved January 14, 2016.", + "source_name": "RSA Shell Crew", "url": "https://www.emc.com/collateral/white-papers/h12756-wp-shell-crew.pdf" - }, + }, { - "description": "ThreatConnect Research Team. (2015, February 27). The Anthem Hack: All Roads Lead to China. Retrieved January 26, 2016.", - "source_name": "ThreatConnect Anthem", + "description": "ThreatConnect Research Team. (2015, February 27). The Anthem Hack: All Roads Lead to China. Retrieved January 26, 2016.", + "source_name": "ThreatConnect Anthem", "url": "https://www.threatconnect.com/the-anthem-hack-all-roads-lead-to-china/" } - ], - "id": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064", - "modified": "2017-05-31T21:31:49.412497Z", - "name": "Deep Panda", + ], + "id": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064", + "modified": "2017-05-31T21:31:49.412497Z", + "name": "Deep Panda", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "intrusion-set" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json b/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json old mode 100755 new mode 100644 similarity index 74% rename from stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json rename to stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json index 84b75b1..541072e --- a/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a.json +++ b/stix2/test/stix2_data/intrusion-set/intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a/20170531213153197755.json @@ -1,44 +1,44 @@ { - "id": "bundle--96a6ea7a-fcff-4aab-925b-a494bcdf0480", + "id": "bundle--96a6ea7a-fcff-4aab-925b-a494bcdf0480", "objects": [ { "aliases": [ "DragonOK" - ], - "created": "2017-05-31T21:31:53.197755Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]", + ], + "created": "2017-05-31T21:31:53.197755Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "DragonOK is a threat group that has targeted Japanese organizations with phishing emails. Due to overlapping TTPs, including similar custom tools, DragonOK is thought to have a direct or indirect relationship with the threat group Moafee. [[Citation: Operation Quantum Entanglement]][[Citation: Symbiotic APT Groups]] It is known to use a variety of malware, including Sysget/HelloBridge, PlugX, PoisonIvy, FormerFirstRat, NFlog, and NewCT. [[Citation: New DragonOK]]", "external_references": [ { - "external_id": "G0017", - "source_name": "mitre-attack", + "external_id": "G0017", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Group/G0017" - }, + }, { - "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.", - "source_name": "Operation Quantum Entanglement", + "description": "Haq, T., Moran, N., Vashisht, S., Scott, M. (2014, September). OPERATION QUANTUM ENTANGLEMENT. Retrieved November 4, 2015.", + "source_name": "Operation Quantum Entanglement", "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/wp-operation-quantum-entanglement.pdf" - }, + }, { - "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.", - "source_name": "Symbiotic APT Groups", + "description": "Haq, T. (2014, October). An Insight into Symbiotic APT Groups. Retrieved November 4, 2015.", + "source_name": "Symbiotic APT Groups", "url": "https://dl.mandiant.com/EE/library/MIRcon2014/MIRcon%202014%20R&D%20Track%20Insight%20into%20Symbiotic%20APT.pdf" - }, + }, { - "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.", - "source_name": "New DragonOK", + "description": "Miller-Osborn, J., Grunzweig, J.. (2015, April). Unit 42 Identifies New DragonOK Backdoor Malware Deployed Against Japanese Targets. Retrieved November 4, 2015.", + "source_name": "New DragonOK", "url": "http://researchcenter.paloaltonetworks.com/2015/04/unit-42-identifies-new-dragonok-backdoor-malware-deployed-against-japanese-targets/" } - ], - "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a", - "modified": "2017-05-31T21:31:53.197755Z", - "name": "DragonOK", + ], + "id": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a", + "modified": "2017-05-31T21:31:53.197755Z", + "name": "DragonOK", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "intrusion-set" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json old mode 100755 new mode 100644 similarity index 64% rename from stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json rename to stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json index 669f00c..8ce4c86 --- a/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38.json +++ b/stix2/test/stix2_data/malware/malware--6b616fc1-1505-48e3-8b2c-0d19337bff38/20170531213258226477.json @@ -1,34 +1,34 @@ { - "id": "bundle--f64de948-7067-4534-8018-85f03d470625", + "id": "bundle--f64de948-7067-4534-8018-85f03d470625", "objects": [ { - "created": "2017-05-31T21:32:58.226477Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", + "created": "2017-05-31T21:32:58.226477Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Rover is malware suspected of being used for espionage purposes. It was used in 2015 in a targeted email sent to an Indian Ambassador to Afghanistan.[[Citation: Palo Alto Rover]]", "external_references": [ { - "external_id": "S0090", - "source_name": "mitre-attack", + "external_id": "S0090", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0090" - }, + }, { - "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", - "source_name": "Palo Alto Rover", + "description": "Ray, V., Hayashi, K. (2016, February 29). New Malware \u2018Rover\u2019 Targets Indian Ambassador to Afghanistan. Retrieved February 29, 2016.", + "source_name": "Palo Alto Rover", "url": "http://researchcenter.paloaltonetworks.com/2016/02/new-malware-rover-targets-indian-ambassador-to-afghanistan/" } - ], - "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + ], + "id": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", "labels": [ "malware" - ], - "modified": "2017-05-31T21:32:58.226477Z", - "name": "Rover", + ], + "modified": "2017-05-31T21:32:58.226477Z", + "name": "Rover", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "malware" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json b/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json old mode 100755 new mode 100644 similarity index 63% rename from stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json rename to stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json index 9eaf8ad..fda1b6a --- a/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841.json +++ b/stix2/test/stix2_data/malware/malware--92ec0cbd-2c30-44a2-b270-73f4ec949841/20170531213326565056.json @@ -1,34 +1,34 @@ { - "id": "bundle--c633942b-545c-4c87-91b7-9fe5740365e0", + "id": "bundle--c633942b-545c-4c87-91b7-9fe5740365e0", "objects": [ { - "created": "2017-05-31T21:33:26.565056Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "RTM is custom malware written in Delphi. It is used by the group of the same name (RTM).[[Citation: ESET RTM Feb 2017]]", + "created": "2017-05-31T21:33:26.565056Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "RTM is custom malware written in Delphi. It is used by the group of the same name (RTM).[[Citation: ESET RTM Feb 2017]]", "external_references": [ { - "external_id": "S0148", - "source_name": "mitre-attack", + "external_id": "S0148", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0148" - }, + }, { - "description": "Faou, M. and Boutin, J.. (2017, February). Read The Manual: A Guide to the RTM Banking Trojan. Retrieved March 9, 2017.", - "source_name": "ESET RTM Feb 2017", + "description": "Faou, M. and Boutin, J.. (2017, February). Read The Manual: A Guide to the RTM Banking Trojan. Retrieved March 9, 2017.", + "source_name": "ESET RTM Feb 2017", "url": "https://www.welivesecurity.com/wp-content/uploads/2017/02/Read-The-Manual.pdf" } - ], - "id": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", + ], + "id": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", "labels": [ "malware" - ], - "modified": "2017-05-31T21:33:26.565056Z", - "name": "RTM", + ], + "modified": "2017-05-31T21:33:26.565056Z", + "name": "RTM", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "malware" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json b/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json old mode 100755 new mode 100644 similarity index 65% rename from stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json rename to stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json index 224f6a9..a60341e --- a/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e.json +++ b/stix2/test/stix2_data/malware/malware--96b08451-b27a-4ff6-893f-790e26393a8e/20170531213248482655.json @@ -1,34 +1,34 @@ { - "id": "bundle--09ce4338-8741-4fcf-9738-d216c8e40974", + "id": "bundle--09ce4338-8741-4fcf-9738-d216c8e40974", "objects": [ { - "created": "2017-05-31T21:32:48.482655Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Sakula is a remote access tool (RAT) that first surfaced in 2012 and was used in intrusions throughout 2015.[[Citation: Dell Sakula]]\n\nAliases: Sakula, Sakurel, VIPER", + "created": "2017-05-31T21:32:48.482655Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Sakula is a remote access tool (RAT) that first surfaced in 2012 and was used in intrusions throughout 2015.[[Citation: Dell Sakula]]\n\nAliases: Sakula, Sakurel, VIPER", "external_references": [ { - "external_id": "S0074", - "source_name": "mitre-attack", + "external_id": "S0074", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0074" - }, + }, { - "description": "Dell SecureWorks Counter Threat Unit Threat Intelligence. (2015, July 30). Sakula Malware Family. Retrieved January 26, 2016.", - "source_name": "Dell Sakula", + "description": "Dell SecureWorks Counter Threat Unit Threat Intelligence. (2015, July 30). Sakula Malware Family. Retrieved January 26, 2016.", + "source_name": "Dell Sakula", "url": "http://www.secureworks.com/cyber-threat-intelligence/threats/sakula-malware-family/" } - ], - "id": "malware--96b08451-b27a-4ff6-893f-790e26393a8e", + ], + "id": "malware--96b08451-b27a-4ff6-893f-790e26393a8e", "labels": [ "malware" - ], - "modified": "2017-05-31T21:32:48.482655Z", - "name": "Sakula", + ], + "modified": "2017-05-31T21:32:48.482655Z", + "name": "Sakula", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "malware" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json b/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json old mode 100755 new mode 100644 similarity index 62% rename from stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json rename to stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json index 3e1c870..d68624e --- a/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b.json +++ b/stix2/test/stix2_data/malware/malware--b42378e0-f147-496f-992a-26a49705395b/20170531213215263882.json @@ -1,34 +1,34 @@ { - "id": "bundle--611947ce-ae3b-4fdb-b297-aed8eab22e4f", + "id": "bundle--611947ce-ae3b-4fdb-b297-aed8eab22e4f", "objects": [ { - "created": "2017-05-31T21:32:15.263882Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "PoisonIvy is a popular remote access tool (RAT) that has been used by many groups.[[Citation: FireEye Poison Ivy]]\n\nAliases: PoisonIvy, Poison Ivy", + "created": "2017-05-31T21:32:15.263882Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "PoisonIvy is a popular remote access tool (RAT) that has been used by many groups.[[Citation: FireEye Poison Ivy]]\n\nAliases: PoisonIvy, Poison Ivy", "external_references": [ { - "external_id": "S0012", - "source_name": "mitre-attack", + "external_id": "S0012", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0012" - }, + }, { - "description": "FireEye. (2014). POISON IVY: Assessing Damage and Extracting Intelligence. Retrieved November 12, 2014.", - "source_name": "FireEye Poison Ivy", + "description": "FireEye. (2014). POISON IVY: Assessing Damage and Extracting Intelligence. Retrieved November 12, 2014.", + "source_name": "FireEye Poison Ivy", "url": "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-poison-ivy.pdf" } - ], - "id": "malware--b42378e0-f147-496f-992a-26a49705395b", + ], + "id": "malware--b42378e0-f147-496f-992a-26a49705395b", "labels": [ "malware" - ], - "modified": "2017-05-31T21:32:15.263882Z", - "name": "PoisonIvy", + ], + "modified": "2017-05-31T21:32:15.263882Z", + "name": "PoisonIvy", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "malware" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json b/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json index bcae183..93f94a7 100755 --- a/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json +++ b/stix2/test/stix2_data/marking-definition/marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168.json @@ -1,16 +1,16 @@ { - "id": "bundle--0f4a3025-7788-4f25-a0c7-26171056dfae", + "id": "bundle--0f4a3025-7788-4f25-a0c7-26171056dfae", "objects": [ { - "created": "2017-06-01T00:00:00Z", + "created": "2017-06-01T00:00:00Z", "definition": { "statement": "Copyright 2017, The MITRE Corporation" - }, - "definition_type": "statement", - "id": "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168", + }, + "definition_type": "statement", + "id": "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168", "type": "marking-definition" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json b/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json old mode 100755 new mode 100644 similarity index 58% rename from stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json rename to stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json index ac59925..87d9a7c --- a/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883.json +++ b/stix2/test/stix2_data/relationship/relationship--0d4a7788-7f3b-4df8-a498-31a38003c883/20170531213327182784.json @@ -1,20 +1,20 @@ { - "id": "bundle--7e715462-dd9d-40b9-968a-10ef0ecf126d", + "id": "bundle--7e715462-dd9d-40b9-968a-10ef0ecf126d", "objects": [ { - "created": "2017-05-31T21:33:27.182784Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--0d4a7788-7f3b-4df8-a498-31a38003c883", - "modified": "2017-05-31T21:33:27.182784Z", + "created": "2017-05-31T21:33:27.182784Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--0d4a7788-7f3b-4df8-a498-31a38003c883", + "modified": "2017-05-31T21:33:27.182784Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a", - "target_ref": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", + ], + "relationship_type": "uses", + "source_ref": "attack-pattern--b3d682b6-98f2-4fb0-aa3b-b4df007ca70a", + "target_ref": "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json b/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json old mode 100755 new mode 100644 similarity index 59% rename from stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json rename to stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json index ee97edf..231b57f --- a/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227.json +++ b/stix2/test/stix2_data/relationship/relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227/20170531213327082801.json @@ -1,20 +1,20 @@ { - "id": "bundle--a53eef35-abfc-4bcd-b84e-a048f7b4a9bf", + "id": "bundle--a53eef35-abfc-4bcd-b84e-a048f7b4a9bf", "objects": [ { - "created": "2017-05-31T21:33:27.082801Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227", - "modified": "2017-05-31T21:33:27.082801Z", + "created": "2017-05-31T21:33:27.082801Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--0e55ee98-0c6d-43d4-b424-b18a0036b227", + "modified": "2017-05-31T21:33:27.082801Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22", - "target_ref": "tool--242f3da3-4425-4d11-8f5c-b842886da966", + ], + "relationship_type": "uses", + "source_ref": "attack-pattern--0a3ead4e-6d47-4ccb-854c-a6a4f9d96b22", + "target_ref": "tool--242f3da3-4425-4d11-8f5c-b842886da966", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json b/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json old mode 100755 new mode 100644 similarity index 57% rename from stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json rename to stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json index ff0d8cc..00af1c7 --- a/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432.json +++ b/stix2/test/stix2_data/relationship/relationship--1e91cd45-a725-4965-abe3-700694374432/20170531213327018782.json @@ -1,20 +1,20 @@ { - "id": "bundle--0b9f6412-314f-44e3-8779-9738c9578ef5", + "id": "bundle--0b9f6412-314f-44e3-8779-9738c9578ef5", "objects": [ { - "created": "2017-05-31T21:33:27.018782Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--1e91cd45-a725-4965-abe3-700694374432", - "modified": "2017-05-31T21:33:27.018782Z", + "created": "2017-05-31T21:33:27.018782Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--1e91cd45-a725-4965-abe3-700694374432", + "modified": "2017-05-31T21:33:27.018782Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "mitigates", - "source_ref": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f", - "target_ref": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b", + ], + "relationship_type": "mitigates", + "source_ref": "course-of-action--95ddb356-7ba0-4bd9-a889-247262b8946f", + "target_ref": "attack-pattern--0f20e3cb-245b-4a61-8a91-2d93f7cb0e9b", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json b/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json old mode 100755 new mode 100644 similarity index 59% rename from stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json rename to stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json index 36d1482..a2ad396 --- a/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e.json +++ b/stix2/test/stix2_data/relationship/relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e/20170531213327100701.json @@ -1,20 +1,20 @@ { - "id": "bundle--6d5b04a8-efb2-4179-990e-74f1dcc76e0c", + "id": "bundle--6d5b04a8-efb2-4179-990e-74f1dcc76e0c", "objects": [ { - "created": "2017-05-31T21:33:27.100701Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e", - "modified": "2017-05-31T21:33:27.100701Z", + "created": "2017-05-31T21:33:27.100701Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--3a3084f9-0302-4fd5-9b8a-e0db10f5345e", + "modified": "2017-05-31T21:33:27.100701Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475", - "target_ref": "tool--03342581-f790-4f03-ba41-e82e67392e23", + ], + "relationship_type": "uses", + "source_ref": "attack-pattern--7e150503-88e7-4861-866b-ff1ac82c4475", + "target_ref": "tool--03342581-f790-4f03-ba41-e82e67392e23", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json b/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json old mode 100755 new mode 100644 similarity index 58% rename from stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json rename to stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json index 888cc3b..453dfef --- a/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1.json +++ b/stix2/test/stix2_data/relationship/relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1/20170531213327143973.json @@ -1,20 +1,20 @@ { - "id": "bundle--a7efc025-040d-49c7-bf97-e5a1120ecacc", + "id": "bundle--a7efc025-040d-49c7-bf97-e5a1120ecacc", "objects": [ { - "created": "2017-05-31T21:33:27.143973Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1", - "modified": "2017-05-31T21:33:27.143973Z", + "created": "2017-05-31T21:33:27.143973Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--3a3ed0b2-0c38-441f-ac40-53b873e545d1", + "modified": "2017-05-31T21:33:27.143973Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9", - "target_ref": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + ], + "relationship_type": "uses", + "source_ref": "attack-pattern--774a3188-6ba9-4dc4-879d-d54ee48a5ce9", + "target_ref": "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json b/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json old mode 100755 new mode 100644 similarity index 57% rename from stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json rename to stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json index d9078d1..fd22d63 --- a/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719.json +++ b/stix2/test/stix2_data/relationship/relationship--592d0c31-e61f-495e-a60e-70d7be59a719/20170531213327021562.json @@ -1,20 +1,20 @@ { - "id": "bundle--9f013d47-7704-41c2-9749-23d0d94af94d", + "id": "bundle--9f013d47-7704-41c2-9749-23d0d94af94d", "objects": [ { - "created": "2017-05-31T21:33:27.021562Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--592d0c31-e61f-495e-a60e-70d7be59a719", - "modified": "2017-05-31T21:33:27.021562Z", + "created": "2017-05-31T21:33:27.021562Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--592d0c31-e61f-495e-a60e-70d7be59a719", + "modified": "2017-05-31T21:33:27.021562Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "mitigates", - "source_ref": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", - "target_ref": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c", + ], + "relationship_type": "mitigates", + "source_ref": "course-of-action--d9727aee-48b8-4fdb-89e2-4c49746ba4dd", + "target_ref": "attack-pattern--ae676644-d2d2-41b7-af7e-9bed1b55898c", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json b/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json old mode 100755 new mode 100644 similarity index 59% rename from stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json rename to stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json index ef1c4b2..40a50db --- a/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1.json +++ b/stix2/test/stix2_data/relationship/relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1/20170531213327044387.json @@ -1,20 +1,20 @@ { - "id": "bundle--15167b24-4cee-4c96-a140-32a6c37df4b4", + "id": "bundle--15167b24-4cee-4c96-a140-32a6c37df4b4", "objects": [ { - "created": "2017-05-31T21:33:27.044387Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1", - "modified": "2017-05-31T21:33:27.044387Z", + "created": "2017-05-31T21:33:27.044387Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--70dc6b5c-c524-429e-a6ab-0dd40f0482c1", + "modified": "2017-05-31T21:33:27.044387Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064", - "target_ref": "malware--96b08451-b27a-4ff6-893f-790e26393a8e", + ], + "relationship_type": "uses", + "source_ref": "intrusion-set--a653431d-6a5e-4600-8ad3-609b5af57064", + "target_ref": "malware--96b08451-b27a-4ff6-893f-790e26393a8e", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json b/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json old mode 100755 new mode 100644 similarity index 59% rename from stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json rename to stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json index 1f20179..edfc864 --- a/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d.json +++ b/stix2/test/stix2_data/relationship/relationship--8797579b-e3be-4209-a71b-255a4d08243d/20170531213327051532.json @@ -1,20 +1,20 @@ { - "id": "bundle--ff845dca-7036-416f-aae0-95030994c49f", + "id": "bundle--ff845dca-7036-416f-aae0-95030994c49f", "objects": [ { - "created": "2017-05-31T21:33:27.051532Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "id": "relationship--8797579b-e3be-4209-a71b-255a4d08243d", - "modified": "2017-05-31T21:33:27.051532Z", + "created": "2017-05-31T21:33:27.051532Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "id": "relationship--8797579b-e3be-4209-a71b-255a4d08243d", + "modified": "2017-05-31T21:33:27.051532Z", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], - "relationship_type": "uses", - "source_ref": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a", - "target_ref": "malware--b42378e0-f147-496f-992a-26a49705395b", + ], + "relationship_type": "uses", + "source_ref": "intrusion-set--f3bdec95-3d62-42d9-a840-29630f6cdc1a", + "target_ref": "malware--b42378e0-f147-496f-992a-26a49705395b", "type": "relationship" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" } \ No newline at end of file diff --git a/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json b/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json old mode 100755 new mode 100644 similarity index 64% rename from stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json rename to stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json index 9d47880..713e4d0 --- a/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23.json +++ b/stix2/test/stix2_data/tool/tool--03342581-f790-4f03-ba41-e82e67392e23/20170531213231601148.json @@ -1,39 +1,39 @@ { - "id": "bundle--d8826afc-1561-4362-a4e3-05a4c2c3ac3c", + "id": "bundle--d8826afc-1561-4362-a4e3-05a4c2c3ac3c", "objects": [ { - "created": "2017-05-31T21:32:31.601148Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "The Net utility is a component of the Windows operating system. It is used in command-line operations for control of users, groups, services, and network connections.Net has a great deal of functionality,[[Citation: Savill 1999]] much of which is useful for an adversary, such as gathering system and network information for [[Discovery]], moving laterally through [[Windows admin shares]] using net use commands, and interacting with services.\n\nAliases: Net, net.exe", + "created": "2017-05-31T21:32:31.601148Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "The Net utility is a component of the Windows operating system. It is used in command-line operations for control of users, groups, services, and network connections.Net has a great deal of functionality,[[Citation: Savill 1999]] much of which is useful for an adversary, such as gathering system and network information for [[Discovery]], moving laterally through [[Windows admin shares]] using net use commands, and interacting with services.\n\nAliases: Net, net.exe", "external_references": [ { - "external_id": "S0039", - "source_name": "mitre-attack", + "external_id": "S0039", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0039" - }, + }, { - "description": "Microsoft. (2006, October 18). Net.exe Utility. Retrieved September 22, 2015.", - "source_name": "Microsoft Net Utility", + "description": "Microsoft. (2006, October 18). Net.exe Utility. Retrieved September 22, 2015.", + "source_name": "Microsoft Net Utility", "url": "https://msdn.microsoft.com/en-us/library/aa939914" - }, + }, { - "description": "Savill, J. (1999, March 4). Net.exe reference. Retrieved September 22, 2015.", - "source_name": "Savill 1999", + "description": "Savill, J. (1999, March 4). Net.exe reference. Retrieved September 22, 2015.", + "source_name": "Savill 1999", "url": "http://windowsitpro.com/windows/netexe-reference" } - ], - "id": "tool--03342581-f790-4f03-ba41-e82e67392e23", + ], + "id": "tool--03342581-f790-4f03-ba41-e82e67392e23", "labels": [ "tool" - ], - "modified": "2017-05-31T21:32:31.601148Z", - "name": "Net", + ], + "modified": "2017-05-31T21:32:31.601148Z", + "name": "Net", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "tool" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json b/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json old mode 100755 new mode 100644 similarity index 58% rename from stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json rename to stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json index 281888e..a82da42 --- a/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966.json +++ b/stix2/test/stix2_data/tool/tool--242f3da3-4425-4d11-8f5c-b842886da966/20170531213212684914.json @@ -1,34 +1,34 @@ { - "id": "bundle--7dbde18f-6f14-4bf0-8389-505c89d6d5a6", + "id": "bundle--7dbde18f-6f14-4bf0-8389-505c89d6d5a6", "objects": [ { - "created": "2017-05-31T21:32:12.684914Z", - "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", - "description": "Windows Credential Editor is a password dumping tool.[[Citation: Amplia WCE]]\n\nAliases: Windows Credential Editor, WCE", + "created": "2017-05-31T21:32:12.684914Z", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "description": "Windows Credential Editor is a password dumping tool.[[Citation: Amplia WCE]]\n\nAliases: Windows Credential Editor, WCE", "external_references": [ { - "external_id": "S0005", - "source_name": "mitre-attack", + "external_id": "S0005", + "source_name": "mitre-attack", "url": "https://attack.mitre.org/wiki/Software/S0005" - }, + }, { - "description": "Amplia Security. (n.d.). Windows Credentials Editor (WCE) F.A.Q.. Retrieved December 17, 2015.", - "source_name": "Amplia WCE", + "description": "Amplia Security. (n.d.). Windows Credentials Editor (WCE) F.A.Q.. Retrieved December 17, 2015.", + "source_name": "Amplia WCE", "url": "http://www.ampliasecurity.com/research/wcefaq.html" } - ], - "id": "tool--242f3da3-4425-4d11-8f5c-b842886da966", + ], + "id": "tool--242f3da3-4425-4d11-8f5c-b842886da966", "labels": [ "tool" - ], - "modified": "2017-05-31T21:32:12.684914Z", - "name": "Windows Credential Editor", + ], + "modified": "2017-05-31T21:32:12.684914Z", + "name": "Windows Credential Editor", "object_marking_refs": [ "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" - ], + ], "type": "tool" } - ], - "spec_version": "2.0", + ], + "spec_version": "2.0", "type": "bundle" -} +} \ No newline at end of file diff --git a/stix2/test/test_datastore_filesystem.py b/stix2/test/test_datastore_filesystem.py index 17661d0..bab11b3 100644 --- a/stix2/test/test_datastore_filesystem.py +++ b/stix2/test/test_datastore_filesystem.py @@ -1,12 +1,17 @@ +import errno import json import os import shutil +import stat import pytest from stix2 import (Bundle, Campaign, CustomObject, FileSystemSink, FileSystemSource, FileSystemStore, Filter, Identity, - Indicator, Malware, Relationship, properties) + Indicator, Malware, Relationship, parse, properties) +from stix2.datastore.filesystem import (AuthSet, _find_search_optimizations, + _get_matching_dir_entries, + _timestamp2filename) from stix2.test.constants import (CAMPAIGN_ID, CAMPAIGN_KWARGS, IDENTITY_ID, IDENTITY_KWARGS, INDICATOR_ID, INDICATOR_KWARGS, MALWARE_ID, MALWARE_KWARGS, @@ -97,7 +102,20 @@ def rel_fs_store(): yield fs for o in stix_objs: - os.remove(os.path.join(FS_PATH, o.type, o.id + '.json')) + filepath = os.path.join(FS_PATH, o.type, o.id, + _timestamp2filename(o.modified) + '.json') + + # Some test-scoped fixtures (e.g. fs_store) delete all campaigns, so by + # the time this module-scoped fixture tears itself down, it may find + # its campaigns already gone, which causes not-found errors. + try: + os.remove(filepath) + except OSError as e: + # 3 is the ERROR_PATH_NOT_FOUND windows error code. Which has an + # errno symbolic value, but not the windows meaning... + if e.errno in (errno.ENOENT, 3): + continue + raise e def test_filesystem_source_nonexistent_folder(): @@ -182,14 +200,16 @@ def test_filesystem_sink_add_python_stix_object(fs_sink, fs_source): fs_sink.add(camp1) - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) + filepath = os.path.join(FS_PATH, "campaign", camp1.id, + _timestamp2filename(camp1.modified) + ".json") + assert os.path.exists(filepath) camp1_r = fs_source.get(camp1.id) assert camp1_r.id == camp1.id assert camp1_r.name == "Hannibal" assert "War Elephant" in camp1_r.aliases - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + os.remove(filepath) def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): @@ -200,19 +220,30 @@ def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source): "objective": "German and French Intelligence Services", "aliases": ["Purple Robes"], "id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f", - "created": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z", + "modified": "2017-05-31T21:31:53.197755Z" } fs_sink.add(camp2) - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp2["id"] + ".json")) + # Need to get the exact "modified" timestamp which would have been + # in effect at the time the object was saved to the sink, which determines + # the filename it would have been saved as. It may not be exactly the same + # as what's in the dict, since the parsing process can enforce a precision + # constraint (e.g. truncate to milliseconds), which results in a slightly + # different name. + camp2obj = parse(camp2) + filepath = os.path.join(FS_PATH, "campaign", camp2obj["id"], + _timestamp2filename(camp2obj["modified"]) + ".json") + + assert os.path.exists(filepath) camp2_r = fs_source.get(camp2["id"]) assert camp2_r.id == camp2["id"] assert camp2_r.name == camp2["name"] assert "Purple Robes" in camp2_r.aliases - os.remove(os.path.join(FS_PATH, "campaign", camp2_r.id + ".json")) + os.remove(filepath) def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): @@ -228,53 +259,74 @@ def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source): "objective": "Bulgarian, Albanian and Romanian Intelligence Services", "aliases": ["Huns"], "id": "campaign--b8f86161-ccae-49de-973a-4ca320c62478", - "created": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z", + "modified": "2017-05-31T21:31:53.197755Z" } ] } fs_sink.add(bund) - assert os.path.exists(os.path.join(FS_PATH, "campaign", bund["objects"][0]["id"] + ".json")) + camp_obj = parse(bund["objects"][0]) + filepath = os.path.join(FS_PATH, "campaign", camp_obj["id"], + _timestamp2filename(camp_obj["modified"]) + ".json") + + assert os.path.exists(filepath) camp3_r = fs_source.get(bund["objects"][0]["id"]) assert camp3_r.id == bund["objects"][0]["id"] assert camp3_r.name == bund["objects"][0]["name"] assert "Huns" in camp3_r.aliases - os.remove(os.path.join(FS_PATH, "campaign", camp3_r.id + ".json")) + os.remove(filepath) def test_filesystem_sink_add_json_stix_object(fs_sink, fs_source): # add json-encoded stix obj camp4 = '{"type": "campaign", "id":"campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d",'\ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' + ' "created":"2017-05-31T21:31:53.197755Z",'\ + ' "modified":"2017-05-31T21:31:53.197755Z",'\ + ' "name": "Ghengis Khan", "objective": "China and Russian infrastructure"}' fs_sink.add(camp4) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d" + ".json")) + camp4obj = parse(camp4) + filepath = os.path.join(FS_PATH, "campaign", + "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d", + _timestamp2filename(camp4obj["modified"]) + ".json") + + assert os.path.exists(filepath) camp4_r = fs_source.get("campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d") assert camp4_r.id == "campaign--6a6ca372-ba07-42cc-81ef-9840fc1f963d" assert camp4_r.name == "Ghengis Khan" - os.remove(os.path.join(FS_PATH, "campaign", camp4_r.id + ".json")) + os.remove(filepath) def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source): # add json-encoded stix bundle bund2 = '{"type": "bundle", "id": "bundle--3d267103-8475-4d8f-b321-35ec6eccfa37",' \ ' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",' \ - ' "created":"2017-05-31T21:31:53.197755Z", "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' + ' "created":"2017-05-31T21:31:53.197755Z",'\ + ' "modified":"2017-05-31T21:31:53.197755Z",'\ + ' "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}' fs_sink.add(bund2) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b" + ".json")) + bund2obj = parse(bund2) + camp_obj = bund2obj["objects"][0] + + filepath = os.path.join(FS_PATH, "campaign", + "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b", + _timestamp2filename(camp_obj["modified"]) + ".json") + + assert os.path.exists(filepath) camp5_r = fs_source.get("campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b") assert camp5_r.id == "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b" assert camp5_r.name == "Spartacus" - os.remove(os.path.join(FS_PATH, "campaign", camp5_r.id + ".json")) + os.remove(filepath) def test_filesystem_sink_add_objects_list(fs_sink, fs_source): @@ -289,13 +341,23 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source): "objective": "Central and Eastern Europe military commands and departments", "aliases": ["The Frenchmen"], "id": "campaign--122818b6-1112-4fb0-b11b-b111107ca70a", - "created": "2017-05-31T21:31:53.197755Z" + "created": "2017-05-31T21:31:53.197755Z", + "modified": "2017-05-31T21:31:53.197755Z" } fs_sink.add([camp6, camp7]) - assert os.path.exists(os.path.join(FS_PATH, "campaign", camp6.id + ".json")) - assert os.path.exists(os.path.join(FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a" + ".json")) + camp7obj = parse(camp7) + + camp6filepath = os.path.join(FS_PATH, "campaign", camp6.id, + _timestamp2filename(camp6["modified"]) + + ".json") + camp7filepath = os.path.join( + FS_PATH, "campaign", "campaign--122818b6-1112-4fb0-b11b-b111107ca70a", + _timestamp2filename(camp7obj["modified"]) + ".json") + + assert os.path.exists(camp6filepath) + assert os.path.exists(camp7filepath) camp6_r = fs_source.get(camp6.id) assert camp6_r.id == camp6.id @@ -306,8 +368,8 @@ def test_filesystem_sink_add_objects_list(fs_sink, fs_source): assert "The Frenchmen" in camp7_r.aliases # remove all added objects - os.remove(os.path.join(FS_PATH, "campaign", camp6_r.id + ".json")) - os.remove(os.path.join(FS_PATH, "campaign", camp7_r.id + ".json")) + os.remove(camp6filepath) + os.remove(camp7filepath) def test_filesystem_store_get_stored_as_bundle(fs_store): @@ -375,8 +437,11 @@ def test_filesystem_store_add(fs_store): assert camp1_r.id == camp1.id assert camp1_r.name == camp1.name + filepath = os.path.join(FS_PATH, "campaign", camp1_r.id, + _timestamp2filename(camp1_r.modified) + ".json") + # remove - os.remove(os.path.join(FS_PATH, "campaign", camp1_r.id + ".json")) + os.remove(filepath) def test_filesystem_store_add_as_bundle(): @@ -387,7 +452,10 @@ def test_filesystem_store_add_as_bundle(): aliases=["Ragnar"]) fs_store.add(camp1) - with open(os.path.join(FS_PATH, "campaign", camp1.id + ".json")) as bundle_file: + filepath = os.path.join(FS_PATH, "campaign", camp1.id, + _timestamp2filename(camp1.modified) + ".json") + + with open(filepath) as bundle_file: assert '"type": "bundle"' in bundle_file.read() camp1_r = fs_store.get(camp1.id) @@ -527,3 +595,334 @@ def test_related_to_by_target(rel_fs_store): assert len(resp) == 2 assert any(x['id'] == CAMPAIGN_ID for x in resp) assert any(x['id'] == INDICATOR_ID for x in resp) + + +def test_auth_set_white1(): + auth_set = AuthSet({"A"}, set()) + + assert auth_set.auth_type == AuthSet.WHITE + assert auth_set.values == {"A"} + + +def test_auth_set_white2(): + auth_set = AuthSet(set(), set()) + + assert auth_set.auth_type == AuthSet.WHITE + assert len(auth_set.values) == 0 + + +def test_auth_set_white3(): + auth_set = AuthSet({"A", "B"}, {"B", "C"}) + + assert auth_set.auth_type == AuthSet.WHITE + assert auth_set.values == {"A"} + + +def test_auth_set_black1(): + auth_set = AuthSet(None, {"B", "C"}) + + assert auth_set.auth_type == AuthSet.BLACK + assert auth_set.values == {"B", "C"} + + +def test_optimize_types1(): + filters = [ + Filter("type", "=", "foo") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"foo"} + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types2(): + filters = [ + Filter("type", "=", "foo"), + Filter("type", "=", "bar") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert len(auth_types.values) == 0 + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types3(): + filters = [ + Filter("type", "in", ["A", "B", "C"]), + Filter("type", "in", ["B", "C", "D"]) + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"B", "C"} + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types4(): + filters = [ + Filter("type", "in", ["A", "B", "C"]), + Filter("type", "in", ["D", "E", "F"]) + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert len(auth_types.values) == 0 + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types5(): + filters = [ + Filter("type", "in", ["foo", "bar"]), + Filter("type", "!=", "bar") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"foo"} + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types6(): + filters = [ + Filter("type", "!=", "foo"), + Filter("type", "!=", "bar") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.BLACK + assert auth_types.values == {"foo", "bar"} + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types7(): + filters = [ + Filter("type", "=", "foo"), + Filter("type", "!=", "foo") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert len(auth_types.values) == 0 + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types8(): + filters = [] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.BLACK + assert len(auth_types.values) == 0 + assert auth_ids.auth_type == AuthSet.BLACK + assert len(auth_ids.values) == 0 + + +def test_optimize_types_ids1(): + filters = [ + Filter("type", "in", ["foo", "bar"]), + Filter("id", "=", "foo--00000000-0000-0000-0000-000000000000") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"foo"} + assert auth_ids.auth_type == AuthSet.WHITE + assert auth_ids.values == {"foo--00000000-0000-0000-0000-000000000000"} + + +def test_optimize_types_ids2(): + filters = [ + Filter("type", "=", "foo"), + Filter("id", "=", "bar--00000000-0000-0000-0000-000000000000") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert len(auth_types.values) == 0 + assert auth_ids.auth_type == AuthSet.WHITE + assert len(auth_ids.values) == 0 + + +def test_optimize_types_ids3(): + filters = [ + Filter("type", "in", ["foo", "bar"]), + Filter("id", "!=", "bar--00000000-0000-0000-0000-000000000000") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"foo", "bar"} + assert auth_ids.auth_type == AuthSet.BLACK + assert auth_ids.values == {"bar--00000000-0000-0000-0000-000000000000"} + + +def test_optimize_types_ids4(): + filters = [ + Filter("type", "in", ["A", "B", "C"]), + Filter("id", "in", [ + "B--00000000-0000-0000-0000-000000000000", + "C--00000000-0000-0000-0000-000000000000", + "D--00000000-0000-0000-0000-000000000000", + ]) + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"B", "C"} + assert auth_ids.auth_type == AuthSet.WHITE + assert auth_ids.values == { + "B--00000000-0000-0000-0000-000000000000", + "C--00000000-0000-0000-0000-000000000000" + } + + +def test_optimize_types_ids5(): + filters = [ + Filter("type", "in", ["A", "B", "C"]), + Filter("type", "!=", "C"), + Filter("id", "in", [ + "B--00000000-0000-0000-0000-000000000000", + "C--00000000-0000-0000-0000-000000000000", + "D--00000000-0000-0000-0000-000000000000" + ]), + Filter("id", "!=", "D--00000000-0000-0000-0000-000000000000") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"B"} + assert auth_ids.auth_type == AuthSet.WHITE + assert auth_ids.values == {"B--00000000-0000-0000-0000-000000000000"} + + +def test_optimize_types_ids6(): + filters = [ + Filter("id", "=", "A--00000000-0000-0000-0000-000000000000") + ] + + auth_types, auth_ids = _find_search_optimizations(filters) + + assert auth_types.auth_type == AuthSet.WHITE + assert auth_types.values == {"A"} + assert auth_ids.auth_type == AuthSet.WHITE + assert auth_ids.values == {"A--00000000-0000-0000-0000-000000000000"} + + +def test_search_auth_set_white1(): + auth_set = AuthSet( + {"attack-pattern", "doesntexist"}, + set() + ) + + results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) + assert results == ["attack-pattern"] + + results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISREG) + assert len(results) == 0 + + +def test_search_auth_set_white2(): + auth_set = AuthSet( + { + "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38", + "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841" + + }, + { + "malware--92ec0cbd-2c30-44a2-b270-73f4ec949841", + "malware--96b08451-b27a-4ff6-893f-790e26393a8e", + "doesntexist" + } + ) + + results = _get_matching_dir_entries( + os.path.join(FS_PATH, "malware"), + auth_set, stat.S_ISDIR + ) + + assert results == ["malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"] + + +def test_search_auth_set_white3(): + auth_set = AuthSet({"20170531213258226477", "doesntexist"}, set()) + + results = _get_matching_dir_entries( + os.path.join(FS_PATH, "malware", + "malware--6b616fc1-1505-48e3-8b2c-0d19337bff38"), + auth_set, stat.S_ISREG, ".json" + ) + + assert results == ["20170531213258226477.json"] + + +def test_search_auth_set_black1(): + auth_set = AuthSet( + None, + {"tool--242f3da3-4425-4d11-8f5c-b842886da966", "doesntexist"} + ) + + results = _get_matching_dir_entries( + os.path.join(FS_PATH, "tool"), + auth_set, stat.S_ISDIR + ) + + assert set(results) == { + "tool--03342581-f790-4f03-ba41-e82e67392e23" + } + + +def test_search_auth_set_white_empty(): + auth_set = AuthSet( + set(), + set() + ) + + results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) + + assert len(results) == 0 + + +def test_search_auth_set_black_empty(rel_fs_store): + # Ensure rel_fs_store fixture has run so that the type directories are + # predictable (it adds "campaign"). + auth_set = AuthSet( + None, + set() + ) + + results = _get_matching_dir_entries(FS_PATH, auth_set, stat.S_ISDIR) + + # Should get all dirs + assert set(results) == { + "attack-pattern", + "campaign", + "course-of-action", + "identity", + "indicator", + "intrusion-set", + "malware", + "marking-definition", + "relationship", + "tool" + } diff --git a/stix2/utils.py b/stix2/utils.py index 64c4a32..63b3d45 100644 --- a/stix2/utils.py +++ b/stix2/utils.py @@ -7,6 +7,8 @@ import json from dateutil import parser import pytz +import stix2.base + from .exceptions import (InvalidValueError, RevokeError, UnmodifiablePropertyError) @@ -364,3 +366,20 @@ def remove_custom_stix(stix_obj): def get_type_from_id(stix_id): return stix_id.split('--', 1)[0] + + +def is_marking(obj_or_id): + """Determines whether the given object or object ID is/is for a marking + definition. + + :param obj_or_id: A STIX object or object ID as a string. + :return: True if a marking definition, False otherwise. + """ + + if isinstance(obj_or_id, (stix2.base._STIXBase, dict)): + result = obj_or_id["type"] == "marking-definition" + else: + # it's a string ID + result = obj_or_id.startswith("marking-definition--") + + return result