From 9467e101bf95d058385b1f5a0ed385957756811e Mon Sep 17 00:00:00 2001 From: niclas Date: Thu, 22 Feb 2024 12:12:31 +0100 Subject: [PATCH] Add [config] optional "private" relations --- tools/tidal-api/README.md | 19 ++++++++++++++++++- tools/tidal-api/config.json | 6 ++++++ tools/tidal-api/main.py | 21 ++++++++++++++------- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md index e0412d8..3a34cd0 100644 --- a/tools/tidal-api/README.md +++ b/tools/tidal-api/README.md @@ -42,7 +42,7 @@ The configuration file is located in `config.json` and maps the fields of the Ti >Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration. ### Extraction Configuration -The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this: +The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in an object in the API response. The extraction configuration looks like this: ```json { "extract": "", @@ -56,6 +56,23 @@ The extraction configuration is a dictionary that maps the fields of the Tidal C - `multiple`: Extracts multiple values from the API response - `reverse`: Gets the value of the key and writes it into an array (no subkey needed) +### "Private" Relations +The Tidal Cyber API provides relations between different objects. Some of these relations point to objects that are not part of the galaxies created based on the API response nor are they part of the MISP galaxy. These relations can be marked as `private` in the config file. For example: +```json + "related": { + "tactic": { + "mode": "public", + "dest-uuid": "tactic_id", + "type": "uses" + }, + "sub_technique": { + "mode": "private", + "dest-uuid": "id", + "type": "sub-technique-of" + } + }, +``` + ## Usage ```bash python3 main.py create-galaxy -v --type diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json index 9d2010b..2aaf32b 100644 --- a/tools/tidal-api/config.json +++ b/tools/tidal-api/config.json @@ -134,10 +134,12 @@ }, "related": { "groups": { + "mode": "private", "dest-uuid": "group_id", "type": "used-by" }, "associated_software": { + "mode": "private", "dest-uuid": "id", "type": "related-to" } @@ -175,6 +177,7 @@ }, "related": { "associated_groups": { + "mode": "private", "dest-uuid": "id", "type": "related-to" } @@ -214,10 +217,12 @@ }, "related": { "tactic": { + "mode": "public", "dest-uuid": "tactic_id", "type": "uses" }, "sub_technique": { + "mode": "private", "dest-uuid": "id", "type": "sub-technique-of" } @@ -236,6 +241,7 @@ }, "related": { "techniques": { + "mode": "public", "dest-uuid": "technique_id", "type": "uses" } diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 98b29d7..f2e248f 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -16,7 +16,7 @@ CLUSTER_CONFIGS = config["CLUSTER_CONFIGS"] VALUE_FIELDS = config["VALUE_FIELDS"] -def create_cluster_values(data, cluster): +def create_cluster_values(data, cluster, add_private): value_fields = VALUE_FIELDS[cluster.internal_type] for entry in data["data"]: values = {} @@ -28,7 +28,7 @@ def create_cluster_values(data, cluster): metadata = create_metadata(entry, value) values["meta"] = metadata case "related": - relations = create_relations(entry, value) + relations = create_relations(entry, value, add_private) values["related"] = relations case "uuid": values[key] = entry.get(value) @@ -63,13 +63,17 @@ def create_metadata(data, format): return metadata -def create_relations(data, format): +def create_relations(data, format, add_private): relations = [] for i in range(len(list(format))): for relation in data[list(format)[i]]: + if not add_private and list(format.values())[i].get("mode") == "private": + continue relation_entry = {} for relation_key, relation_value in list(format.values())[i].items(): if relation_key != "type": + if relation_key == "mode": + continue relation_entry[relation_key] = relation.get(relation_value) else: relation_entry[relation_key] = relation_value @@ -77,14 +81,14 @@ def create_relations(data, format): return relations -def create_galaxy_and_cluster(galaxy_type, version): +def create_galaxy_and_cluster(galaxy_type, version, add_private=False): api = TidalAPI() galaxy = Galaxy(**GALAXY_CONFIGS[galaxy_type], version=version) galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{galaxy_type}.json") cluster = Cluster(**CLUSTER_CONFIGS[galaxy_type], internal_type=galaxy_type) data = api.get_data(galaxy_type) - create_cluster_values(data, cluster) + create_cluster_values(data, cluster, add_private) cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") print(f"Galaxy tidal-{galaxy_type} created") @@ -93,9 +97,9 @@ def create_galaxy_and_cluster(galaxy_type, version): def create_galaxy(args): if args.all: for galaxy_type in GALAXY_CONFIGS: - create_galaxy_and_cluster(galaxy_type, args.version) + create_galaxy_and_cluster(galaxy_type, args.version, args.addprivate) else: - create_galaxy_and_cluster(args.type, args.version) + create_galaxy_and_cluster(args.type, args.version, args.addprivate) if __name__ == "__main__": @@ -118,6 +122,9 @@ if __name__ == "__main__": galaxy_parser.add_argument( "--all", action="store_true", help="Flag to create all predefined galaxy types" ) + galaxy_parser.add_argument( + "--addprivate", action="store_true", help="Flag to add private relations" + ) galaxy_parser.set_defaults(func=create_galaxy) args = parser.parse_args()