From a33e9e2a140d5ea8f90ef3d80d2f3157dc9fd332 Mon Sep 17 00:00:00 2001 From: niclas Date: Mon, 19 Feb 2024 11:30:10 +0100 Subject: [PATCH 01/10] Add [tidal] scipts to create new galaxies --- tools/tidal-api/api.py | 15 ++++++ tools/tidal-api/create_campaigns.py | 62 ++++++++++++++++++++++++ tools/tidal-api/create_groups.py | 73 +++++++++++++++++++++++++++++ tools/tidal-api/create_software.py | 73 +++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 tools/tidal-api/api.py create mode 100644 tools/tidal-api/create_campaigns.py create mode 100644 tools/tidal-api/create_groups.py create mode 100644 tools/tidal-api/create_software.py diff --git a/tools/tidal-api/api.py b/tools/tidal-api/api.py new file mode 100644 index 0000000..0e313f6 --- /dev/null +++ b/tools/tidal-api/api.py @@ -0,0 +1,15 @@ +import requests + +class TidalAPI: + def __init__(self): + self.base_url = 'https://app-api.tidalcyber.com/api/v1/' + + def get_data(self, endpoint): + url = self.base_url + endpoint + try: + response = requests.get(url) + return response.json() + except Exception as e: + print(f'Error: {e}') + return None + diff --git a/tools/tidal-api/create_campaigns.py b/tools/tidal-api/create_campaigns.py new file mode 100644 index 0000000..9604361 --- /dev/null +++ b/tools/tidal-api/create_campaigns.py @@ -0,0 +1,62 @@ +from api import TidalAPI +import json + +VERSION = 1 +GALAXY_PATH = "../../galaxies/" +CLUSTER_PATH = "../../clusters/" +GALAXY_UUID = "43a8fce6-08d3-46c2-957d-53606efe2c48" + +def create_galaxy(): + galaxy = {} + galaxy["description"] = "Tidal Campaigns Galaxy" + galaxy["name"] = "Tidal Campaigns" + galaxy["namespace"] = "tidal" + galaxy["type"] = "campaigns" + galaxy["uuid"] = GALAXY_UUID + galaxy["version"] = VERSION + return galaxy + +def create_cluster(galaxy, data): + cluster = {} + values = [] + + for campaigns in data["data"]: + value = {} + relations = [] + + value["description"] = campaigns["description"] + + value["meta"] = {} + value["meta"]["source"] = campaigns["source"] + value["meta"]["campaign-attack-id"] = campaigns["campaign_attack_id"] + value["meta"]["first-seen"] = campaigns["first_seen"] + value["meta"]["last-seen"] = campaigns["last_seen"] + value["meta"]["tags"] = campaigns["tags"] + value["meta"]["owner"] = campaigns["owner_name"] + + value["related"] = relations + value["uuid"] = campaigns["id"] + value["value"] = campaigns["name"] + values.append(value) + + cluster["authors"] = ["Tidal"] + cluster["category"] = "Threat campaigns" + cluster["description"] = "Tidal Campaigns" + cluster["name"] = "Tidal Campaigns" + cluster["source"] = "https://app-api.tidalcyber.com/api/v1/campaigns" + cluster["type"] = "campaigns" + cluster["uuid"] = galaxy["uuid"] + cluster["values"] = values + return cluster + +if __name__ == "__main__": + api = TidalAPI() + data = api.get_data('campaigns') + galaxy = create_galaxy() + cluster = create_cluster(galaxy, data) + + with open(GALAXY_PATH + "tidal-campaigns.json", "w") as galaxy_file: + json.dump(galaxy, galaxy_file, indent=4) + + with open(CLUSTER_PATH + "tidal-campaigns.json", "w") as cluster_file: + json.dump(cluster, cluster_file, indent=4) diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py new file mode 100644 index 0000000..940053a --- /dev/null +++ b/tools/tidal-api/create_groups.py @@ -0,0 +1,73 @@ +from api import TidalAPI +import json + +VERSION = 1 +GALAXY_PATH = "../../galaxies/" +CLUSTER_PATH = "../../clusters/" +GALAXY_UUID = "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6" + +def create_galaxy(): + galaxy = {} + galaxy["description"] = "Tidal Threat Group Galaxy" + galaxy["name"] = "Tidal Threat Group" + galaxy["namespace"] = "tidal" + galaxy["type"] = "threat-group" + galaxy["uuid"] = GALAXY_UUID + galaxy["version"] = VERSION + return galaxy + + +def create_cluster(galaxy, data): + cluster = {} + values = [] + + for group in data["data"]: + value = {} + relations = [] + # TODO check for id and associated_group_id and add to relations + for entry in group["associated_groups"]: + relation = {} + relation["dest-uuid"] = entry["id"] + relation["type"] = "related-to" + relations.append(relation) + + value["description"] = group["description"] + + value["meta"] = {} + value["meta"]["source"] = group["source"] + value["meta"]["group-attack-id"] = group["group_attack_id"] + value["meta"]["country"] = [country["country_code"] for country in group["country"]] + value["meta"]["observed_country"] = [country["country_code"] for country in group["observed_country"]] + value["meta"]["motive"] = [motive["name"] for motive in group["observed_motivation"]] + value["meta"]["target-category"] = [sector["name"] for sector in group["observed_sector"]] + value["meta"]["tags"] = group["tags"] + value["meta"]["owner"] = group["owner_name"] + + value["related"] = relations + value["uuid"] = group["id"] + value["value"] = group["name"] + values.append(value) + + cluster["authors"] = ["Tidal"] + cluster["category"] = "Threat Group" + cluster["description"] = "Tidal Threat Groups" + cluster["name"] = "Tidal Threat Group" + cluster["source"] = "https://app-api.tidalcyber.com/api/v1/groups" + cluster["type"] = "threat-group" + cluster["uuid"] = galaxy["uuid"] + cluster["values"] = values + return cluster + + +if __name__ == "__main__": + + api = TidalAPI() + data = api.get_data("groups") + galaxy = create_galaxy() + cluster = create_cluster(galaxy, data) + + with open(GALAXY_PATH + "tidal-threat-group.json", "w") as galaxy_file: + json.dump(galaxy, galaxy_file, indent=4) + + with open(CLUSTER_PATH + "tidal-threat-group.json", "w") as cluster_file: + json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py new file mode 100644 index 0000000..1219c21 --- /dev/null +++ b/tools/tidal-api/create_software.py @@ -0,0 +1,73 @@ +from api import TidalAPI +import json + +VERSION = 1 +GALAXY_PATH = "../../galaxies/" +CLUSTER_PATH = "../../clusters/" +GALAXY_UUID = "38d62d8b-4c49-489a-9bc4-8e294c4f04f7" + +def create_galaxy(): + galaxy = {} + galaxy["description"] = "Tidal Software Galaxy" + galaxy["name"] = "Tidal Software" + galaxy["namespace"] = "tidal" + galaxy["type"] = "software" + galaxy["uuid"] = GALAXY_UUID + galaxy["version"] = VERSION + return galaxy + +def create_cluster(galaxy, data): + cluster = {} + values = [] + + for software in data["data"]: + value = {} + relations = [] + # TODO check for relations etc. + for entry in software["groups"]: + relation = {} + relation["dest-uuid"] = entry["id"] + relation["type"] = "used-by" + relations.append(relation) + for entry in software["associated_software"]: + relation = {} + relation["dest-uuid"] = entry["id"] + relation["type"] = "related-to" + relations.append(relation) + + value["description"] = software["description"] + + value["meta"] = {} + value["meta"]["source"] = software["source"] + value["meta"]["type"] = software["type"] + value["meta"]["software-attack-id"] = software["software_attack_id"] + value["meta"]["platforms"] = software["platforms"] + value["meta"]["tags"] = software["tags"] + value["meta"]["owner"] = software["owner_name"] + + value["related"] = relations + value["uuid"] = software["id"] + value["value"] = software["name"] + values.append(value) + + cluster["authors"] = ["Tidal"] + cluster["category"] = "Threat software" + cluster["description"] = "Tidal Threat Groups" + cluster["name"] = "Tidal Threat software" + cluster["source"] = "https://app-api.tidalcyber.com/api/v1/software" + cluster["type"] = "threat-software" + cluster["uuid"] = galaxy["uuid"] + cluster["values"] = values + return cluster + +if __name__ == "__main__": + api = TidalAPI() + data = api.get_data('software') + galaxy = create_galaxy() + cluster = create_cluster(galaxy, data) + + with open(GALAXY_PATH + "tidal-software.json", "w") as galaxy_file: + json.dump(galaxy, galaxy_file, indent=4) + + with open(CLUSTER_PATH + "tidal-software.json", "w") as cluster_file: + json.dump(cluster, cluster_file, indent=4) \ No newline at end of file From 059de052ad43114558bd594fb9c2bf5c35a498f7 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 20 Feb 2024 11:56:55 +0100 Subject: [PATCH 02/10] chg [tidal] only generate set metadata --- tools/tidal-api/create_campaigns.py | 26 +++++++++++++++++----- tools/tidal-api/create_groups.py | 34 ++++++++++++++++++++++------- tools/tidal-api/create_software.py | 26 +++++++++++++++++----- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/tools/tidal-api/create_campaigns.py b/tools/tidal-api/create_campaigns.py index 9604361..8b6123c 100644 --- a/tools/tidal-api/create_campaigns.py +++ b/tools/tidal-api/create_campaigns.py @@ -26,13 +26,27 @@ def create_cluster(galaxy, data): value["description"] = campaigns["description"] + # Metadata fields + source = campaigns["source"] + campaign_attack_id = campaigns["campaign_attack_id"] + first_seen = campaigns["first_seen"] + last_seen = campaigns["last_seen"] + tags = campaigns["tags"] + owner = campaigns["owner_name"] + value["meta"] = {} - value["meta"]["source"] = campaigns["source"] - value["meta"]["campaign-attack-id"] = campaigns["campaign_attack_id"] - value["meta"]["first-seen"] = campaigns["first_seen"] - value["meta"]["last-seen"] = campaigns["last_seen"] - value["meta"]["tags"] = campaigns["tags"] - value["meta"]["owner"] = campaigns["owner_name"] + if source: + value["meta"]["source"] = source + if campaign_attack_id: + value["meta"]["campaign-attack-id"] = campaign_attack_id + if first_seen: + value["meta"]["first-seen"] = first_seen + if last_seen: + value["meta"]["last-seen"] = last_seen + if tags: + value["meta"]["tags"] = tags + if owner: + value["meta"]["owner"] = owner value["related"] = relations value["uuid"] = campaigns["id"] diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py index 940053a..163d80f 100644 --- a/tools/tidal-api/create_groups.py +++ b/tools/tidal-api/create_groups.py @@ -33,15 +33,33 @@ def create_cluster(galaxy, data): value["description"] = group["description"] + # Metadata fields + source = group["source"] + group_attack_id = group["group_attack_id"] + country = [country["country_name"] for country in group["country"]] + observed_country = [country["country_code"] for country in group["observed_country"]] + motive = [motive["name"] for motive in group["observed_motivation"]] + target_category = [sector["name"] for sector in group["observed_sector"]] + tags = group["tags"] + owner = group["owner_name"] + value["meta"] = {} - value["meta"]["source"] = group["source"] - value["meta"]["group-attack-id"] = group["group_attack_id"] - value["meta"]["country"] = [country["country_code"] for country in group["country"]] - value["meta"]["observed_country"] = [country["country_code"] for country in group["observed_country"]] - value["meta"]["motive"] = [motive["name"] for motive in group["observed_motivation"]] - value["meta"]["target-category"] = [sector["name"] for sector in group["observed_sector"]] - value["meta"]["tags"] = group["tags"] - value["meta"]["owner"] = group["owner_name"] + if source: + value["meta"]["source"] = source + if group_attack_id: + value["meta"]["group-attack-id"] = group_attack_id + if country: + value["meta"]["country"] = country + if observed_country: + value["meta"]["observed_country"] = observed_country + if motive: + value["meta"]["motive"] = motive + if target_category: + value["meta"]["target-category"] = target_category + if tags: + value["meta"]["tags"] = tags + if owner: + value["meta"]["owner"] = owner value["related"] = relations value["uuid"] = group["id"] diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py index 1219c21..9cebad3 100644 --- a/tools/tidal-api/create_software.py +++ b/tools/tidal-api/create_software.py @@ -37,13 +37,27 @@ def create_cluster(galaxy, data): value["description"] = software["description"] + # Metadata fields + source = software["source"] + type = software["type"] + software_attack_id = software["software_attack_id"] + platforms = software["platforms"] + tags = software["tags"] + owner = software["owner_name"] + value["meta"] = {} - value["meta"]["source"] = software["source"] - value["meta"]["type"] = software["type"] - value["meta"]["software-attack-id"] = software["software_attack_id"] - value["meta"]["platforms"] = software["platforms"] - value["meta"]["tags"] = software["tags"] - value["meta"]["owner"] = software["owner_name"] + if source: + value["meta"]["source"] = source + if type: + value["meta"]["type"] = type + if software_attack_id: + value["meta"]["software-attack-id"] = software_attack_id + if platforms: + value["meta"]["platforms"] = platforms + if tags: + value["meta"]["tags"] = tags + if owner: + value["meta"]["owner"] = owner value["related"] = relations value["uuid"] = software["id"] From 108e43e1ca9574c78f98a2d771ff7099e1ac5d9b Mon Sep 17 00:00:00 2001 From: niclas Date: Wed, 21 Feb 2024 16:24:48 +0100 Subject: [PATCH 03/10] Refactor [creation] script --- tools/tidal-api/api/__init__.py | 0 tools/tidal-api/{ => api}/api.py | 0 tools/tidal-api/create_groups.py | 4 +- tools/tidal-api/create_software.py | 19 +++ tools/tidal-api/main.py | 218 +++++++++++++++++++++++++++++ tools/tidal-api/models/__init__.py | 0 tools/tidal-api/models/cluster.py | 23 +++ tools/tidal-api/models/galaxy.py | 14 ++ tools/tidal-api/utils/__init__.py | 0 tools/tidal-api/utils/extractor.py | 6 + 10 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 tools/tidal-api/api/__init__.py rename tools/tidal-api/{ => api}/api.py (100%) create mode 100644 tools/tidal-api/main.py create mode 100644 tools/tidal-api/models/__init__.py create mode 100644 tools/tidal-api/models/cluster.py create mode 100644 tools/tidal-api/models/galaxy.py create mode 100644 tools/tidal-api/utils/__init__.py create mode 100644 tools/tidal-api/utils/extractor.py diff --git a/tools/tidal-api/api/__init__.py b/tools/tidal-api/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/tidal-api/api.py b/tools/tidal-api/api/api.py similarity index 100% rename from tools/tidal-api/api.py rename to tools/tidal-api/api/api.py diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py index 163d80f..18431a6 100644 --- a/tools/tidal-api/create_groups.py +++ b/tools/tidal-api/create_groups.py @@ -84,8 +84,8 @@ if __name__ == "__main__": galaxy = create_galaxy() cluster = create_cluster(galaxy, data) - with open(GALAXY_PATH + "tidal-threat-group.json", "w") as galaxy_file: + with open(GALAXY_PATH + "tidal-threat-groups.json", "w") as galaxy_file: json.dump(galaxy, galaxy_file, indent=4) - with open(CLUSTER_PATH + "tidal-threat-group.json", "w") as cluster_file: + with open(CLUSTER_PATH + "tidal-threat-groups.json", "w") as cluster_file: json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py index 9cebad3..27ff409 100644 --- a/tools/tidal-api/create_software.py +++ b/tools/tidal-api/create_software.py @@ -1,5 +1,6 @@ from api import TidalAPI import json +import re VERSION = 1 GALAXY_PATH = "../../galaxies/" @@ -38,6 +39,7 @@ def create_cluster(galaxy, data): value["description"] = software["description"] # Metadata fields + links = extract_links(software["description"]) source = software["source"] type = software["type"] software_attack_id = software["software_attack_id"] @@ -46,6 +48,8 @@ def create_cluster(galaxy, data): owner = software["owner_name"] value["meta"] = {} + if links: + value["meta"]["refs"] = list(links) if source: value["meta"]["source"] = source if type: @@ -74,6 +78,21 @@ def create_cluster(galaxy, data): cluster["values"] = values return cluster +def extract_links(text): + # extract markdown links and return text without links and the links + # urls = re.findall(r'https?://[^\s\)]+', text) + regular_links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text) + # sup_links = re.findall(r'\[\[([^\]]+)\]\((https?://[^\s\)]+)\)\]', text) + + # Extracting URLs from the tuples + regular_links_urls = set([url for text, url in regular_links]) + # sup_links_urls = [url for text, url in sup_links] + + # text_without_links = re.sub(r'\[([^\]]+)\]\(https?://[^\s\)]+\)', r'\1', text) + # text_without_sup = re.sub(r'.*<\/sup>', '', text_without_links) + + return regular_links_urls + if __name__ == "__main__": api = TidalAPI() data = api.get_data('software') diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py new file mode 100644 index 0000000..8bf9691 --- /dev/null +++ b/tools/tidal-api/main.py @@ -0,0 +1,218 @@ +from api.api import TidalAPI +from models.galaxy import Galaxy +from models.cluster import Cluster +from utils.extractor import extract_links +import argparse + +CLUSTER_PATH = "../../clusters/" +GALAXY_PATH = "../../galaxies/" + +UUIDS = { + "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", + "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", + "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", +} + +GALAXY_CONFIGS = { + "software": { + "name": "Tidal Software", + "namespace": "tidal", + "description": "Tidal Software Galaxy", + "type": "software", + "uuid": UUIDS["software"], + }, + "groups": { + "name": "Tidal Groups", + "namespace": "tidal", + "description": "Tidal Groups Galaxy", + "type": "groups", + "uuid": UUIDS["groups"], + }, + "campaigns": { + "name": "Tidal Campaigns", + "namespace": "tidal", + "description": "Tidal Campaigns Galaxy", + "type": "campaigns", + "uuid": UUIDS["campaigns"], + } +} + +CLUSTER_CONFIGS = { + "software": { + "authors": "Tidal", + "category": "Software", + "description": "Tidal Software Cluster", + "name": "Tidal Software", + "source": "Tidal", + "type": "software", + "uuid": UUIDS["software"], + "values": [] + }, + "groups": { + "authors": "Tidal", + "category": "Threat Groups", + "description": "Tidal Threat Groups Cluster", + "name": "Tidal Threat Groups", + "source": "Tidal", + "type": "groups", + "uuid": UUIDS["groups"], + "values": [] + }, + "campaigns": { + "authors": "Tidal", + "category": "Campaigns", + "description": "Tidal Campaigns Cluster", + "name": "Tidal Campaigns", + "source": "Tidal", + "type": "campaigns", + "uuid": UUIDS["campaigns"], + "values": [] + } +} + +VALUE_FIELDS = { + "software": { + "description": "description", + "meta": { + "source": "source", + "type": "type", + "software-attack-id": "software_attack_id", + "platforms": "platforms", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "groups": { + "dest-uuid": "group_id", + "type": "used-by" + }, + "associated_software": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "groups": { + "description": "description", + "meta": { + "source": "source", + "group-attack-id": "group_attack_id", + "country": {"extract": "single", "key": "country", "subkey": "country_code"}, + "observed_country": {"extract": "multiple", "key": "observed_country", "subkey": "country_code"}, + "observed_motivation": {"extract": "multiple", "key": "observed_motivation", "subkey": "name"}, + "target-category": {"extract": "multiple", "key": "observed_sector", "subkey": "name"}, + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "associated_groups": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "campaigns": { + "description": "description", + "meta": { + "source": "source", + "campaign-attack-id": "campaign_attack_id", + "first_seen": "first_seen", + "last_seen": "last_seen", + "tags": "tags", + "owner": "owner_name" + }, + "related": {}, + "uuid": "id", + "value": "name" + } + +} + +def create_cluster_values(data, cluster): + value_fields = VALUE_FIELDS[cluster.internal_type] + for entry in data["data"]: + values = {} + for key, value in value_fields.items(): + match key: + case "description": + values[value] = entry.get(key) + case "meta": + metadata = create_metadata(entry, value) + values["meta"] = metadata + case "related": + relations = create_relations(entry, value) + values["related"] = relations + case "uuid": + values[key] = entry.get(value) + case "value": + values[key] = entry.get(value) + case _: + print(f"Error: Invalid configuration for {key} in {cluster.internal_type} value fields.") + cluster.add_value(values) + +def create_metadata(data, format): + metadata = {} + for meta_key, meta_value in format.items(): + if isinstance(meta_value, dict): + if meta_value.get("extract") == "single" and data.get(meta_value["key"]): + metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"]) + elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]): + metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])] + elif data.get(meta_value): + metadata[meta_key] = data.get(meta_value) + return metadata + + +def create_relations(data, format): + relations = [] + for i in range(len(list(format))): + for relation in data[list(format)[i]]: + relation_entry = {} + for relation_key, relation_value in list(format.values())[i].items(): + if relation_key != "type": + relation_entry[relation_key] = relation.get(relation_value) + else: + relation_entry[relation_key] = relation_value + relations.append(relation_entry) + return relations + + +def create_galaxy_and_cluster(galaxy_type, version): + api = TidalAPI() + galaxy = Galaxy(**GALAXY_CONFIGS[galaxy_type], version=version) + galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{galaxy_type}.json") + + cluster = Cluster(**CLUSTER_CONFIGS[galaxy_type], internal_type=galaxy_type) + data = api.get_data(galaxy_type) + create_cluster_values(data, cluster) + cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") + + print(f"Galaxy {galaxy_type} created") + +def create_galaxy(args): + if args.all: + for galaxy_type in GALAXY_CONFIGS: + create_galaxy_and_cluster(galaxy_type, args.version) + else: + create_galaxy_and_cluster(args.type, args.version) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Create a galaxy and cluster for Tidal API") + subparsers = parser.add_subparsers(dest="command") + + galaxy_parser = subparsers.add_parser("create_galaxy", help="Create a galaxy from the Tidal API") + galaxy_parser.add_argument("--type", choices=list(GALAXY_CONFIGS.keys()) + ['all'], help="The type of the galaxy") + galaxy_parser.add_argument("-v", "--version", type=int, required=True, help="The version of the galaxy") + galaxy_parser.add_argument("--all", action="store_true", help="Flag to create all predefined galaxy types") + galaxy_parser.set_defaults(func=create_galaxy) + + args = parser.parse_args() + if hasattr(args, 'func'): + args.func(args) + else: + parser.print_help() + diff --git a/tools/tidal-api/models/__init__.py b/tools/tidal-api/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py new file mode 100644 index 0000000..2436b89 --- /dev/null +++ b/tools/tidal-api/models/cluster.py @@ -0,0 +1,23 @@ +import json + +class Cluster(): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str, values: list, internal_type: str): + self.authors = authors + self.category = category + self.description = description + self.name = name + self.source = source + self.type = type + self.uuid = uuid + self.values = values + self.internal_type = internal_type + + def add_value(self, value): + self.values.append(value) + + def save_to_file(self, path): + with open(path, "w") as file: + file.write(json.dumps(self.__dict__, indent=4)) + + def get_config(self): + return self.__dict__ diff --git a/tools/tidal-api/models/galaxy.py b/tools/tidal-api/models/galaxy.py new file mode 100644 index 0000000..2de73b2 --- /dev/null +++ b/tools/tidal-api/models/galaxy.py @@ -0,0 +1,14 @@ +import json + +class Galaxy(): + def __init__(self, description, name, namespace, type, uuid, version): + self.description = description + self.name = name + self.namespace = namespace + self.type = type + self.uuid = uuid + self.version = version + + def save_to_file(self, path): + with open(path, "w") as file: + file.write(json.dumps(self.__dict__, indent=4)) diff --git a/tools/tidal-api/utils/__init__.py b/tools/tidal-api/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/tidal-api/utils/extractor.py b/tools/tidal-api/utils/extractor.py new file mode 100644 index 0000000..cfa534e --- /dev/null +++ b/tools/tidal-api/utils/extractor.py @@ -0,0 +1,6 @@ +import re + +def extract_links(text): + links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text) + urls = set([url for text, url in links]) + return urls \ No newline at end of file From b9746f2b4155526573aa7a647fd572f4989fe5d9 Mon Sep 17 00:00:00 2001 From: niclas Date: Thu, 22 Feb 2024 10:18:18 +0100 Subject: [PATCH 04/10] chg [config] external config file --- tools/tidal-api/README.md | 67 ++++++++ tools/tidal-api/config.json | 249 ++++++++++++++++++++++++++++ tools/tidal-api/create_campaigns.py | 76 --------- tools/tidal-api/create_groups.py | 91 ---------- tools/tidal-api/create_software.py | 106 ------------ tools/tidal-api/main.py | 133 +-------------- tools/tidal-api/utils/config.py | 14 ++ 7 files changed, 339 insertions(+), 397 deletions(-) create mode 100644 tools/tidal-api/README.md create mode 100644 tools/tidal-api/config.json delete mode 100644 tools/tidal-api/create_campaigns.py delete mode 100644 tools/tidal-api/create_groups.py delete mode 100644 tools/tidal-api/create_software.py create mode 100644 tools/tidal-api/utils/config.py diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md new file mode 100644 index 0000000..8e8b992 --- /dev/null +++ b/tools/tidal-api/README.md @@ -0,0 +1,67 @@ +# Tidal Cyber API + +This is a tool generating MISP galaxies and clusters from Tidal Cyber API. + +## Endpoints +https://app-api.tidalcyber.com/api/v1/technique + +https://app-api.tidalcyber.com/api/v1/references + +https://app-api.tidalcyber.com/api/v1/tactic + +https://app-api.tidalcyber.com/api/v1/campaigns/ + +https://app-api.tidalcyber.com/api/v1/software/ + +https://app-api.tidalcyber.com/api/v1/groups/ + +## Configuration +The configuration file is located in `config.json` and maps the fields of the Tidal API to the Galaxy and Cluster fields. It consists of the following sections: +- `UUID`: The UUID of the galaxy to be created +- `GALAXY_CONFIGS`: The configuration of the galaxies to be created in the `galaxies` folder of the MISP-galaxy repository + - `name`: The name of the galaxy + - `namespace`: The namespace of the galaxy + - `description`: The description of the galaxy + - `type`: The type of the galaxy + - `uuid`: The UUID of the galaxy (will be inserted from the `UUID` section) +- `CLUSTER_CONFIGS`: The configuration of the clusters to be created in the `clusters` folder of the MISP-galaxy repository + - `authors`: The authors of the cluster + - `category`: The category of the cluster + - `description`: The description of the cluster + - `name`: The name of the cluster + - `source`: The source of the cluster + - `type`: The type of the cluster + - `uuid`: The UUID of the cluster (will be inserted from the `UUID` section) + - `values`: The values of the cluster (will be inserted from the `VALUE_FIELDS` section) +- `VALUE_FIELDS`: Defines the mapping of the fields in the Tidal Cyber API to the fields in the MISP cluster values array + - `description`: The description of the cluster value + - `meta`: The metadata of the cluster value + - `related`: The related cluster values of the cluster value (you can define a `type` for each relation type in the config which will not be mapped to a field of the API) + - `uuid`: The UUID of the cluster value + - `value`: The value of the cluster value + >Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration. + +### Extraction Configuration +The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this: +```json +{ + "extract": , + "key": , + "subkey": +} +``` +**Extract modes**: + +- `single`: Extracts a single value from the API response +- `multiple`: Extracts multiple values from the API response +- `reverse`: Gets the value of the key and writes it into an array (no subkey needed) + +## Usage +```bash +python3 main.py create-galaxy -v --type +``` +To build all galaxies and clusters, run the following command: + +```bash +python3 main.py create-galaxy -v --all +``` \ No newline at end of file diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json new file mode 100644 index 0000000..dd50dbe --- /dev/null +++ b/tools/tidal-api/config.json @@ -0,0 +1,249 @@ +{ + "UUIDS": { + "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", + "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", + "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", + "technique": "3e28b683-8159-4398-8729-7248eac9aa45", + "tactic": "16a396e2-a4a9-4dfd-be0a-6ba75fb5382c", + "references": "cf5e180f-26e9-42b2-b10c-c9e55f448750" + }, + "GALAXY_CONFIGS": { + "software": { + "name": "Tidal Software", + "namespace": "tidal", + "description": "Tidal Software Galaxy", + "type": "software", + "uuid": "" + }, + "groups": { + "name": "Tidal Groups", + "namespace": "tidal", + "description": "Tidal Groups Galaxy", + "type": "groups", + "uuid": "" + }, + "campaigns": { + "name": "Tidal Campaigns", + "namespace": "tidal", + "description": "Tidal Campaigns Galaxy", + "type": "campaigns", + "uuid": "" + }, + "technique": { + "name": "Tidal Technique", + "namespace": "tidal", + "description": "Tidal Technique Galaxy", + "type": "technique", + "uuid": "" + }, + "tactic": { + "name": "Tidal Tactic", + "namespace": "tidal", + "description": "Tidal Tactic Galaxy", + "type": "tactics", + "uuid": "" + }, + "references": { + "name": "Tidal References", + "namespace": "tidal", + "description": "Tidal References Galaxy", + "type": "references", + "uuid": "" + } + }, + "CLUSTER_CONFIGS": { + "software": { + "authors": "Tidal", + "category": "Software", + "description": "Tidal Software Cluster", + "name": "Tidal Software", + "source": "Tidal", + "type": "software", + "uuid": "", + "values": [] + }, + "groups": { + "authors": "Tidal", + "category": "Threat Groups", + "description": "Tidal Threat Groups Cluster", + "name": "Tidal Threat Groups", + "source": "Tidal", + "type": "groups", + "uuid": "", + "values": [] + }, + "campaigns": { + "authors": "Tidal", + "category": "Campaigns", + "description": "Tidal Campaigns Cluster", + "name": "Tidal Campaigns", + "source": "Tidal", + "type": "campaigns", + "uuid": "", + "values": [] + }, + "technique": { + "authors": "Tidal", + "category": "Techniques", + "description": "Tidal Techniques Cluster", + "name": "Tidal Techniques", + "source": "Tidal", + "type": "technique", + "uuid": "", + "values": [] + }, + "tactic": { + "authors": "Tidal", + "category": "Tactics", + "description": "Tidal Tactics Cluster", + "name": "Tidal Tactics", + "source": "Tidal", + "type": "tactic", + "uuid": "", + "values": [] + }, + "references": { + "authors": "Tidal", + "category": "References", + "description": "Tidal References Cluster", + "name": "Tidal References", + "source": "Tidal", + "type": "references", + "uuid": "", + "values": [] + } + }, + "VALUE_FIELDS": { + "software": { + "description": "description", + "meta": { + "source": "source", + "type": "type", + "software-attack-id": "software_attack_id", + "platforms": "platforms", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "groups": { + "dest-uuid": "group_id", + "type": "used-by" + }, + "associated_software": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "groups": { + "description": "description", + "meta": { + "source": "source", + "group-attack-id": "group_attack_id", + "country": { + "extract": "single", + "key": "country", + "subkey": "country_code" + }, + "observed_country": { + "extract": "multiple", + "key": "observed_country", + "subkey": "country_code" + }, + "observed_motivation": { + "extract": "multiple", + "key": "observed_motivation", + "subkey": "name" + }, + "target-category": { + "extract": "multiple", + "key": "observed_sector", + "subkey": "name" + }, + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "associated_groups": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "campaigns": { + "description": "description", + "meta": { + "source": "source", + "campaign-attack-id": "campaign_attack_id", + "first_seen": "first_seen", + "last_seen": "last_seen", + "tags": "tags", + "owner": "owner_name" + }, + "related": {}, + "uuid": "id", + "value": "name" + }, + "technique": { + "description": "description", + "meta": { + "source": "source", + "platforms": "platforms", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "tactic": { + "dest-uuid": "tactic_id", + "type": "uses" + }, + "sub_technique": { + "dest-uuid": "id", + "type": "sub-technique-of" + } + }, + "uuid": "id", + "value": "name" + }, + "tactic": { + "description": "description", + "meta": { + "source": "source", + "tactic-attack-id": "tactic_attack_id", + "ordinal_position": "ordinal_position", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "techniques": { + "dest-uuid": "technique_id", + "type": "uses" + } + }, + "uuid": "id", + "value": "name" + }, + "references": { + "description": "description", + "meta": { + "source": "source", + "refs": { + "extract": "reverse", + "key": "url" + }, + "owner": "owner_name", + "title": "title", + "author": "authors", + "date_accessed": "date_accessed", + "date_published": "date_published" + }, + "related": {}, + "uuid": "id", + "value": "name" + } + } +} \ No newline at end of file diff --git a/tools/tidal-api/create_campaigns.py b/tools/tidal-api/create_campaigns.py deleted file mode 100644 index 8b6123c..0000000 --- a/tools/tidal-api/create_campaigns.py +++ /dev/null @@ -1,76 +0,0 @@ -from api import TidalAPI -import json - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "43a8fce6-08d3-46c2-957d-53606efe2c48" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Campaigns Galaxy" - galaxy["name"] = "Tidal Campaigns" - galaxy["namespace"] = "tidal" - galaxy["type"] = "campaigns" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for campaigns in data["data"]: - value = {} - relations = [] - - value["description"] = campaigns["description"] - - # Metadata fields - source = campaigns["source"] - campaign_attack_id = campaigns["campaign_attack_id"] - first_seen = campaigns["first_seen"] - last_seen = campaigns["last_seen"] - tags = campaigns["tags"] - owner = campaigns["owner_name"] - - value["meta"] = {} - if source: - value["meta"]["source"] = source - if campaign_attack_id: - value["meta"]["campaign-attack-id"] = campaign_attack_id - if first_seen: - value["meta"]["first-seen"] = first_seen - if last_seen: - value["meta"]["last-seen"] = last_seen - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = campaigns["id"] - value["value"] = campaigns["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat campaigns" - cluster["description"] = "Tidal Campaigns" - cluster["name"] = "Tidal Campaigns" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/campaigns" - cluster["type"] = "campaigns" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - -if __name__ == "__main__": - api = TidalAPI() - data = api.get_data('campaigns') - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-campaigns.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-campaigns.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py deleted file mode 100644 index 18431a6..0000000 --- a/tools/tidal-api/create_groups.py +++ /dev/null @@ -1,91 +0,0 @@ -from api import TidalAPI -import json - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Threat Group Galaxy" - galaxy["name"] = "Tidal Threat Group" - galaxy["namespace"] = "tidal" - galaxy["type"] = "threat-group" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for group in data["data"]: - value = {} - relations = [] - # TODO check for id and associated_group_id and add to relations - for entry in group["associated_groups"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "related-to" - relations.append(relation) - - value["description"] = group["description"] - - # Metadata fields - source = group["source"] - group_attack_id = group["group_attack_id"] - country = [country["country_name"] for country in group["country"]] - observed_country = [country["country_code"] for country in group["observed_country"]] - motive = [motive["name"] for motive in group["observed_motivation"]] - target_category = [sector["name"] for sector in group["observed_sector"]] - tags = group["tags"] - owner = group["owner_name"] - - value["meta"] = {} - if source: - value["meta"]["source"] = source - if group_attack_id: - value["meta"]["group-attack-id"] = group_attack_id - if country: - value["meta"]["country"] = country - if observed_country: - value["meta"]["observed_country"] = observed_country - if motive: - value["meta"]["motive"] = motive - if target_category: - value["meta"]["target-category"] = target_category - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = group["id"] - value["value"] = group["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat Group" - cluster["description"] = "Tidal Threat Groups" - cluster["name"] = "Tidal Threat Group" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/groups" - cluster["type"] = "threat-group" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - - -if __name__ == "__main__": - - api = TidalAPI() - data = api.get_data("groups") - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-threat-groups.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-threat-groups.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py deleted file mode 100644 index 27ff409..0000000 --- a/tools/tidal-api/create_software.py +++ /dev/null @@ -1,106 +0,0 @@ -from api import TidalAPI -import json -import re - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "38d62d8b-4c49-489a-9bc4-8e294c4f04f7" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Software Galaxy" - galaxy["name"] = "Tidal Software" - galaxy["namespace"] = "tidal" - galaxy["type"] = "software" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for software in data["data"]: - value = {} - relations = [] - # TODO check for relations etc. - for entry in software["groups"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "used-by" - relations.append(relation) - for entry in software["associated_software"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "related-to" - relations.append(relation) - - value["description"] = software["description"] - - # Metadata fields - links = extract_links(software["description"]) - source = software["source"] - type = software["type"] - software_attack_id = software["software_attack_id"] - platforms = software["platforms"] - tags = software["tags"] - owner = software["owner_name"] - - value["meta"] = {} - if links: - value["meta"]["refs"] = list(links) - if source: - value["meta"]["source"] = source - if type: - value["meta"]["type"] = type - if software_attack_id: - value["meta"]["software-attack-id"] = software_attack_id - if platforms: - value["meta"]["platforms"] = platforms - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = software["id"] - value["value"] = software["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat software" - cluster["description"] = "Tidal Threat Groups" - cluster["name"] = "Tidal Threat software" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/software" - cluster["type"] = "threat-software" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - -def extract_links(text): - # extract markdown links and return text without links and the links - # urls = re.findall(r'https?://[^\s\)]+', text) - regular_links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text) - # sup_links = re.findall(r'\[\[([^\]]+)\]\((https?://[^\s\)]+)\)\]', text) - - # Extracting URLs from the tuples - regular_links_urls = set([url for text, url in regular_links]) - # sup_links_urls = [url for text, url in sup_links] - - # text_without_links = re.sub(r'\[([^\]]+)\]\(https?://[^\s\)]+\)', r'\1', text) - # text_without_sup = re.sub(r'.*<\/sup>', '', text_without_links) - - return regular_links_urls - -if __name__ == "__main__": - api = TidalAPI() - data = api.get_data('software') - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-software.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-software.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 8bf9691..008f171 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -2,135 +2,18 @@ from api.api import TidalAPI from models.galaxy import Galaxy from models.cluster import Cluster from utils.extractor import extract_links +from utils.config import load_config import argparse CLUSTER_PATH = "../../clusters/" GALAXY_PATH = "../../galaxies/" -UUIDS = { - "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", - "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", - "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", -} +config = load_config('./config.json') -GALAXY_CONFIGS = { - "software": { - "name": "Tidal Software", - "namespace": "tidal", - "description": "Tidal Software Galaxy", - "type": "software", - "uuid": UUIDS["software"], - }, - "groups": { - "name": "Tidal Groups", - "namespace": "tidal", - "description": "Tidal Groups Galaxy", - "type": "groups", - "uuid": UUIDS["groups"], - }, - "campaigns": { - "name": "Tidal Campaigns", - "namespace": "tidal", - "description": "Tidal Campaigns Galaxy", - "type": "campaigns", - "uuid": UUIDS["campaigns"], - } -} - -CLUSTER_CONFIGS = { - "software": { - "authors": "Tidal", - "category": "Software", - "description": "Tidal Software Cluster", - "name": "Tidal Software", - "source": "Tidal", - "type": "software", - "uuid": UUIDS["software"], - "values": [] - }, - "groups": { - "authors": "Tidal", - "category": "Threat Groups", - "description": "Tidal Threat Groups Cluster", - "name": "Tidal Threat Groups", - "source": "Tidal", - "type": "groups", - "uuid": UUIDS["groups"], - "values": [] - }, - "campaigns": { - "authors": "Tidal", - "category": "Campaigns", - "description": "Tidal Campaigns Cluster", - "name": "Tidal Campaigns", - "source": "Tidal", - "type": "campaigns", - "uuid": UUIDS["campaigns"], - "values": [] - } -} - -VALUE_FIELDS = { - "software": { - "description": "description", - "meta": { - "source": "source", - "type": "type", - "software-attack-id": "software_attack_id", - "platforms": "platforms", - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "groups": { - "dest-uuid": "group_id", - "type": "used-by" - }, - "associated_software": { - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "groups": { - "description": "description", - "meta": { - "source": "source", - "group-attack-id": "group_attack_id", - "country": {"extract": "single", "key": "country", "subkey": "country_code"}, - "observed_country": {"extract": "multiple", "key": "observed_country", "subkey": "country_code"}, - "observed_motivation": {"extract": "multiple", "key": "observed_motivation", "subkey": "name"}, - "target-category": {"extract": "multiple", "key": "observed_sector", "subkey": "name"}, - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "associated_groups": { - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "campaigns": { - "description": "description", - "meta": { - "source": "source", - "campaign-attack-id": "campaign_attack_id", - "first_seen": "first_seen", - "last_seen": "last_seen", - "tags": "tags", - "owner": "owner_name" - }, - "related": {}, - "uuid": "id", - "value": "name" - } - -} +UUIDS = config['UUIDS'] +GALAXY_CONFIGS = config['GALAXY_CONFIGS'] +CLUSTER_CONFIGS = config['CLUSTER_CONFIGS'] +VALUE_FIELDS = config['VALUE_FIELDS'] def create_cluster_values(data, cluster): value_fields = VALUE_FIELDS[cluster.internal_type] @@ -162,6 +45,8 @@ def create_metadata(data, format): metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"]) elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]): metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])] + elif meta_value.get("extract") == "reverse" and data.get(meta_value["key"]): + metadata[meta_key] = [data.get(meta_value["key"])] elif data.get(meta_value): metadata[meta_key] = data.get(meta_value) return metadata @@ -191,7 +76,7 @@ def create_galaxy_and_cluster(galaxy_type, version): create_cluster_values(data, cluster) cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") - print(f"Galaxy {galaxy_type} created") + print(f"Galaxy tidal-{galaxy_type} created") def create_galaxy(args): if args.all: diff --git a/tools/tidal-api/utils/config.py b/tools/tidal-api/utils/config.py new file mode 100644 index 0000000..d69e997 --- /dev/null +++ b/tools/tidal-api/utils/config.py @@ -0,0 +1,14 @@ +import json + +def load_config(file_path): + with open(file_path, 'r') as file: + config = json.load(file) + return link_uuids(config) + +def link_uuids(config): + uuids = config["UUIDS"] + for key, galaxy_config in config["GALAXY_CONFIGS"].items(): + galaxy_config["uuid"] = uuids[key] + for key, cluster_config in config["CLUSTER_CONFIGS"].items(): + cluster_config["uuid"] = uuids[key] + return config \ No newline at end of file From 9d2dfba0b996c843ff8edb98507f957f94efc00f Mon Sep 17 00:00:00 2001 From: niclas Date: Thu, 22 Feb 2024 10:52:23 +0100 Subject: [PATCH 05/10] Fix [config] metadata mapping --- tools/tidal-api/README.md | 8 ++--- tools/tidal-api/config.json | 24 ++++++++++++--- tools/tidal-api/main.py | 60 ++++++++++++++++++++++++++----------- 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md index 8e8b992..e0412d8 100644 --- a/tools/tidal-api/README.md +++ b/tools/tidal-api/README.md @@ -45,9 +45,9 @@ The configuration file is located in `config.json` and maps the fields of the Ti The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this: ```json { - "extract": , - "key": , - "subkey": + "extract": "", + "key": "", + "subkey": "" } ``` **Extract modes**: @@ -64,4 +64,4 @@ To build all galaxies and clusters, run the following command: ```bash python3 main.py create-galaxy -v --all -``` \ No newline at end of file +``` diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json index dd50dbe..9d2010b 100644 --- a/tools/tidal-api/config.json +++ b/tools/tidal-api/config.json @@ -120,8 +120,16 @@ "source": "source", "type": "type", "software-attack-id": "software_attack_id", - "platforms": "platforms", - "tags": "tags", + "platforms": { + "extract": "multiple", + "key": "platforms", + "subkey": "name" + }, + "tags": { + "extract": "multiple", + "key": "tags", + "subkey": "tag" + }, "owner": "owner_name" }, "related": { @@ -192,8 +200,16 @@ "description": "description", "meta": { "source": "source", - "platforms": "platforms", - "tags": "tags", + "platforms": { + "extract": "multiple", + "key": "platforms", + "subkey": "name" + }, + "tags": { + "extract": "multiple", + "key": "tags", + "subkey": "tag" + }, "owner": "owner_name" }, "related": { diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 008f171..98b29d7 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -8,12 +8,13 @@ import argparse CLUSTER_PATH = "../../clusters/" GALAXY_PATH = "../../galaxies/" -config = load_config('./config.json') +config = load_config("./config.json") + +UUIDS = config["UUIDS"] +GALAXY_CONFIGS = config["GALAXY_CONFIGS"] +CLUSTER_CONFIGS = config["CLUSTER_CONFIGS"] +VALUE_FIELDS = config["VALUE_FIELDS"] -UUIDS = config['UUIDS'] -GALAXY_CONFIGS = config['GALAXY_CONFIGS'] -CLUSTER_CONFIGS = config['CLUSTER_CONFIGS'] -VALUE_FIELDS = config['VALUE_FIELDS'] def create_cluster_values(data, cluster): value_fields = VALUE_FIELDS[cluster.internal_type] @@ -34,23 +35,33 @@ def create_cluster_values(data, cluster): case "value": values[key] = entry.get(value) case _: - print(f"Error: Invalid configuration for {key} in {cluster.internal_type} value fields.") + print( + f"Error: Invalid configuration for {key} in {cluster.internal_type} value fields." + ) cluster.add_value(values) + def create_metadata(data, format): metadata = {} for meta_key, meta_value in format.items(): if isinstance(meta_value, dict): if meta_value.get("extract") == "single" and data.get(meta_value["key"]): - metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"]) - elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]): - metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])] + metadata[meta_key] = data.get(meta_value["key"])[0].get( + meta_value["subkey"] + ) + elif meta_value.get("extract") == "multiple" and data.get( + meta_value["key"] + ): + metadata[meta_key] = [ + entry.get(meta_value["subkey"]) + for entry in data.get(meta_value["key"]) + ] elif meta_value.get("extract") == "reverse" and data.get(meta_value["key"]): metadata[meta_key] = [data.get(meta_value["key"])] elif data.get(meta_value): metadata[meta_key] = data.get(meta_value) return metadata - + def create_relations(data, format): relations = [] @@ -64,7 +75,7 @@ def create_relations(data, format): relation_entry[relation_key] = relation_value relations.append(relation_entry) return relations - + def create_galaxy_and_cluster(galaxy_type, version): api = TidalAPI() @@ -78,6 +89,7 @@ def create_galaxy_and_cluster(galaxy_type, version): print(f"Galaxy tidal-{galaxy_type} created") + def create_galaxy(args): if args.all: for galaxy_type in GALAXY_CONFIGS: @@ -85,19 +97,31 @@ def create_galaxy(args): else: create_galaxy_and_cluster(args.type, args.version) + if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Create a galaxy and cluster for Tidal API") + parser = argparse.ArgumentParser( + description="Create a galaxy and cluster for Tidal API" + ) subparsers = parser.add_subparsers(dest="command") - galaxy_parser = subparsers.add_parser("create_galaxy", help="Create a galaxy from the Tidal API") - galaxy_parser.add_argument("--type", choices=list(GALAXY_CONFIGS.keys()) + ['all'], help="The type of the galaxy") - galaxy_parser.add_argument("-v", "--version", type=int, required=True, help="The version of the galaxy") - galaxy_parser.add_argument("--all", action="store_true", help="Flag to create all predefined galaxy types") + galaxy_parser = subparsers.add_parser( + "create_galaxy", help="Create a galaxy from the Tidal API" + ) + galaxy_parser.add_argument( + "--type", + choices=list(GALAXY_CONFIGS.keys()) + ["all"], + help="The type of the galaxy", + ) + galaxy_parser.add_argument( + "-v", "--version", type=int, required=True, help="The version of the galaxy" + ) + galaxy_parser.add_argument( + "--all", action="store_true", help="Flag to create all predefined galaxy types" + ) galaxy_parser.set_defaults(func=create_galaxy) args = parser.parse_args() - if hasattr(args, 'func'): + if hasattr(args, "func"): args.func(args) else: parser.print_help() - From 9467e101bf95d058385b1f5a0ed385957756811e Mon Sep 17 00:00:00 2001 From: niclas Date: Thu, 22 Feb 2024 12:12:31 +0100 Subject: [PATCH 06/10] Add [config] optional "private" relations --- tools/tidal-api/README.md | 19 ++++++++++++++++++- tools/tidal-api/config.json | 6 ++++++ tools/tidal-api/main.py | 21 ++++++++++++++------- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md index e0412d8..3a34cd0 100644 --- a/tools/tidal-api/README.md +++ b/tools/tidal-api/README.md @@ -42,7 +42,7 @@ The configuration file is located in `config.json` and maps the fields of the Ti >Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration. ### Extraction Configuration -The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this: +The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in an object in the API response. The extraction configuration looks like this: ```json { "extract": "", @@ -56,6 +56,23 @@ The extraction configuration is a dictionary that maps the fields of the Tidal C - `multiple`: Extracts multiple values from the API response - `reverse`: Gets the value of the key and writes it into an array (no subkey needed) +### "Private" Relations +The Tidal Cyber API provides relations between different objects. Some of these relations point to objects that are not part of the galaxies created based on the API response nor are they part of the MISP galaxy. These relations can be marked as `private` in the config file. For example: +```json + "related": { + "tactic": { + "mode": "public", + "dest-uuid": "tactic_id", + "type": "uses" + }, + "sub_technique": { + "mode": "private", + "dest-uuid": "id", + "type": "sub-technique-of" + } + }, +``` + ## Usage ```bash python3 main.py create-galaxy -v --type diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json index 9d2010b..2aaf32b 100644 --- a/tools/tidal-api/config.json +++ b/tools/tidal-api/config.json @@ -134,10 +134,12 @@ }, "related": { "groups": { + "mode": "private", "dest-uuid": "group_id", "type": "used-by" }, "associated_software": { + "mode": "private", "dest-uuid": "id", "type": "related-to" } @@ -175,6 +177,7 @@ }, "related": { "associated_groups": { + "mode": "private", "dest-uuid": "id", "type": "related-to" } @@ -214,10 +217,12 @@ }, "related": { "tactic": { + "mode": "public", "dest-uuid": "tactic_id", "type": "uses" }, "sub_technique": { + "mode": "private", "dest-uuid": "id", "type": "sub-technique-of" } @@ -236,6 +241,7 @@ }, "related": { "techniques": { + "mode": "public", "dest-uuid": "technique_id", "type": "uses" } diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 98b29d7..f2e248f 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -16,7 +16,7 @@ CLUSTER_CONFIGS = config["CLUSTER_CONFIGS"] VALUE_FIELDS = config["VALUE_FIELDS"] -def create_cluster_values(data, cluster): +def create_cluster_values(data, cluster, add_private): value_fields = VALUE_FIELDS[cluster.internal_type] for entry in data["data"]: values = {} @@ -28,7 +28,7 @@ def create_cluster_values(data, cluster): metadata = create_metadata(entry, value) values["meta"] = metadata case "related": - relations = create_relations(entry, value) + relations = create_relations(entry, value, add_private) values["related"] = relations case "uuid": values[key] = entry.get(value) @@ -63,13 +63,17 @@ def create_metadata(data, format): return metadata -def create_relations(data, format): +def create_relations(data, format, add_private): relations = [] for i in range(len(list(format))): for relation in data[list(format)[i]]: + if not add_private and list(format.values())[i].get("mode") == "private": + continue relation_entry = {} for relation_key, relation_value in list(format.values())[i].items(): if relation_key != "type": + if relation_key == "mode": + continue relation_entry[relation_key] = relation.get(relation_value) else: relation_entry[relation_key] = relation_value @@ -77,14 +81,14 @@ def create_relations(data, format): return relations -def create_galaxy_and_cluster(galaxy_type, version): +def create_galaxy_and_cluster(galaxy_type, version, add_private=False): api = TidalAPI() galaxy = Galaxy(**GALAXY_CONFIGS[galaxy_type], version=version) galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{galaxy_type}.json") cluster = Cluster(**CLUSTER_CONFIGS[galaxy_type], internal_type=galaxy_type) data = api.get_data(galaxy_type) - create_cluster_values(data, cluster) + create_cluster_values(data, cluster, add_private) cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") print(f"Galaxy tidal-{galaxy_type} created") @@ -93,9 +97,9 @@ def create_galaxy_and_cluster(galaxy_type, version): def create_galaxy(args): if args.all: for galaxy_type in GALAXY_CONFIGS: - create_galaxy_and_cluster(galaxy_type, args.version) + create_galaxy_and_cluster(galaxy_type, args.version, args.addprivate) else: - create_galaxy_and_cluster(args.type, args.version) + create_galaxy_and_cluster(args.type, args.version, args.addprivate) if __name__ == "__main__": @@ -118,6 +122,9 @@ if __name__ == "__main__": galaxy_parser.add_argument( "--all", action="store_true", help="Flag to create all predefined galaxy types" ) + galaxy_parser.add_argument( + "--addprivate", action="store_true", help="Flag to add private relations" + ) galaxy_parser.set_defaults(func=create_galaxy) args = parser.parse_args() From 35b8192208093ea5827d440a50ae6c7a5c736955 Mon Sep 17 00:00:00 2001 From: niclas Date: Fri, 23 Feb 2024 11:14:00 +0100 Subject: [PATCH 07/10] refactor [tool] code --- tools/tidal-api/README.md | 84 -------- tools/tidal-api/config.json | 271 ------------------------ tools/tidal-api/config/campaigns.json | 17 ++ tools/tidal-api/config/groups.json | 17 ++ tools/tidal-api/config/references.json | 17 ++ tools/tidal-api/config/software.json | 17 ++ tools/tidal-api/config/tactic.json | 17 ++ tools/tidal-api/config/technique.json | 17 ++ tools/tidal-api/main.py | 179 ++++++---------- tools/tidal-api/models/cluster.py | 273 ++++++++++++++++++++++++- tools/tidal-api/models/galaxy.py | 19 +- tools/tidal-api/utils/__init__.py | 0 tools/tidal-api/utils/config.py | 14 -- tools/tidal-api/utils/extractor.py | 6 - 14 files changed, 444 insertions(+), 504 deletions(-) delete mode 100644 tools/tidal-api/README.md delete mode 100644 tools/tidal-api/config.json create mode 100644 tools/tidal-api/config/campaigns.json create mode 100644 tools/tidal-api/config/groups.json create mode 100644 tools/tidal-api/config/references.json create mode 100644 tools/tidal-api/config/software.json create mode 100644 tools/tidal-api/config/tactic.json create mode 100644 tools/tidal-api/config/technique.json delete mode 100644 tools/tidal-api/utils/__init__.py delete mode 100644 tools/tidal-api/utils/config.py delete mode 100644 tools/tidal-api/utils/extractor.py diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md deleted file mode 100644 index 3a34cd0..0000000 --- a/tools/tidal-api/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Tidal Cyber API - -This is a tool generating MISP galaxies and clusters from Tidal Cyber API. - -## Endpoints -https://app-api.tidalcyber.com/api/v1/technique - -https://app-api.tidalcyber.com/api/v1/references - -https://app-api.tidalcyber.com/api/v1/tactic - -https://app-api.tidalcyber.com/api/v1/campaigns/ - -https://app-api.tidalcyber.com/api/v1/software/ - -https://app-api.tidalcyber.com/api/v1/groups/ - -## Configuration -The configuration file is located in `config.json` and maps the fields of the Tidal API to the Galaxy and Cluster fields. It consists of the following sections: -- `UUID`: The UUID of the galaxy to be created -- `GALAXY_CONFIGS`: The configuration of the galaxies to be created in the `galaxies` folder of the MISP-galaxy repository - - `name`: The name of the galaxy - - `namespace`: The namespace of the galaxy - - `description`: The description of the galaxy - - `type`: The type of the galaxy - - `uuid`: The UUID of the galaxy (will be inserted from the `UUID` section) -- `CLUSTER_CONFIGS`: The configuration of the clusters to be created in the `clusters` folder of the MISP-galaxy repository - - `authors`: The authors of the cluster - - `category`: The category of the cluster - - `description`: The description of the cluster - - `name`: The name of the cluster - - `source`: The source of the cluster - - `type`: The type of the cluster - - `uuid`: The UUID of the cluster (will be inserted from the `UUID` section) - - `values`: The values of the cluster (will be inserted from the `VALUE_FIELDS` section) -- `VALUE_FIELDS`: Defines the mapping of the fields in the Tidal Cyber API to the fields in the MISP cluster values array - - `description`: The description of the cluster value - - `meta`: The metadata of the cluster value - - `related`: The related cluster values of the cluster value (you can define a `type` for each relation type in the config which will not be mapped to a field of the API) - - `uuid`: The UUID of the cluster value - - `value`: The value of the cluster value - >Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration. - -### Extraction Configuration -The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in an object in the API response. The extraction configuration looks like this: -```json -{ - "extract": "", - "key": "", - "subkey": "" -} -``` -**Extract modes**: - -- `single`: Extracts a single value from the API response -- `multiple`: Extracts multiple values from the API response -- `reverse`: Gets the value of the key and writes it into an array (no subkey needed) - -### "Private" Relations -The Tidal Cyber API provides relations between different objects. Some of these relations point to objects that are not part of the galaxies created based on the API response nor are they part of the MISP galaxy. These relations can be marked as `private` in the config file. For example: -```json - "related": { - "tactic": { - "mode": "public", - "dest-uuid": "tactic_id", - "type": "uses" - }, - "sub_technique": { - "mode": "private", - "dest-uuid": "id", - "type": "sub-technique-of" - } - }, -``` - -## Usage -```bash -python3 main.py create-galaxy -v --type -``` -To build all galaxies and clusters, run the following command: - -```bash -python3 main.py create-galaxy -v --all -``` diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json deleted file mode 100644 index 2aaf32b..0000000 --- a/tools/tidal-api/config.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "UUIDS": { - "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", - "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", - "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", - "technique": "3e28b683-8159-4398-8729-7248eac9aa45", - "tactic": "16a396e2-a4a9-4dfd-be0a-6ba75fb5382c", - "references": "cf5e180f-26e9-42b2-b10c-c9e55f448750" - }, - "GALAXY_CONFIGS": { - "software": { - "name": "Tidal Software", - "namespace": "tidal", - "description": "Tidal Software Galaxy", - "type": "software", - "uuid": "" - }, - "groups": { - "name": "Tidal Groups", - "namespace": "tidal", - "description": "Tidal Groups Galaxy", - "type": "groups", - "uuid": "" - }, - "campaigns": { - "name": "Tidal Campaigns", - "namespace": "tidal", - "description": "Tidal Campaigns Galaxy", - "type": "campaigns", - "uuid": "" - }, - "technique": { - "name": "Tidal Technique", - "namespace": "tidal", - "description": "Tidal Technique Galaxy", - "type": "technique", - "uuid": "" - }, - "tactic": { - "name": "Tidal Tactic", - "namespace": "tidal", - "description": "Tidal Tactic Galaxy", - "type": "tactics", - "uuid": "" - }, - "references": { - "name": "Tidal References", - "namespace": "tidal", - "description": "Tidal References Galaxy", - "type": "references", - "uuid": "" - } - }, - "CLUSTER_CONFIGS": { - "software": { - "authors": "Tidal", - "category": "Software", - "description": "Tidal Software Cluster", - "name": "Tidal Software", - "source": "Tidal", - "type": "software", - "uuid": "", - "values": [] - }, - "groups": { - "authors": "Tidal", - "category": "Threat Groups", - "description": "Tidal Threat Groups Cluster", - "name": "Tidal Threat Groups", - "source": "Tidal", - "type": "groups", - "uuid": "", - "values": [] - }, - "campaigns": { - "authors": "Tidal", - "category": "Campaigns", - "description": "Tidal Campaigns Cluster", - "name": "Tidal Campaigns", - "source": "Tidal", - "type": "campaigns", - "uuid": "", - "values": [] - }, - "technique": { - "authors": "Tidal", - "category": "Techniques", - "description": "Tidal Techniques Cluster", - "name": "Tidal Techniques", - "source": "Tidal", - "type": "technique", - "uuid": "", - "values": [] - }, - "tactic": { - "authors": "Tidal", - "category": "Tactics", - "description": "Tidal Tactics Cluster", - "name": "Tidal Tactics", - "source": "Tidal", - "type": "tactic", - "uuid": "", - "values": [] - }, - "references": { - "authors": "Tidal", - "category": "References", - "description": "Tidal References Cluster", - "name": "Tidal References", - "source": "Tidal", - "type": "references", - "uuid": "", - "values": [] - } - }, - "VALUE_FIELDS": { - "software": { - "description": "description", - "meta": { - "source": "source", - "type": "type", - "software-attack-id": "software_attack_id", - "platforms": { - "extract": "multiple", - "key": "platforms", - "subkey": "name" - }, - "tags": { - "extract": "multiple", - "key": "tags", - "subkey": "tag" - }, - "owner": "owner_name" - }, - "related": { - "groups": { - "mode": "private", - "dest-uuid": "group_id", - "type": "used-by" - }, - "associated_software": { - "mode": "private", - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "groups": { - "description": "description", - "meta": { - "source": "source", - "group-attack-id": "group_attack_id", - "country": { - "extract": "single", - "key": "country", - "subkey": "country_code" - }, - "observed_country": { - "extract": "multiple", - "key": "observed_country", - "subkey": "country_code" - }, - "observed_motivation": { - "extract": "multiple", - "key": "observed_motivation", - "subkey": "name" - }, - "target-category": { - "extract": "multiple", - "key": "observed_sector", - "subkey": "name" - }, - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "associated_groups": { - "mode": "private", - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "campaigns": { - "description": "description", - "meta": { - "source": "source", - "campaign-attack-id": "campaign_attack_id", - "first_seen": "first_seen", - "last_seen": "last_seen", - "tags": "tags", - "owner": "owner_name" - }, - "related": {}, - "uuid": "id", - "value": "name" - }, - "technique": { - "description": "description", - "meta": { - "source": "source", - "platforms": { - "extract": "multiple", - "key": "platforms", - "subkey": "name" - }, - "tags": { - "extract": "multiple", - "key": "tags", - "subkey": "tag" - }, - "owner": "owner_name" - }, - "related": { - "tactic": { - "mode": "public", - "dest-uuid": "tactic_id", - "type": "uses" - }, - "sub_technique": { - "mode": "private", - "dest-uuid": "id", - "type": "sub-technique-of" - } - }, - "uuid": "id", - "value": "name" - }, - "tactic": { - "description": "description", - "meta": { - "source": "source", - "tactic-attack-id": "tactic_attack_id", - "ordinal_position": "ordinal_position", - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "techniques": { - "mode": "public", - "dest-uuid": "technique_id", - "type": "uses" - } - }, - "uuid": "id", - "value": "name" - }, - "references": { - "description": "description", - "meta": { - "source": "source", - "refs": { - "extract": "reverse", - "key": "url" - }, - "owner": "owner_name", - "title": "title", - "author": "authors", - "date_accessed": "date_accessed", - "date_published": "date_published" - }, - "related": {}, - "uuid": "id", - "value": "name" - } - } -} \ No newline at end of file diff --git a/tools/tidal-api/config/campaigns.json b/tools/tidal-api/config/campaigns.json new file mode 100644 index 0000000..8463932 --- /dev/null +++ b/tools/tidal-api/config/campaigns.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal Campaigns", + "namespace": "tidal", + "description": "Tidal Campaigns Galaxy", + "type": "campaigns", + "uuid": "43a8fce6-08d3-46c2-957d-53606efe2c48" + }, + "cluster": { + "authors": "Tidal", + "category": "Campaigns", + "description": "Tidal Campaigns Cluster", + "name": "Tidal Campaigns", + "source": "Tidal", + "type": "campaigns" + } +} \ No newline at end of file diff --git a/tools/tidal-api/config/groups.json b/tools/tidal-api/config/groups.json new file mode 100644 index 0000000..152b33c --- /dev/null +++ b/tools/tidal-api/config/groups.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal Groups", + "namespace": "tidal", + "description": "Tidal Groups Galaxy", + "type": "groups", + "uuid": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6" + }, + "cluster": { + "authors": "Tidal", + "category": "Threat Groups", + "description": "Tidal Threat Groups Cluster", + "name": "Tidal Threat Groups", + "source": "Tidal", + "type": "groups" + } +} \ No newline at end of file diff --git a/tools/tidal-api/config/references.json b/tools/tidal-api/config/references.json new file mode 100644 index 0000000..697861e --- /dev/null +++ b/tools/tidal-api/config/references.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal References", + "namespace": "tidal", + "description": "Tidal References Galaxy", + "type": "references", + "uuid": "43a8fce6-08d3-46c2-957d-53606efe2c48" + }, + "cluster": { + "authors": "Tidal", + "category": "References", + "description": "Tidal References Cluster", + "name": "Tidal References", + "source": "Tidal", + "type": "references" + } +} \ No newline at end of file diff --git a/tools/tidal-api/config/software.json b/tools/tidal-api/config/software.json new file mode 100644 index 0000000..58bd485 --- /dev/null +++ b/tools/tidal-api/config/software.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal Software", + "namespace": "tidal", + "description": "Tidal Software Galaxy", + "type": "software", + "uuid": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7" + }, + "cluster": { + "authors": "Tidal", + "category": "Software", + "description": "Tidal Software Cluster", + "name": "Tidal Software", + "source": "Tidal", + "type": "software" + } +} \ No newline at end of file diff --git a/tools/tidal-api/config/tactic.json b/tools/tidal-api/config/tactic.json new file mode 100644 index 0000000..31d592c --- /dev/null +++ b/tools/tidal-api/config/tactic.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal Tactic", + "namespace": "tidal", + "description": "Tidal Tactic Galaxy", + "type": "tactic", + "uuid": "43a8fce6-08d3-46c2-957d-53606efe2c48" + }, + "cluster": { + "authors": "Tidal", + "category": "Tactic", + "description": "Tidal Tactic Cluster", + "name": "Tidal Tactic", + "source": "Tidal", + "type": "tactic" + } +} \ No newline at end of file diff --git a/tools/tidal-api/config/technique.json b/tools/tidal-api/config/technique.json new file mode 100644 index 0000000..4f72c96 --- /dev/null +++ b/tools/tidal-api/config/technique.json @@ -0,0 +1,17 @@ +{ + "galaxy": { + "name": "Tidal Technique", + "namespace": "tidal", + "description": "Tidal Technique Galaxy", + "type": "technique", + "uuid": "43a8fce6-08d3-46c2-957d-53606efe2c48" + }, + "cluster": { + "authors": "Tidal", + "category": "Technique", + "description": "Tidal Technique Cluster", + "name": "Tidal Technique", + "source": "Tidal", + "type": "technique" + } +} \ No newline at end of file diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index f2e248f..d402dbb 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -1,134 +1,89 @@ from api.api import TidalAPI from models.galaxy import Galaxy -from models.cluster import Cluster -from utils.extractor import extract_links -from utils.config import load_config +from models.cluster import GroupCluster, SoftwareCluster, CampaignsCluster, TechniqueCluster, TacticCluster, ReferencesCluster import argparse +import json +import os -CLUSTER_PATH = "../../clusters/" -GALAXY_PATH = "../../galaxies/" +CONFIG = "./config" +GALAXY_PATH = "../../galaxies" +CLUSTER_PATH = "../../clusters" -config = load_config("./config.json") - -UUIDS = config["UUIDS"] -GALAXY_CONFIGS = config["GALAXY_CONFIGS"] -CLUSTER_CONFIGS = config["CLUSTER_CONFIGS"] -VALUE_FIELDS = config["VALUE_FIELDS"] - - -def create_cluster_values(data, cluster, add_private): - value_fields = VALUE_FIELDS[cluster.internal_type] - for entry in data["data"]: - values = {} - for key, value in value_fields.items(): - match key: - case "description": - values[value] = entry.get(key) - case "meta": - metadata = create_metadata(entry, value) - values["meta"] = metadata - case "related": - relations = create_relations(entry, value, add_private) - values["related"] = relations - case "uuid": - values[key] = entry.get(value) - case "value": - values[key] = entry.get(value) - case _: - print( - f"Error: Invalid configuration for {key} in {cluster.internal_type} value fields." - ) - cluster.add_value(values) - - -def create_metadata(data, format): - metadata = {} - for meta_key, meta_value in format.items(): - if isinstance(meta_value, dict): - if meta_value.get("extract") == "single" and data.get(meta_value["key"]): - metadata[meta_key] = data.get(meta_value["key"])[0].get( - meta_value["subkey"] - ) - elif meta_value.get("extract") == "multiple" and data.get( - meta_value["key"] - ): - metadata[meta_key] = [ - entry.get(meta_value["subkey"]) - for entry in data.get(meta_value["key"]) - ] - elif meta_value.get("extract") == "reverse" and data.get(meta_value["key"]): - metadata[meta_key] = [data.get(meta_value["key"])] - elif data.get(meta_value): - metadata[meta_key] = data.get(meta_value) - return metadata - - -def create_relations(data, format, add_private): - relations = [] - for i in range(len(list(format))): - for relation in data[list(format)[i]]: - if not add_private and list(format.values())[i].get("mode") == "private": - continue - relation_entry = {} - for relation_key, relation_value in list(format.values())[i].items(): - if relation_key != "type": - if relation_key == "mode": - continue - relation_entry[relation_key] = relation.get(relation_value) - else: - relation_entry[relation_key] = relation_value - relations.append(relation_entry) - return relations - - -def create_galaxy_and_cluster(galaxy_type, version, add_private=False): +def create_galaxy(endpoint: str, version: int): api = TidalAPI() - galaxy = Galaxy(**GALAXY_CONFIGS[galaxy_type], version=version) - galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{galaxy_type}.json") + data = api.get_data(endpoint) + with open(f"{CONFIG}/{endpoint}.json", "r") as file: + config = json.load(file) - cluster = Cluster(**CLUSTER_CONFIGS[galaxy_type], internal_type=galaxy_type) - data = api.get_data(galaxy_type) - create_cluster_values(data, cluster, add_private) - cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") + galaxy = Galaxy(**config["galaxy"], version=version) + galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{endpoint}.json") + + match endpoint: + case "groups": + cluster = GroupCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case "software": + cluster = SoftwareCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case "campaigns": + cluster = CampaignsCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case "technique": + cluster = TechniqueCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case "tactic": + cluster = TacticCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case "references": + cluster = ReferencesCluster(**config["cluster"], uuid=galaxy.uuid) + cluster.add_values(data) + case _: + print("Error: Invalid endpoint") + return - print(f"Galaxy tidal-{galaxy_type} created") + cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{endpoint}.json") + print(f"Galaxy tidal-{endpoint} created") - -def create_galaxy(args): +def main(args, galaxies): if args.all: - for galaxy_type in GALAXY_CONFIGS: - create_galaxy_and_cluster(galaxy_type, args.version, args.addprivate) + for galaxy in galaxies: + create_galaxy(galaxy, args.version) else: - create_galaxy_and_cluster(args.type, args.version, args.addprivate) - + create_galaxy(args.type, args.version) + if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Create a galaxy and cluster for Tidal API" - ) - subparsers = parser.add_subparsers(dest="command") - galaxy_parser = subparsers.add_parser( - "create_galaxy", help="Create a galaxy from the Tidal API" + galaxies = [] + for f in os.listdir(CONFIG): + if f.endswith(".json"): + galaxies.append(f.split(".")[0]) + + + parser = argparse.ArgumentParser( + description="Create galaxy and cluster json files from Tidal API" ) - galaxy_parser.add_argument( + parser.add_argument( + "--all", + action="store_true", + help="Create all galaxies and clusters", + ) + parser.add_argument( "--type", - choices=list(GALAXY_CONFIGS.keys()) + ["all"], - help="The type of the galaxy", + choices=galaxies, + help="The type of the file to create", ) - galaxy_parser.add_argument( - "-v", "--version", type=int, required=True, help="The version of the galaxy" + parser.add_argument( + "-v", + "--version", + type=int, + required=True, + help="The version of the galaxy", ) - galaxy_parser.add_argument( - "--all", action="store_true", help="Flag to create all predefined galaxy types" - ) - galaxy_parser.add_argument( - "--addprivate", action="store_true", help="Flag to add private relations" - ) - galaxy_parser.set_defaults(func=create_galaxy) + parser.set_defaults(func=main) args = parser.parse_args() if hasattr(args, "func"): - args.func(args) + args.func(args, galaxies=galaxies) else: - parser.print_help() + parser.print_help() \ No newline at end of file diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py index 2436b89..4c75a98 100644 --- a/tools/tidal-api/models/cluster.py +++ b/tools/tidal-api/models/cluster.py @@ -1,7 +1,79 @@ +from dataclasses import dataclass, field, asdict import json +@dataclass +class Meta: + pass + +@dataclass +class GroupsMeta(): + source: str = None + group_attack_id: str = None + country: str = None + observed_countries: list = None + observed_motivations: list = None + target_categories: list = None + tags: list = None + owner: str = None + +@dataclass +class SoftwareMeta(): + source: str = None + type: str = None + software_attack_id: str = None + platforms: list = None + tags: list = None + owner: str = None + +@dataclass +class TechniqueMeta(): + source: str = None + platforms: list = None + tags: list = None + owner: str = None + +@dataclass +class TacticMeta(): + source: str = None + tactic_attack_id: str = None + ordinal_position: int = None + tags: list = None + owner: str = None + +@dataclass +class ReferencesMeta(): + source: str = None + refs: list = None + title: str = None + author: str = None + date_accessed: str = None + date_published: str = None + owner: str = None + +@dataclass +class CampaignsMeta(): + source: str = None + campaign_attack_id: str = None + first_seen: str = None + last_seen: str = None + tags: list = None + owner: str = None + +@dataclass +class ClusterValue: + description: str = "" + meta: Meta = field(default_factory=Meta) + related: list = field(default_factory=list) + uuid: str = "" + value: str = "" + + def return_value(self): + value_dict = asdict(self) + value_dict['meta'] = {k: v for k, v in asdict(self.meta).items() if v is not None} + return value_dict + class Cluster(): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str, values: list, internal_type: str): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): self.authors = authors self.category = category self.description = description @@ -9,15 +81,200 @@ class Cluster(): self.source = source self.type = type self.uuid = uuid - self.values = values - self.internal_type = internal_type + self.values = [] - def add_value(self, value): - self.values.append(value) + def add_values(self): + print("This method should be implemented in the child class") def save_to_file(self, path): with open(path, "w") as file: - file.write(json.dumps(self.__dict__, indent=4)) + file.write(json.dumps(self.__dict__(), indent=4)) - def get_config(self): - return self.__dict__ + def __str__(self) -> str: + return f"Cluster: {self.name} - {self.type} - {self.uuid}" + + def __dict__(self) -> dict: + return { + "authors": self.authors, + "category": self.category, + "description": self.description, + "name": self.name, + "source": self.source, + "type": self.type, + "uuid": self.uuid, + "values": self.values, + } + +class GroupCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = GroupsMeta( + source=entry.get("source"), + group_attack_id=entry.get("group_attack_id"), + country=entry.get("country")[0].get("country_code") if entry.get("country") else None, + observed_countries=[x.get("country_code") for x in entry.get("observed_country")], + observed_motivations=[x.get("name") for x in entry.get("observed_motivation")], + target_categories=[x.get("name") for x in entry.get("observed_sector")], + tags=[x.get("tag") for x in entry.get("tags")], + owner=entry.get("owner_name"), + ) + related = [] + for relation in entry.get("associated_groups"): + related.append({ + "dest-uuid": relation.get("id"), + "type": "related-to", + } + ) + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=related, + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) + + +class SoftwareCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = SoftwareMeta( + source=entry.get("source"), + type=entry.get("type"), + software_attack_id=entry.get("software_attack_id"), + platforms=[x.get("name") for x in entry.get("platforms")], + tags=[x.get("tag") for x in entry.get("tags")], + owner=entry.get("owner_name"), + ) + related = [] + for relation in entry.get("groups"): + related.append({ + "dest-uuid": relation.get("group_id"), + "type": "used-by", + } + ) + for relation in entry.get("associated_software"): + related.append({ + "dest-uuid": relation.get("id"), + "type": "related-to", + } + ) + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=related, + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) + +class TechniqueCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = TechniqueMeta( + source=entry.get("source"), + platforms=[x.get("name") for x in entry.get("platforms")], + tags=[x.get("tag") for x in entry.get("tags")], + owner=entry.get("owner_name"), + ) + related = [] + for relation in entry.get("tactic"): + related.append({ + "dest-uuid": relation.get("tactic_id"), + "type": "uses", + } + ) + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=related, + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) + +class TacticCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = TacticMeta( + source=entry.get("source"), + tactic_attack_id=entry.get("tactic_attack_id"), + ordinal_position=entry.get("ordinal_position"), + tags=[x.get("tag") for x in entry.get("tags")], + owner=entry.get("owner_name"), + ) + related = [] + for relation in entry.get("techniques"): + related.append({ + "dest-uuid": relation.get("technique_id"), + "type": "uses", + } + ) + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=related, + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) + +class ReferencesCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = ReferencesMeta( + source=entry.get("source"), + refs=[entry.get("url")], + title=entry.get("title"), + author=entry.get("author"), + date_accessed=entry.get("date_accessed"), + date_published=entry.get("date_published"), + owner=entry.get("owner_name"), + ) + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=[], + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) + +class CampaignsCluster(Cluster): + def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + super().__init__(authors, category, description, name, source, type, uuid) + + def add_values(self, data): + for entry in data["data"]: + meta = CampaignsMeta( + source=entry.get("source"), + campaign_attack_id=entry.get("campaign_attack_id"), + first_seen=entry.get("first_seen"), + last_seen=entry.get("last_seen"), + tags=[x.get("tag") for x in entry.get("tags")], + owner=entry.get("owner_name"), + ) + related = [] + value = ClusterValue( + description=entry.get("description"), + meta=meta, + related=related, + uuid=entry.get("id"), + value=entry.get("name"), + ) + self.values.append(value.return_value()) \ No newline at end of file diff --git a/tools/tidal-api/models/galaxy.py b/tools/tidal-api/models/galaxy.py index 2de73b2..9b091e4 100644 --- a/tools/tidal-api/models/galaxy.py +++ b/tools/tidal-api/models/galaxy.py @@ -1,14 +1,15 @@ import json +from dataclasses import dataclass, asdict +@dataclass class Galaxy(): - def __init__(self, description, name, namespace, type, uuid, version): - self.description = description - self.name = name - self.namespace = namespace - self.type = type - self.uuid = uuid - self.version = version + description: str + name: str + namespace: str + type: str + uuid: str + version: str - def save_to_file(self, path): + def save_to_file(self, path: str): with open(path, "w") as file: - file.write(json.dumps(self.__dict__, indent=4)) + file.write(json.dumps(asdict(self), indent=4)) \ No newline at end of file diff --git a/tools/tidal-api/utils/__init__.py b/tools/tidal-api/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/tidal-api/utils/config.py b/tools/tidal-api/utils/config.py deleted file mode 100644 index d69e997..0000000 --- a/tools/tidal-api/utils/config.py +++ /dev/null @@ -1,14 +0,0 @@ -import json - -def load_config(file_path): - with open(file_path, 'r') as file: - config = json.load(file) - return link_uuids(config) - -def link_uuids(config): - uuids = config["UUIDS"] - for key, galaxy_config in config["GALAXY_CONFIGS"].items(): - galaxy_config["uuid"] = uuids[key] - for key, cluster_config in config["CLUSTER_CONFIGS"].items(): - cluster_config["uuid"] = uuids[key] - return config \ No newline at end of file diff --git a/tools/tidal-api/utils/extractor.py b/tools/tidal-api/utils/extractor.py deleted file mode 100644 index cfa534e..0000000 --- a/tools/tidal-api/utils/extractor.py +++ /dev/null @@ -1,6 +0,0 @@ -import re - -def extract_links(text): - links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text) - urls = set([url for text, url in links]) - return urls \ No newline at end of file From a311ce6a1c8af74efb81f44d57ac8d63b78c9446 Mon Sep 17 00:00:00 2001 From: niclas Date: Fri, 23 Feb 2024 11:25:07 +0100 Subject: [PATCH 08/10] Add [technique] subtechnique --- tools/tidal-api/main.py | 18 ++- tools/tidal-api/models/cluster.py | 212 +++++++++++++++++++++++------- 2 files changed, 179 insertions(+), 51 deletions(-) diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index d402dbb..9438567 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -1,6 +1,13 @@ from api.api import TidalAPI from models.galaxy import Galaxy -from models.cluster import GroupCluster, SoftwareCluster, CampaignsCluster, TechniqueCluster, TacticCluster, ReferencesCluster +from models.cluster import ( + GroupCluster, + SoftwareCluster, + CampaignsCluster, + TechniqueCluster, + TacticCluster, + ReferencesCluster, +) import argparse import json import os @@ -9,6 +16,7 @@ CONFIG = "./config" GALAXY_PATH = "../../galaxies" CLUSTER_PATH = "../../clusters" + def create_galaxy(endpoint: str, version: int): api = TidalAPI() data = api.get_data(endpoint) @@ -17,7 +25,7 @@ def create_galaxy(endpoint: str, version: int): galaxy = Galaxy(**config["galaxy"], version=version) galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{endpoint}.json") - + match endpoint: case "groups": cluster = GroupCluster(**config["cluster"], uuid=galaxy.uuid) @@ -44,13 +52,14 @@ def create_galaxy(endpoint: str, version: int): cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{endpoint}.json") print(f"Galaxy tidal-{endpoint} created") + def main(args, galaxies): if args.all: for galaxy in galaxies: create_galaxy(galaxy, args.version) else: create_galaxy(args.type, args.version) - + if __name__ == "__main__": @@ -59,7 +68,6 @@ if __name__ == "__main__": if f.endswith(".json"): galaxies.append(f.split(".")[0]) - parser = argparse.ArgumentParser( description="Create galaxy and cluster json files from Tidal API" ) @@ -86,4 +94,4 @@ if __name__ == "__main__": if hasattr(args, "func"): args.func(args, galaxies=galaxies) else: - parser.print_help() \ No newline at end of file + parser.print_help() diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py index 4c75a98..fca9171 100644 --- a/tools/tidal-api/models/cluster.py +++ b/tools/tidal-api/models/cluster.py @@ -1,12 +1,14 @@ from dataclasses import dataclass, field, asdict import json + @dataclass class Meta: pass + @dataclass -class GroupsMeta(): +class GroupsMeta(Meta): source: str = None group_attack_id: str = None country: str = None @@ -16,8 +18,9 @@ class GroupsMeta(): tags: list = None owner: str = None + @dataclass -class SoftwareMeta(): +class SoftwareMeta(Meta): source: str = None type: str = None software_attack_id: str = None @@ -25,23 +28,32 @@ class SoftwareMeta(): tags: list = None owner: str = None + @dataclass -class TechniqueMeta(): +class TechniqueMeta(Meta): source: str = None platforms: list = None tags: list = None owner: str = None + @dataclass -class TacticMeta(): +class SubTechniqueMeta(Meta): + source: str = None + technique_attack_id: str = None + + +@dataclass +class TacticMeta(Meta): source: str = None tactic_attack_id: str = None ordinal_position: int = None tags: list = None owner: str = None + @dataclass -class ReferencesMeta(): +class ReferencesMeta(Meta): source: str = None refs: list = None title: str = None @@ -50,8 +62,9 @@ class ReferencesMeta(): date_published: str = None owner: str = None + @dataclass -class CampaignsMeta(): +class CampaignsMeta(Meta): source: str = None campaign_attack_id: str = None first_seen: str = None @@ -59,6 +72,7 @@ class CampaignsMeta(): tags: list = None owner: str = None + @dataclass class ClusterValue: description: str = "" @@ -69,11 +83,23 @@ class ClusterValue: def return_value(self): value_dict = asdict(self) - value_dict['meta'] = {k: v for k, v in asdict(self.meta).items() if v is not None} + value_dict["meta"] = { + k: v for k, v in asdict(self.meta).items() if v is not None + } return value_dict -class Cluster(): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + +class Cluster: + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): self.authors = authors self.category = category self.description = description @@ -85,7 +111,7 @@ class Cluster(): def add_values(self): print("This method should be implemented in the child class") - + def save_to_file(self, path): with open(path, "w") as file: file.write(json.dumps(self.__dict__(), indent=4)) @@ -104,29 +130,48 @@ class Cluster(): "uuid": self.uuid, "values": self.values, } - + + class GroupCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) - + def add_values(self, data): for entry in data["data"]: meta = GroupsMeta( source=entry.get("source"), group_attack_id=entry.get("group_attack_id"), - country=entry.get("country")[0].get("country_code") if entry.get("country") else None, - observed_countries=[x.get("country_code") for x in entry.get("observed_country")], - observed_motivations=[x.get("name") for x in entry.get("observed_motivation")], + country=( + entry.get("country")[0].get("country_code") + if entry.get("country") + else None + ), + observed_countries=[ + x.get("country_code") for x in entry.get("observed_country") + ], + observed_motivations=[ + x.get("name") for x in entry.get("observed_motivation") + ], target_categories=[x.get("name") for x in entry.get("observed_sector")], tags=[x.get("tag") for x in entry.get("tags")], owner=entry.get("owner_name"), ) related = [] for relation in entry.get("associated_groups"): - related.append({ - "dest-uuid": relation.get("id"), - "type": "related-to", - } + related.append( + { + "dest-uuid": relation.get("id"), + "type": "related-to", + } ) value = ClusterValue( description=entry.get("description"), @@ -139,9 +184,18 @@ class GroupCluster(Cluster): class SoftwareCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) - + def add_values(self, data): for entry in data["data"]: meta = SoftwareMeta( @@ -154,16 +208,18 @@ class SoftwareCluster(Cluster): ) related = [] for relation in entry.get("groups"): - related.append({ - "dest-uuid": relation.get("group_id"), - "type": "used-by", - } + related.append( + { + "dest-uuid": relation.get("group_id"), + "type": "used-by", + } ) for relation in entry.get("associated_software"): - related.append({ - "dest-uuid": relation.get("id"), - "type": "related-to", - } + related.append( + { + "dest-uuid": relation.get("id"), + "type": "related-to", + } ) value = ClusterValue( description=entry.get("description"), @@ -174,10 +230,20 @@ class SoftwareCluster(Cluster): ) self.values.append(value.return_value()) + class TechniqueCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) - + def add_values(self, data): for entry in data["data"]: meta = TechniqueMeta( @@ -188,10 +254,11 @@ class TechniqueCluster(Cluster): ) related = [] for relation in entry.get("tactic"): - related.append({ - "dest-uuid": relation.get("tactic_id"), - "type": "uses", - } + related.append( + { + "dest-uuid": relation.get("tactic_id"), + "type": "uses", + } ) value = ClusterValue( description=entry.get("description"), @@ -202,10 +269,42 @@ class TechniqueCluster(Cluster): ) self.values.append(value.return_value()) + for sub_technique in entry.get("sub_technique"): + meta = SubTechniqueMeta( + source=sub_technique.get("source"), + technique_attack_id=sub_technique.get("technique_attack_id"), + ) + related = [] + for relation in sub_technique.get("tactic"): + related.append( + { + "dest-uuid": relation.get("tactic_id"), + "type": "uses", + } + ) + value = ClusterValue( + description=sub_technique.get("description"), + meta=meta, + related=related, + uuid=sub_technique.get("id"), + value=sub_technique.get("name"), + ) + self.values.append(value.return_value()) + + class TacticCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) - + def add_values(self, data): for entry in data["data"]: meta = TacticMeta( @@ -217,10 +316,11 @@ class TacticCluster(Cluster): ) related = [] for relation in entry.get("techniques"): - related.append({ - "dest-uuid": relation.get("technique_id"), - "type": "uses", - } + related.append( + { + "dest-uuid": relation.get("technique_id"), + "type": "uses", + } ) value = ClusterValue( description=entry.get("description"), @@ -231,10 +331,20 @@ class TacticCluster(Cluster): ) self.values.append(value.return_value()) + class ReferencesCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) - + def add_values(self, data): for entry in data["data"]: meta = ReferencesMeta( @@ -255,8 +365,18 @@ class ReferencesCluster(Cluster): ) self.values.append(value.return_value()) + class CampaignsCluster(Cluster): - def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str): + def __init__( + self, + authors: str, + category: str, + description: str, + name: str, + source: str, + type: str, + uuid: str, + ): super().__init__(authors, category, description, name, source, type, uuid) def add_values(self, data): @@ -277,4 +397,4 @@ class CampaignsCluster(Cluster): uuid=entry.get("id"), value=entry.get("name"), ) - self.values.append(value.return_value()) \ No newline at end of file + self.values.append(value.return_value()) From 5062c6162099efa296a81690d17849ed398b3353 Mon Sep 17 00:00:00 2001 From: niclas Date: Fri, 23 Feb 2024 14:54:25 +0100 Subject: [PATCH 09/10] Add [tidal] relation enrichment with mitre --- tools/tidal-api/main.py | 16 ++- tools/tidal-api/models/cluster.py | 168 ++++++++++++++++++++++++------ 2 files changed, 149 insertions(+), 35 deletions(-) diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 9438567..559a4f5 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -17,7 +17,7 @@ GALAXY_PATH = "../../galaxies" CLUSTER_PATH = "../../clusters" -def create_galaxy(endpoint: str, version: int): +def create_galaxy(endpoint: str, version: int, extended_relations: bool = False): api = TidalAPI() data = api.get_data(endpoint) with open(f"{CONFIG}/{endpoint}.json", "r") as file: @@ -28,10 +28,10 @@ def create_galaxy(endpoint: str, version: int): match endpoint: case "groups": - cluster = GroupCluster(**config["cluster"], uuid=galaxy.uuid) + cluster = GroupCluster(**config["cluster"], uuid=galaxy.uuid, enrichment=extended_relations) cluster.add_values(data) case "software": - cluster = SoftwareCluster(**config["cluster"], uuid=galaxy.uuid) + cluster = SoftwareCluster(**config["cluster"], uuid=galaxy.uuid, enrichment=extended_relations) cluster.add_values(data) case "campaigns": cluster = CampaignsCluster(**config["cluster"], uuid=galaxy.uuid) @@ -56,9 +56,9 @@ def create_galaxy(endpoint: str, version: int): def main(args, galaxies): if args.all: for galaxy in galaxies: - create_galaxy(galaxy, args.version) + create_galaxy(galaxy, args.version, args.extended_relations) else: - create_galaxy(args.type, args.version) + create_galaxy(args.type, args.version, args.extended_relations) if __name__ == "__main__": @@ -72,6 +72,7 @@ if __name__ == "__main__": description="Create galaxy and cluster json files from Tidal API" ) parser.add_argument( + "-a", "--all", action="store_true", help="Create all galaxies and clusters", @@ -88,6 +89,11 @@ if __name__ == "__main__": required=True, help="The version of the galaxy", ) + parser.add_argument( + "--extended-relations", + action="store_true", + help="Create extended relations in the cluster", + ) parser.set_defaults(func=main) args = parser.parse_args() diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py index fca9171..f85839d 100644 --- a/tools/tidal-api/models/cluster.py +++ b/tools/tidal-api/models/cluster.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field, asdict +from typing import Type import json @@ -19,6 +20,13 @@ class GroupsMeta(Meta): owner: str = None +@dataclass +class AssociatedGroupsMeta(Meta): + id: str = None + owner_id: str = None + owner: str = None + + @dataclass class SoftwareMeta(Meta): source: str = None @@ -29,6 +37,13 @@ class SoftwareMeta(Meta): owner: str = None +@dataclass +class AssociatedSoftwareMeta(Meta): + id: str = None + owner_id: str = None + owner: str = None + + @dataclass class TechniqueMeta(Meta): source: str = None @@ -108,9 +123,10 @@ class Cluster: self.type = type self.uuid = uuid self.values = [] + self.CLUSTER_PATH = "../../clusters" - def add_values(self): - print("This method should be implemented in the child class") + def add_values(self, data: dict, meta_class: Type[Meta]): + pass def save_to_file(self, path): with open(path, "w") as file: @@ -131,6 +147,24 @@ class Cluster: "values": self.values, } + def _get_relation_from_mitre_id( + self, mitre_id: str, cluster: str, meta_key: str, array: bool = False + ): + with open(f"{self.CLUSTER_PATH}/{cluster}.json", "r") as file: + mitre = json.load(file) + for entry in mitre["values"]: + try: + if array: + for id in entry["meta"][meta_key]: + if id == mitre_id: + return entry["uuid"] + else: + if entry["meta"][meta_key] == mitre_id: + return entry["uuid"] + except KeyError: + continue + return None + class GroupCluster(Cluster): def __init__( @@ -142,8 +176,10 @@ class GroupCluster(Cluster): source: str, type: str, uuid: str, + enrichment: bool = False, ): super().__init__(authors, category, description, name, source, type, uuid) + self.enrichment = enrichment def add_values(self, data): for entry in data["data"]: @@ -166,13 +202,39 @@ class GroupCluster(Cluster): owner=entry.get("owner_name"), ) related = [] - for relation in entry.get("associated_groups"): + if self.enrichment: + related_cluster = self._get_relation_from_mitre_id( + entry.get("group_attack_id"), "threat-actor", "synonyms", True + ) + if related_cluster: + related.append( + { + "dest-uuid": related_cluster, + "type": "similar", + } + ) + + for associated_group in entry.get("associated_groups"): + meta = AssociatedGroupsMeta( + id=associated_group.get("id"), + owner_id=associated_group.get("owner_id"), + owner=associated_group.get("owner_name"), + ) + value = ClusterValue( + description=associated_group.get("description"), + meta=meta, + related=[], + uuid=associated_group.get("associated_group_id"), + value=associated_group.get("name"), + ) + self.values.append(value.return_value()) related.append( { - "dest-uuid": relation.get("id"), - "type": "related-to", + "dest-uuid": associated_group.get("associated_group_id"), + "type": "similar", } ) + value = ClusterValue( description=entry.get("description"), meta=meta, @@ -193,8 +255,10 @@ class SoftwareCluster(Cluster): source: str, type: str, uuid: str, + enrichment: bool = False, ): super().__init__(authors, category, description, name, source, type, uuid) + self.enrichment = enrichment def add_values(self, data): for entry in data["data"]: @@ -214,13 +278,50 @@ class SoftwareCluster(Cluster): "type": "used-by", } ) - for relation in entry.get("associated_software"): + if self.enrichment: + related_cluster = self._get_relation_from_mitre_id( + entry.get("software_attack_id"), "mitre-tool", "external_id" + ) + if related_cluster: + related.append( + { + "dest-uuid": related_cluster, + "type": "similar", + } + ) + + related_cluster = self._get_relation_from_mitre_id( + entry.get("software_attack_id"), "mitre-malware", "external_id" + ) + if related_cluster: + related.append( + { + "dest-uuid": related_cluster, + "type": "similar", + } + ) + + for associated_software in entry.get("associated_software"): + meta = AssociatedSoftwareMeta( + id=associated_software.get("id"), + owner_id=associated_software.get("owner_id"), + owner=associated_software.get("owner_name"), + ) + value = ClusterValue( + description=associated_software.get("description"), + meta=meta, + related=[], + uuid=associated_software.get("associated_software_id"), + value=associated_software.get("name"), + ) + self.values.append(value.return_value()) related.append( { - "dest-uuid": relation.get("id"), - "type": "related-to", + "dest-uuid": associated_software.get("associated_software_id"), + "type": "similar", } ) + value = ClusterValue( description=entry.get("description"), meta=meta, @@ -260,6 +361,35 @@ class TechniqueCluster(Cluster): "type": "uses", } ) + + for sub_technique in entry.get("sub_technique"): + meta = SubTechniqueMeta( + source=sub_technique.get("source"), + technique_attack_id=sub_technique.get("technique_attack_id"), + ) + sub_related = [] + for relation in sub_technique.get("tactic"): + sub_related.append( + { + "dest-uuid": relation.get("tactic_id"), + "type": "uses", + } + ) + sub_value = ClusterValue( + description=sub_technique.get("description"), + meta=meta, + related=sub_related, + uuid=sub_technique.get("id"), + value=sub_technique.get("name"), + ) + self.values.append(sub_value.return_value()) + related.append( + { + "dest-uuid": sub_technique.get("id"), + "type": "similar", + } + ) + value = ClusterValue( description=entry.get("description"), meta=meta, @@ -269,28 +399,6 @@ class TechniqueCluster(Cluster): ) self.values.append(value.return_value()) - for sub_technique in entry.get("sub_technique"): - meta = SubTechniqueMeta( - source=sub_technique.get("source"), - technique_attack_id=sub_technique.get("technique_attack_id"), - ) - related = [] - for relation in sub_technique.get("tactic"): - related.append( - { - "dest-uuid": relation.get("tactic_id"), - "type": "uses", - } - ) - value = ClusterValue( - description=sub_technique.get("description"), - meta=meta, - related=related, - uuid=sub_technique.get("id"), - value=sub_technique.get("name"), - ) - self.values.append(value.return_value()) - class TacticCluster(Cluster): def __init__( From a0f3ed587343e2023173bfe36d55b917575d3153 Mon Sep 17 00:00:00 2001 From: niclas Date: Mon, 26 Feb 2024 09:40:42 +0100 Subject: [PATCH 10/10] Add [tidal] relations for associated objects --- tools/tidal-api/models/cluster.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py index f85839d..8b355ea 100644 --- a/tools/tidal-api/models/cluster.py +++ b/tools/tidal-api/models/cluster.py @@ -215,15 +215,22 @@ class GroupCluster(Cluster): ) for associated_group in entry.get("associated_groups"): - meta = AssociatedGroupsMeta( + associated_meta = AssociatedGroupsMeta( id=associated_group.get("id"), owner_id=associated_group.get("owner_id"), owner=associated_group.get("owner_name"), ) + associated_related = [] + associated_related.append( + { + "dest-uuid": entry.get("id"), + "type": "similar", + } + ) value = ClusterValue( description=associated_group.get("description"), - meta=meta, - related=[], + meta=associated_meta, + related=associated_related, uuid=associated_group.get("associated_group_id"), value=associated_group.get("name"), ) @@ -302,15 +309,22 @@ class SoftwareCluster(Cluster): ) for associated_software in entry.get("associated_software"): - meta = AssociatedSoftwareMeta( + associated_meta = AssociatedSoftwareMeta( id=associated_software.get("id"), owner_id=associated_software.get("owner_id"), owner=associated_software.get("owner_name"), ) + associated_related = [] + associated_related.append( + { + "dest-uuid": entry.get("id"), + "type": "similar", + } + ) value = ClusterValue( description=associated_software.get("description"), - meta=meta, - related=[], + meta=associated_meta, + related=associated_related, uuid=associated_software.get("associated_software_id"), value=associated_software.get("name"), ) @@ -363,7 +377,7 @@ class TechniqueCluster(Cluster): ) for sub_technique in entry.get("sub_technique"): - meta = SubTechniqueMeta( + sub_meta = SubTechniqueMeta( source=sub_technique.get("source"), technique_attack_id=sub_technique.get("technique_attack_id"), ) @@ -377,7 +391,7 @@ class TechniqueCluster(Cluster): ) sub_value = ClusterValue( description=sub_technique.get("description"), - meta=meta, + meta=sub_meta, related=sub_related, uuid=sub_technique.get("id"), value=sub_technique.get("name"),