From b9746f2b4155526573aa7a647fd572f4989fe5d9 Mon Sep 17 00:00:00 2001 From: niclas Date: Thu, 22 Feb 2024 10:18:18 +0100 Subject: [PATCH] chg [config] external config file --- tools/tidal-api/README.md | 67 ++++++++ tools/tidal-api/config.json | 249 ++++++++++++++++++++++++++++ tools/tidal-api/create_campaigns.py | 76 --------- tools/tidal-api/create_groups.py | 91 ---------- tools/tidal-api/create_software.py | 106 ------------ tools/tidal-api/main.py | 133 +-------------- tools/tidal-api/utils/config.py | 14 ++ 7 files changed, 339 insertions(+), 397 deletions(-) create mode 100644 tools/tidal-api/README.md create mode 100644 tools/tidal-api/config.json delete mode 100644 tools/tidal-api/create_campaigns.py delete mode 100644 tools/tidal-api/create_groups.py delete mode 100644 tools/tidal-api/create_software.py create mode 100644 tools/tidal-api/utils/config.py diff --git a/tools/tidal-api/README.md b/tools/tidal-api/README.md new file mode 100644 index 0000000..8e8b992 --- /dev/null +++ b/tools/tidal-api/README.md @@ -0,0 +1,67 @@ +# Tidal Cyber API + +This is a tool generating MISP galaxies and clusters from Tidal Cyber API. + +## Endpoints +https://app-api.tidalcyber.com/api/v1/technique + +https://app-api.tidalcyber.com/api/v1/references + +https://app-api.tidalcyber.com/api/v1/tactic + +https://app-api.tidalcyber.com/api/v1/campaigns/ + +https://app-api.tidalcyber.com/api/v1/software/ + +https://app-api.tidalcyber.com/api/v1/groups/ + +## Configuration +The configuration file is located in `config.json` and maps the fields of the Tidal API to the Galaxy and Cluster fields. It consists of the following sections: +- `UUID`: The UUID of the galaxy to be created +- `GALAXY_CONFIGS`: The configuration of the galaxies to be created in the `galaxies` folder of the MISP-galaxy repository + - `name`: The name of the galaxy + - `namespace`: The namespace of the galaxy + - `description`: The description of the galaxy + - `type`: The type of the galaxy + - `uuid`: The UUID of the galaxy (will be inserted from the `UUID` section) +- `CLUSTER_CONFIGS`: The configuration of the clusters to be created in the `clusters` folder of the MISP-galaxy repository + - `authors`: The authors of the cluster + - `category`: The category of the cluster + - `description`: The description of the cluster + - `name`: The name of the cluster + - `source`: The source of the cluster + - `type`: The type of the cluster + - `uuid`: The UUID of the cluster (will be inserted from the `UUID` section) + - `values`: The values of the cluster (will be inserted from the `VALUE_FIELDS` section) +- `VALUE_FIELDS`: Defines the mapping of the fields in the Tidal Cyber API to the fields in the MISP cluster values array + - `description`: The description of the cluster value + - `meta`: The metadata of the cluster value + - `related`: The related cluster values of the cluster value (you can define a `type` for each relation type in the config which will not be mapped to a field of the API) + - `uuid`: The UUID of the cluster value + - `value`: The value of the cluster value + >Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration. + +### Extraction Configuration +The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this: +```json +{ + "extract": , + "key": , + "subkey": +} +``` +**Extract modes**: + +- `single`: Extracts a single value from the API response +- `multiple`: Extracts multiple values from the API response +- `reverse`: Gets the value of the key and writes it into an array (no subkey needed) + +## Usage +```bash +python3 main.py create-galaxy -v --type +``` +To build all galaxies and clusters, run the following command: + +```bash +python3 main.py create-galaxy -v --all +``` \ No newline at end of file diff --git a/tools/tidal-api/config.json b/tools/tidal-api/config.json new file mode 100644 index 0000000..dd50dbe --- /dev/null +++ b/tools/tidal-api/config.json @@ -0,0 +1,249 @@ +{ + "UUIDS": { + "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", + "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", + "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", + "technique": "3e28b683-8159-4398-8729-7248eac9aa45", + "tactic": "16a396e2-a4a9-4dfd-be0a-6ba75fb5382c", + "references": "cf5e180f-26e9-42b2-b10c-c9e55f448750" + }, + "GALAXY_CONFIGS": { + "software": { + "name": "Tidal Software", + "namespace": "tidal", + "description": "Tidal Software Galaxy", + "type": "software", + "uuid": "" + }, + "groups": { + "name": "Tidal Groups", + "namespace": "tidal", + "description": "Tidal Groups Galaxy", + "type": "groups", + "uuid": "" + }, + "campaigns": { + "name": "Tidal Campaigns", + "namespace": "tidal", + "description": "Tidal Campaigns Galaxy", + "type": "campaigns", + "uuid": "" + }, + "technique": { + "name": "Tidal Technique", + "namespace": "tidal", + "description": "Tidal Technique Galaxy", + "type": "technique", + "uuid": "" + }, + "tactic": { + "name": "Tidal Tactic", + "namespace": "tidal", + "description": "Tidal Tactic Galaxy", + "type": "tactics", + "uuid": "" + }, + "references": { + "name": "Tidal References", + "namespace": "tidal", + "description": "Tidal References Galaxy", + "type": "references", + "uuid": "" + } + }, + "CLUSTER_CONFIGS": { + "software": { + "authors": "Tidal", + "category": "Software", + "description": "Tidal Software Cluster", + "name": "Tidal Software", + "source": "Tidal", + "type": "software", + "uuid": "", + "values": [] + }, + "groups": { + "authors": "Tidal", + "category": "Threat Groups", + "description": "Tidal Threat Groups Cluster", + "name": "Tidal Threat Groups", + "source": "Tidal", + "type": "groups", + "uuid": "", + "values": [] + }, + "campaigns": { + "authors": "Tidal", + "category": "Campaigns", + "description": "Tidal Campaigns Cluster", + "name": "Tidal Campaigns", + "source": "Tidal", + "type": "campaigns", + "uuid": "", + "values": [] + }, + "technique": { + "authors": "Tidal", + "category": "Techniques", + "description": "Tidal Techniques Cluster", + "name": "Tidal Techniques", + "source": "Tidal", + "type": "technique", + "uuid": "", + "values": [] + }, + "tactic": { + "authors": "Tidal", + "category": "Tactics", + "description": "Tidal Tactics Cluster", + "name": "Tidal Tactics", + "source": "Tidal", + "type": "tactic", + "uuid": "", + "values": [] + }, + "references": { + "authors": "Tidal", + "category": "References", + "description": "Tidal References Cluster", + "name": "Tidal References", + "source": "Tidal", + "type": "references", + "uuid": "", + "values": [] + } + }, + "VALUE_FIELDS": { + "software": { + "description": "description", + "meta": { + "source": "source", + "type": "type", + "software-attack-id": "software_attack_id", + "platforms": "platforms", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "groups": { + "dest-uuid": "group_id", + "type": "used-by" + }, + "associated_software": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "groups": { + "description": "description", + "meta": { + "source": "source", + "group-attack-id": "group_attack_id", + "country": { + "extract": "single", + "key": "country", + "subkey": "country_code" + }, + "observed_country": { + "extract": "multiple", + "key": "observed_country", + "subkey": "country_code" + }, + "observed_motivation": { + "extract": "multiple", + "key": "observed_motivation", + "subkey": "name" + }, + "target-category": { + "extract": "multiple", + "key": "observed_sector", + "subkey": "name" + }, + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "associated_groups": { + "dest-uuid": "id", + "type": "related-to" + } + }, + "uuid": "id", + "value": "name" + }, + "campaigns": { + "description": "description", + "meta": { + "source": "source", + "campaign-attack-id": "campaign_attack_id", + "first_seen": "first_seen", + "last_seen": "last_seen", + "tags": "tags", + "owner": "owner_name" + }, + "related": {}, + "uuid": "id", + "value": "name" + }, + "technique": { + "description": "description", + "meta": { + "source": "source", + "platforms": "platforms", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "tactic": { + "dest-uuid": "tactic_id", + "type": "uses" + }, + "sub_technique": { + "dest-uuid": "id", + "type": "sub-technique-of" + } + }, + "uuid": "id", + "value": "name" + }, + "tactic": { + "description": "description", + "meta": { + "source": "source", + "tactic-attack-id": "tactic_attack_id", + "ordinal_position": "ordinal_position", + "tags": "tags", + "owner": "owner_name" + }, + "related": { + "techniques": { + "dest-uuid": "technique_id", + "type": "uses" + } + }, + "uuid": "id", + "value": "name" + }, + "references": { + "description": "description", + "meta": { + "source": "source", + "refs": { + "extract": "reverse", + "key": "url" + }, + "owner": "owner_name", + "title": "title", + "author": "authors", + "date_accessed": "date_accessed", + "date_published": "date_published" + }, + "related": {}, + "uuid": "id", + "value": "name" + } + } +} \ No newline at end of file diff --git a/tools/tidal-api/create_campaigns.py b/tools/tidal-api/create_campaigns.py deleted file mode 100644 index 8b6123c..0000000 --- a/tools/tidal-api/create_campaigns.py +++ /dev/null @@ -1,76 +0,0 @@ -from api import TidalAPI -import json - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "43a8fce6-08d3-46c2-957d-53606efe2c48" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Campaigns Galaxy" - galaxy["name"] = "Tidal Campaigns" - galaxy["namespace"] = "tidal" - galaxy["type"] = "campaigns" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for campaigns in data["data"]: - value = {} - relations = [] - - value["description"] = campaigns["description"] - - # Metadata fields - source = campaigns["source"] - campaign_attack_id = campaigns["campaign_attack_id"] - first_seen = campaigns["first_seen"] - last_seen = campaigns["last_seen"] - tags = campaigns["tags"] - owner = campaigns["owner_name"] - - value["meta"] = {} - if source: - value["meta"]["source"] = source - if campaign_attack_id: - value["meta"]["campaign-attack-id"] = campaign_attack_id - if first_seen: - value["meta"]["first-seen"] = first_seen - if last_seen: - value["meta"]["last-seen"] = last_seen - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = campaigns["id"] - value["value"] = campaigns["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat campaigns" - cluster["description"] = "Tidal Campaigns" - cluster["name"] = "Tidal Campaigns" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/campaigns" - cluster["type"] = "campaigns" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - -if __name__ == "__main__": - api = TidalAPI() - data = api.get_data('campaigns') - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-campaigns.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-campaigns.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py deleted file mode 100644 index 18431a6..0000000 --- a/tools/tidal-api/create_groups.py +++ /dev/null @@ -1,91 +0,0 @@ -from api import TidalAPI -import json - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Threat Group Galaxy" - galaxy["name"] = "Tidal Threat Group" - galaxy["namespace"] = "tidal" - galaxy["type"] = "threat-group" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for group in data["data"]: - value = {} - relations = [] - # TODO check for id and associated_group_id and add to relations - for entry in group["associated_groups"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "related-to" - relations.append(relation) - - value["description"] = group["description"] - - # Metadata fields - source = group["source"] - group_attack_id = group["group_attack_id"] - country = [country["country_name"] for country in group["country"]] - observed_country = [country["country_code"] for country in group["observed_country"]] - motive = [motive["name"] for motive in group["observed_motivation"]] - target_category = [sector["name"] for sector in group["observed_sector"]] - tags = group["tags"] - owner = group["owner_name"] - - value["meta"] = {} - if source: - value["meta"]["source"] = source - if group_attack_id: - value["meta"]["group-attack-id"] = group_attack_id - if country: - value["meta"]["country"] = country - if observed_country: - value["meta"]["observed_country"] = observed_country - if motive: - value["meta"]["motive"] = motive - if target_category: - value["meta"]["target-category"] = target_category - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = group["id"] - value["value"] = group["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat Group" - cluster["description"] = "Tidal Threat Groups" - cluster["name"] = "Tidal Threat Group" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/groups" - cluster["type"] = "threat-group" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - - -if __name__ == "__main__": - - api = TidalAPI() - data = api.get_data("groups") - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-threat-groups.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-threat-groups.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py deleted file mode 100644 index 27ff409..0000000 --- a/tools/tidal-api/create_software.py +++ /dev/null @@ -1,106 +0,0 @@ -from api import TidalAPI -import json -import re - -VERSION = 1 -GALAXY_PATH = "../../galaxies/" -CLUSTER_PATH = "../../clusters/" -GALAXY_UUID = "38d62d8b-4c49-489a-9bc4-8e294c4f04f7" - -def create_galaxy(): - galaxy = {} - galaxy["description"] = "Tidal Software Galaxy" - galaxy["name"] = "Tidal Software" - galaxy["namespace"] = "tidal" - galaxy["type"] = "software" - galaxy["uuid"] = GALAXY_UUID - galaxy["version"] = VERSION - return galaxy - -def create_cluster(galaxy, data): - cluster = {} - values = [] - - for software in data["data"]: - value = {} - relations = [] - # TODO check for relations etc. - for entry in software["groups"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "used-by" - relations.append(relation) - for entry in software["associated_software"]: - relation = {} - relation["dest-uuid"] = entry["id"] - relation["type"] = "related-to" - relations.append(relation) - - value["description"] = software["description"] - - # Metadata fields - links = extract_links(software["description"]) - source = software["source"] - type = software["type"] - software_attack_id = software["software_attack_id"] - platforms = software["platforms"] - tags = software["tags"] - owner = software["owner_name"] - - value["meta"] = {} - if links: - value["meta"]["refs"] = list(links) - if source: - value["meta"]["source"] = source - if type: - value["meta"]["type"] = type - if software_attack_id: - value["meta"]["software-attack-id"] = software_attack_id - if platforms: - value["meta"]["platforms"] = platforms - if tags: - value["meta"]["tags"] = tags - if owner: - value["meta"]["owner"] = owner - - value["related"] = relations - value["uuid"] = software["id"] - value["value"] = software["name"] - values.append(value) - - cluster["authors"] = ["Tidal"] - cluster["category"] = "Threat software" - cluster["description"] = "Tidal Threat Groups" - cluster["name"] = "Tidal Threat software" - cluster["source"] = "https://app-api.tidalcyber.com/api/v1/software" - cluster["type"] = "threat-software" - cluster["uuid"] = galaxy["uuid"] - cluster["values"] = values - return cluster - -def extract_links(text): - # extract markdown links and return text without links and the links - # urls = re.findall(r'https?://[^\s\)]+', text) - regular_links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text) - # sup_links = re.findall(r'\[\[([^\]]+)\]\((https?://[^\s\)]+)\)\]', text) - - # Extracting URLs from the tuples - regular_links_urls = set([url for text, url in regular_links]) - # sup_links_urls = [url for text, url in sup_links] - - # text_without_links = re.sub(r'\[([^\]]+)\]\(https?://[^\s\)]+\)', r'\1', text) - # text_without_sup = re.sub(r'.*<\/sup>', '', text_without_links) - - return regular_links_urls - -if __name__ == "__main__": - api = TidalAPI() - data = api.get_data('software') - galaxy = create_galaxy() - cluster = create_cluster(galaxy, data) - - with open(GALAXY_PATH + "tidal-software.json", "w") as galaxy_file: - json.dump(galaxy, galaxy_file, indent=4) - - with open(CLUSTER_PATH + "tidal-software.json", "w") as cluster_file: - json.dump(cluster, cluster_file, indent=4) \ No newline at end of file diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py index 8bf9691..008f171 100644 --- a/tools/tidal-api/main.py +++ b/tools/tidal-api/main.py @@ -2,135 +2,18 @@ from api.api import TidalAPI from models.galaxy import Galaxy from models.cluster import Cluster from utils.extractor import extract_links +from utils.config import load_config import argparse CLUSTER_PATH = "../../clusters/" GALAXY_PATH = "../../galaxies/" -UUIDS = { - "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7", - "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6", - "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48", -} +config = load_config('./config.json') -GALAXY_CONFIGS = { - "software": { - "name": "Tidal Software", - "namespace": "tidal", - "description": "Tidal Software Galaxy", - "type": "software", - "uuid": UUIDS["software"], - }, - "groups": { - "name": "Tidal Groups", - "namespace": "tidal", - "description": "Tidal Groups Galaxy", - "type": "groups", - "uuid": UUIDS["groups"], - }, - "campaigns": { - "name": "Tidal Campaigns", - "namespace": "tidal", - "description": "Tidal Campaigns Galaxy", - "type": "campaigns", - "uuid": UUIDS["campaigns"], - } -} - -CLUSTER_CONFIGS = { - "software": { - "authors": "Tidal", - "category": "Software", - "description": "Tidal Software Cluster", - "name": "Tidal Software", - "source": "Tidal", - "type": "software", - "uuid": UUIDS["software"], - "values": [] - }, - "groups": { - "authors": "Tidal", - "category": "Threat Groups", - "description": "Tidal Threat Groups Cluster", - "name": "Tidal Threat Groups", - "source": "Tidal", - "type": "groups", - "uuid": UUIDS["groups"], - "values": [] - }, - "campaigns": { - "authors": "Tidal", - "category": "Campaigns", - "description": "Tidal Campaigns Cluster", - "name": "Tidal Campaigns", - "source": "Tidal", - "type": "campaigns", - "uuid": UUIDS["campaigns"], - "values": [] - } -} - -VALUE_FIELDS = { - "software": { - "description": "description", - "meta": { - "source": "source", - "type": "type", - "software-attack-id": "software_attack_id", - "platforms": "platforms", - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "groups": { - "dest-uuid": "group_id", - "type": "used-by" - }, - "associated_software": { - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "groups": { - "description": "description", - "meta": { - "source": "source", - "group-attack-id": "group_attack_id", - "country": {"extract": "single", "key": "country", "subkey": "country_code"}, - "observed_country": {"extract": "multiple", "key": "observed_country", "subkey": "country_code"}, - "observed_motivation": {"extract": "multiple", "key": "observed_motivation", "subkey": "name"}, - "target-category": {"extract": "multiple", "key": "observed_sector", "subkey": "name"}, - "tags": "tags", - "owner": "owner_name" - }, - "related": { - "associated_groups": { - "dest-uuid": "id", - "type": "related-to" - } - }, - "uuid": "id", - "value": "name" - }, - "campaigns": { - "description": "description", - "meta": { - "source": "source", - "campaign-attack-id": "campaign_attack_id", - "first_seen": "first_seen", - "last_seen": "last_seen", - "tags": "tags", - "owner": "owner_name" - }, - "related": {}, - "uuid": "id", - "value": "name" - } - -} +UUIDS = config['UUIDS'] +GALAXY_CONFIGS = config['GALAXY_CONFIGS'] +CLUSTER_CONFIGS = config['CLUSTER_CONFIGS'] +VALUE_FIELDS = config['VALUE_FIELDS'] def create_cluster_values(data, cluster): value_fields = VALUE_FIELDS[cluster.internal_type] @@ -162,6 +45,8 @@ def create_metadata(data, format): metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"]) elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]): metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])] + elif meta_value.get("extract") == "reverse" and data.get(meta_value["key"]): + metadata[meta_key] = [data.get(meta_value["key"])] elif data.get(meta_value): metadata[meta_key] = data.get(meta_value) return metadata @@ -191,7 +76,7 @@ def create_galaxy_and_cluster(galaxy_type, version): create_cluster_values(data, cluster) cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json") - print(f"Galaxy {galaxy_type} created") + print(f"Galaxy tidal-{galaxy_type} created") def create_galaxy(args): if args.all: diff --git a/tools/tidal-api/utils/config.py b/tools/tidal-api/utils/config.py new file mode 100644 index 0000000..d69e997 --- /dev/null +++ b/tools/tidal-api/utils/config.py @@ -0,0 +1,14 @@ +import json + +def load_config(file_path): + with open(file_path, 'r') as file: + config = json.load(file) + return link_uuids(config) + +def link_uuids(config): + uuids = config["UUIDS"] + for key, galaxy_config in config["GALAXY_CONFIGS"].items(): + galaxy_config["uuid"] = uuids[key] + for key, cluster_config in config["CLUSTER_CONFIGS"].items(): + cluster_config["uuid"] = uuids[key] + return config \ No newline at end of file