diff --git a/tools/tidal-api/api/__init__.py b/tools/tidal-api/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/tidal-api/api.py b/tools/tidal-api/api/api.py
similarity index 100%
rename from tools/tidal-api/api.py
rename to tools/tidal-api/api/api.py
diff --git a/tools/tidal-api/create_groups.py b/tools/tidal-api/create_groups.py
index 163d80f..18431a6 100644
--- a/tools/tidal-api/create_groups.py
+++ b/tools/tidal-api/create_groups.py
@@ -84,8 +84,8 @@ if __name__ == "__main__":
galaxy = create_galaxy()
cluster = create_cluster(galaxy, data)
- with open(GALAXY_PATH + "tidal-threat-group.json", "w") as galaxy_file:
+ with open(GALAXY_PATH + "tidal-threat-groups.json", "w") as galaxy_file:
json.dump(galaxy, galaxy_file, indent=4)
- with open(CLUSTER_PATH + "tidal-threat-group.json", "w") as cluster_file:
+ with open(CLUSTER_PATH + "tidal-threat-groups.json", "w") as cluster_file:
json.dump(cluster, cluster_file, indent=4)
\ No newline at end of file
diff --git a/tools/tidal-api/create_software.py b/tools/tidal-api/create_software.py
index 9cebad3..27ff409 100644
--- a/tools/tidal-api/create_software.py
+++ b/tools/tidal-api/create_software.py
@@ -1,5 +1,6 @@
from api import TidalAPI
import json
+import re
VERSION = 1
GALAXY_PATH = "../../galaxies/"
@@ -38,6 +39,7 @@ def create_cluster(galaxy, data):
value["description"] = software["description"]
# Metadata fields
+ links = extract_links(software["description"])
source = software["source"]
type = software["type"]
software_attack_id = software["software_attack_id"]
@@ -46,6 +48,8 @@ def create_cluster(galaxy, data):
owner = software["owner_name"]
value["meta"] = {}
+ if links:
+ value["meta"]["refs"] = list(links)
if source:
value["meta"]["source"] = source
if type:
@@ -74,6 +78,21 @@ def create_cluster(galaxy, data):
cluster["values"] = values
return cluster
+def extract_links(text):
+ # extract markdown links and return text without links and the links
+ # urls = re.findall(r'https?://[^\s\)]+', text)
+ regular_links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text)
+ # sup_links = re.findall(r'\[\[([^\]]+)\]\((https?://[^\s\)]+)\)\]', text)
+
+ # Extracting URLs from the tuples
+ regular_links_urls = set([url for text, url in regular_links])
+ # sup_links_urls = [url for text, url in sup_links]
+
+ # text_without_links = re.sub(r'\[([^\]]+)\]\(https?://[^\s\)]+\)', r'\1', text)
+ # text_without_sup = re.sub(r'.*<\/sup>', '', text_without_links)
+
+ return regular_links_urls
+
if __name__ == "__main__":
api = TidalAPI()
data = api.get_data('software')
diff --git a/tools/tidal-api/main.py b/tools/tidal-api/main.py
new file mode 100644
index 0000000..8bf9691
--- /dev/null
+++ b/tools/tidal-api/main.py
@@ -0,0 +1,218 @@
+from api.api import TidalAPI
+from models.galaxy import Galaxy
+from models.cluster import Cluster
+from utils.extractor import extract_links
+import argparse
+
+CLUSTER_PATH = "../../clusters/"
+GALAXY_PATH = "../../galaxies/"
+
+UUIDS = {
+ "software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7",
+ "groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6",
+ "campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48",
+}
+
+GALAXY_CONFIGS = {
+ "software": {
+ "name": "Tidal Software",
+ "namespace": "tidal",
+ "description": "Tidal Software Galaxy",
+ "type": "software",
+ "uuid": UUIDS["software"],
+ },
+ "groups": {
+ "name": "Tidal Groups",
+ "namespace": "tidal",
+ "description": "Tidal Groups Galaxy",
+ "type": "groups",
+ "uuid": UUIDS["groups"],
+ },
+ "campaigns": {
+ "name": "Tidal Campaigns",
+ "namespace": "tidal",
+ "description": "Tidal Campaigns Galaxy",
+ "type": "campaigns",
+ "uuid": UUIDS["campaigns"],
+ }
+}
+
+CLUSTER_CONFIGS = {
+ "software": {
+ "authors": "Tidal",
+ "category": "Software",
+ "description": "Tidal Software Cluster",
+ "name": "Tidal Software",
+ "source": "Tidal",
+ "type": "software",
+ "uuid": UUIDS["software"],
+ "values": []
+ },
+ "groups": {
+ "authors": "Tidal",
+ "category": "Threat Groups",
+ "description": "Tidal Threat Groups Cluster",
+ "name": "Tidal Threat Groups",
+ "source": "Tidal",
+ "type": "groups",
+ "uuid": UUIDS["groups"],
+ "values": []
+ },
+ "campaigns": {
+ "authors": "Tidal",
+ "category": "Campaigns",
+ "description": "Tidal Campaigns Cluster",
+ "name": "Tidal Campaigns",
+ "source": "Tidal",
+ "type": "campaigns",
+ "uuid": UUIDS["campaigns"],
+ "values": []
+ }
+}
+
+VALUE_FIELDS = {
+ "software": {
+ "description": "description",
+ "meta": {
+ "source": "source",
+ "type": "type",
+ "software-attack-id": "software_attack_id",
+ "platforms": "platforms",
+ "tags": "tags",
+ "owner": "owner_name"
+ },
+ "related": {
+ "groups": {
+ "dest-uuid": "group_id",
+ "type": "used-by"
+ },
+ "associated_software": {
+ "dest-uuid": "id",
+ "type": "related-to"
+ }
+ },
+ "uuid": "id",
+ "value": "name"
+ },
+ "groups": {
+ "description": "description",
+ "meta": {
+ "source": "source",
+ "group-attack-id": "group_attack_id",
+ "country": {"extract": "single", "key": "country", "subkey": "country_code"},
+ "observed_country": {"extract": "multiple", "key": "observed_country", "subkey": "country_code"},
+ "observed_motivation": {"extract": "multiple", "key": "observed_motivation", "subkey": "name"},
+ "target-category": {"extract": "multiple", "key": "observed_sector", "subkey": "name"},
+ "tags": "tags",
+ "owner": "owner_name"
+ },
+ "related": {
+ "associated_groups": {
+ "dest-uuid": "id",
+ "type": "related-to"
+ }
+ },
+ "uuid": "id",
+ "value": "name"
+ },
+ "campaigns": {
+ "description": "description",
+ "meta": {
+ "source": "source",
+ "campaign-attack-id": "campaign_attack_id",
+ "first_seen": "first_seen",
+ "last_seen": "last_seen",
+ "tags": "tags",
+ "owner": "owner_name"
+ },
+ "related": {},
+ "uuid": "id",
+ "value": "name"
+ }
+
+}
+
+def create_cluster_values(data, cluster):
+ value_fields = VALUE_FIELDS[cluster.internal_type]
+ for entry in data["data"]:
+ values = {}
+ for key, value in value_fields.items():
+ match key:
+ case "description":
+ values[value] = entry.get(key)
+ case "meta":
+ metadata = create_metadata(entry, value)
+ values["meta"] = metadata
+ case "related":
+ relations = create_relations(entry, value)
+ values["related"] = relations
+ case "uuid":
+ values[key] = entry.get(value)
+ case "value":
+ values[key] = entry.get(value)
+ case _:
+ print(f"Error: Invalid configuration for {key} in {cluster.internal_type} value fields.")
+ cluster.add_value(values)
+
+def create_metadata(data, format):
+ metadata = {}
+ for meta_key, meta_value in format.items():
+ if isinstance(meta_value, dict):
+ if meta_value.get("extract") == "single" and data.get(meta_value["key"]):
+ metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"])
+ elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]):
+ metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])]
+ elif data.get(meta_value):
+ metadata[meta_key] = data.get(meta_value)
+ return metadata
+
+
+def create_relations(data, format):
+ relations = []
+ for i in range(len(list(format))):
+ for relation in data[list(format)[i]]:
+ relation_entry = {}
+ for relation_key, relation_value in list(format.values())[i].items():
+ if relation_key != "type":
+ relation_entry[relation_key] = relation.get(relation_value)
+ else:
+ relation_entry[relation_key] = relation_value
+ relations.append(relation_entry)
+ return relations
+
+
+def create_galaxy_and_cluster(galaxy_type, version):
+ api = TidalAPI()
+ galaxy = Galaxy(**GALAXY_CONFIGS[galaxy_type], version=version)
+ galaxy.save_to_file(f"{GALAXY_PATH}/tidal-{galaxy_type}.json")
+
+ cluster = Cluster(**CLUSTER_CONFIGS[galaxy_type], internal_type=galaxy_type)
+ data = api.get_data(galaxy_type)
+ create_cluster_values(data, cluster)
+ cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json")
+
+ print(f"Galaxy {galaxy_type} created")
+
+def create_galaxy(args):
+ if args.all:
+ for galaxy_type in GALAXY_CONFIGS:
+ create_galaxy_and_cluster(galaxy_type, args.version)
+ else:
+ create_galaxy_and_cluster(args.type, args.version)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Create a galaxy and cluster for Tidal API")
+ subparsers = parser.add_subparsers(dest="command")
+
+ galaxy_parser = subparsers.add_parser("create_galaxy", help="Create a galaxy from the Tidal API")
+ galaxy_parser.add_argument("--type", choices=list(GALAXY_CONFIGS.keys()) + ['all'], help="The type of the galaxy")
+ galaxy_parser.add_argument("-v", "--version", type=int, required=True, help="The version of the galaxy")
+ galaxy_parser.add_argument("--all", action="store_true", help="Flag to create all predefined galaxy types")
+ galaxy_parser.set_defaults(func=create_galaxy)
+
+ args = parser.parse_args()
+ if hasattr(args, 'func'):
+ args.func(args)
+ else:
+ parser.print_help()
+
diff --git a/tools/tidal-api/models/__init__.py b/tools/tidal-api/models/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/tidal-api/models/cluster.py b/tools/tidal-api/models/cluster.py
new file mode 100644
index 0000000..2436b89
--- /dev/null
+++ b/tools/tidal-api/models/cluster.py
@@ -0,0 +1,23 @@
+import json
+
+class Cluster():
+ def __init__(self, authors: str, category: str, description: str, name: str, source: str, type: str, uuid: str, values: list, internal_type: str):
+ self.authors = authors
+ self.category = category
+ self.description = description
+ self.name = name
+ self.source = source
+ self.type = type
+ self.uuid = uuid
+ self.values = values
+ self.internal_type = internal_type
+
+ def add_value(self, value):
+ self.values.append(value)
+
+ def save_to_file(self, path):
+ with open(path, "w") as file:
+ file.write(json.dumps(self.__dict__, indent=4))
+
+ def get_config(self):
+ return self.__dict__
diff --git a/tools/tidal-api/models/galaxy.py b/tools/tidal-api/models/galaxy.py
new file mode 100644
index 0000000..2de73b2
--- /dev/null
+++ b/tools/tidal-api/models/galaxy.py
@@ -0,0 +1,14 @@
+import json
+
+class Galaxy():
+ def __init__(self, description, name, namespace, type, uuid, version):
+ self.description = description
+ self.name = name
+ self.namespace = namespace
+ self.type = type
+ self.uuid = uuid
+ self.version = version
+
+ def save_to_file(self, path):
+ with open(path, "w") as file:
+ file.write(json.dumps(self.__dict__, indent=4))
diff --git a/tools/tidal-api/utils/__init__.py b/tools/tidal-api/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/tidal-api/utils/extractor.py b/tools/tidal-api/utils/extractor.py
new file mode 100644
index 0000000..cfa534e
--- /dev/null
+++ b/tools/tidal-api/utils/extractor.py
@@ -0,0 +1,6 @@
+import re
+
+def extract_links(text):
+ links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text)
+ urls = set([url for text, url in links])
+ return urls
\ No newline at end of file