MISP
/
misp-galaxy
mirror of https://github.com/MISP/misp-galaxy
2
0
Fork 0
Code Issues Releases Wiki Activity

chg [config] external config file

pull/941/head
niclas 2024-02-22 10:18:18 +01:00
parent 108e43e1ca
commit b9746f2b41
7 changed files with 339 additions and 397 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
tools/tidal-api/README.md Normal file
View File

@ -0,0 +1,67 @@
# Tidal Cyber API
This is a tool generating MISP galaxies and clusters from Tidal Cyber API.
## Endpoints
https://app-api.tidalcyber.com/api/v1/technique
https://app-api.tidalcyber.com/api/v1/references
https://app-api.tidalcyber.com/api/v1/tactic
https://app-api.tidalcyber.com/api/v1/campaigns/
https://app-api.tidalcyber.com/api/v1/software/
https://app-api.tidalcyber.com/api/v1/groups/
## Configuration
The configuration file is located in `config.json` and maps the fields of the Tidal API to the Galaxy and Cluster fields. It consists of the following sections:
- `UUID`: The UUID of the galaxy to be created
- `GALAXY_CONFIGS`: The configuration of the galaxies to be created in the `galaxies` folder of the MISP-galaxy repository
- `name`: The name of the galaxy
- `namespace`: The namespace of the galaxy
- `description`: The description of the galaxy
- `type`: The type of the galaxy
- `uuid`: The UUID of the galaxy (will be inserted from the `UUID` section)
- `CLUSTER_CONFIGS`: The configuration of the clusters to be created in the `clusters` folder of the MISP-galaxy repository
- `authors`: The authors of the cluster
- `category`: The category of the cluster
- `description`: The description of the cluster
- `name`: The name of the cluster
- `source`: The source of the cluster
- `type`: The type of the cluster
- `uuid`: The UUID of the cluster (will be inserted from the `UUID` section)
- `values`: The values of the cluster (will be inserted from the `VALUE_FIELDS` section)
- `VALUE_FIELDS`: Defines the mapping of the fields in the Tidal Cyber API to the fields in the MISP cluster values array
- `description`: The description of the cluster value
- `meta`: The metadata of the cluster value
- `related`: The related cluster values of the cluster value (you can define a `type` for each relation type in the config which will not be mapped to a field of the API)
- `uuid`: The UUID of the cluster value
- `value`: The value of the cluster value
>Note: The fields `meta` can be formatted as the format of the data the API provides sometimes does not match the format defined by the [MISP galaxy format](https://www.misp-standard.org/rfc/misp-standard-galaxy-format.html#name-conventions-and-terminology). You can configure this using an extraction configuration.
### Extraction Configuration
The extraction configuration is a dictionary that maps the fields of the Tidal Cyber API to the fields of the MISP galaxy. It can be used to extract data stored in a array or object in the API response. The extraction configuration looks like this:
```json
{
"extract": <mode>,
"key": <key>,
"subkey": <subkey>
}
```
**Extract modes**:
- `single`: Extracts a single value from the API response
- `multiple`: Extracts multiple values from the API response
- `reverse`: Gets the value of the key and writes it into an array (no subkey needed)
## Usage
```bash
python3 main.py create-galaxy -v <version> --type <galaxy_to_create>
```
To build all galaxies and clusters, run the following command:
```bash
python3 main.py create-galaxy -v <version> --all
```

249
tools/tidal-api/config.json Normal file
View File

@ -0,0 +1,249 @@
{
"UUIDS": {
"software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7",
"groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6",
"campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48",
"technique": "3e28b683-8159-4398-8729-7248eac9aa45",
"tactic": "16a396e2-a4a9-4dfd-be0a-6ba75fb5382c",
"references": "cf5e180f-26e9-42b2-b10c-c9e55f448750"
},
"GALAXY_CONFIGS": {
"software": {
"name": "Tidal Software",
"namespace": "tidal",
"description": "Tidal Software Galaxy",
"type": "software",
"uuid": ""
},
"groups": {
"name": "Tidal Groups",
"namespace": "tidal",
"description": "Tidal Groups Galaxy",
"type": "groups",
"uuid": ""
},
"campaigns": {
"name": "Tidal Campaigns",
"namespace": "tidal",
"description": "Tidal Campaigns Galaxy",
"type": "campaigns",
"uuid": ""
},
"technique": {
"name": "Tidal Technique",
"namespace": "tidal",
"description": "Tidal Technique Galaxy",
"type": "technique",
"uuid": ""
},
"tactic": {
"name": "Tidal Tactic",
"namespace": "tidal",
"description": "Tidal Tactic Galaxy",
"type": "tactics",
"uuid": ""
},
"references": {
"name": "Tidal References",
"namespace": "tidal",
"description": "Tidal References Galaxy",
"type": "references",
"uuid": ""
}
},
"CLUSTER_CONFIGS": {
"software": {
"authors": "Tidal",
"category": "Software",
"description": "Tidal Software Cluster",
"name": "Tidal Software",
"source": "Tidal",
"type": "software",
"uuid": "",
"values": []
},
"groups": {
"authors": "Tidal",
"category": "Threat Groups",
"description": "Tidal Threat Groups Cluster",
"name": "Tidal Threat Groups",
"source": "Tidal",
"type": "groups",
"uuid": "",
"values": []
},
"campaigns": {
"authors": "Tidal",
"category": "Campaigns",
"description": "Tidal Campaigns Cluster",
"name": "Tidal Campaigns",
"source": "Tidal",
"type": "campaigns",
"uuid": "",
"values": []
},
"technique": {
"authors": "Tidal",
"category": "Techniques",
"description": "Tidal Techniques Cluster",
"name": "Tidal Techniques",
"source": "Tidal",
"type": "technique",
"uuid": "",
"values": []
},
"tactic": {
"authors": "Tidal",
"category": "Tactics",
"description": "Tidal Tactics Cluster",
"name": "Tidal Tactics",
"source": "Tidal",
"type": "tactic",
"uuid": "",
"values": []
},
"references": {
"authors": "Tidal",
"category": "References",
"description": "Tidal References Cluster",
"name": "Tidal References",
"source": "Tidal",
"type": "references",
"uuid": "",
"values": []
}
},
"VALUE_FIELDS": {
"software": {
"description": "description",
"meta": {
"source": "source",
"type": "type",
"software-attack-id": "software_attack_id",
"platforms": "platforms",
"tags": "tags",
"owner": "owner_name"
},
"related": {
"groups": {
"dest-uuid": "group_id",
"type": "used-by"
},
"associated_software": {
"dest-uuid": "id",
"type": "related-to"
}
},
"uuid": "id",
"value": "name"
},
"groups": {
"description": "description",
"meta": {
"source": "source",
"group-attack-id": "group_attack_id",
"country": {
"extract": "single",
"key": "country",
"subkey": "country_code"
},
"observed_country": {
"extract": "multiple",
"key": "observed_country",
"subkey": "country_code"
},
"observed_motivation": {
"extract": "multiple",
"key": "observed_motivation",
"subkey": "name"
},
"target-category": {
"extract": "multiple",
"key": "observed_sector",
"subkey": "name"
},
"tags": "tags",
"owner": "owner_name"
},
"related": {
"associated_groups": {
"dest-uuid": "id",
"type": "related-to"
}
},
"uuid": "id",
"value": "name"
},
"campaigns": {
"description": "description",
"meta": {
"source": "source",
"campaign-attack-id": "campaign_attack_id",
"first_seen": "first_seen",
"last_seen": "last_seen",
"tags": "tags",
"owner": "owner_name"
},
"related": {},
"uuid": "id",
"value": "name"
},
"technique": {
"description": "description",
"meta": {
"source": "source",
"platforms": "platforms",
"tags": "tags",
"owner": "owner_name"
},
"related": {
"tactic": {
"dest-uuid": "tactic_id",
"type": "uses"
},
"sub_technique": {
"dest-uuid": "id",
"type": "sub-technique-of"
}
},
"uuid": "id",
"value": "name"
},
"tactic": {
"description": "description",
"meta": {
"source": "source",
"tactic-attack-id": "tactic_attack_id",
"ordinal_position": "ordinal_position",
"tags": "tags",
"owner": "owner_name"
},
"related": {
"techniques": {
"dest-uuid": "technique_id",
"type": "uses"
}
},
"uuid": "id",
"value": "name"
},
"references": {
"description": "description",
"meta": {
"source": "source",
"refs": {
"extract": "reverse",
"key": "url"
},
"owner": "owner_name",
"title": "title",
"author": "authors",
"date_accessed": "date_accessed",
"date_published": "date_published"
},
"related": {},
"uuid": "id",
"value": "name"
}
}
}

76
tools/tidal-api/create_campaigns.py
View File

@ -1,76 +0,0 @@
from api import TidalAPI
import json
VERSION = 1
GALAXY_PATH = "../../galaxies/"
CLUSTER_PATH = "../../clusters/"
GALAXY_UUID = "43a8fce6-08d3-46c2-957d-53606efe2c48"
def create_galaxy():
galaxy = {}
galaxy["description"] = "Tidal Campaigns Galaxy"
galaxy["name"] = "Tidal Campaigns"
galaxy["namespace"] = "tidal"
galaxy["type"] = "campaigns"
galaxy["uuid"] = GALAXY_UUID
galaxy["version"] = VERSION
return galaxy
def create_cluster(galaxy, data):
cluster = {}
values = []
for campaigns in data["data"]:
value = {}
relations = []
value["description"] = campaigns["description"]
# Metadata fields
source = campaigns["source"]
campaign_attack_id = campaigns["campaign_attack_id"]
first_seen = campaigns["first_seen"]
last_seen = campaigns["last_seen"]
tags = campaigns["tags"]
owner = campaigns["owner_name"]
value["meta"] = {}
if source:
value["meta"]["source"] = source
if campaign_attack_id:
value["meta"]["campaign-attack-id"] = campaign_attack_id
if first_seen:
value["meta"]["first-seen"] = first_seen
if last_seen:
value["meta"]["last-seen"] = last_seen
if tags:
value["meta"]["tags"] = tags
if owner:
value["meta"]["owner"] = owner
value["related"] = relations
value["uuid"] = campaigns["id"]
value["value"] = campaigns["name"]
values.append(value)
cluster["authors"] = ["Tidal"]
cluster["category"] = "Threat campaigns"
cluster["description"] = "Tidal Campaigns"
cluster["name"] = "Tidal Campaigns"
cluster["source"] = "https://app-api.tidalcyber.com/api/v1/campaigns"
cluster["type"] = "campaigns"
cluster["uuid"] = galaxy["uuid"]
cluster["values"] = values
return cluster
if __name__ == "__main__":
api = TidalAPI()
data = api.get_data('campaigns')
galaxy = create_galaxy()
cluster = create_cluster(galaxy, data)
with open(GALAXY_PATH + "tidal-campaigns.json", "w") as galaxy_file:
json.dump(galaxy, galaxy_file, indent=4)
with open(CLUSTER_PATH + "tidal-campaigns.json", "w") as cluster_file:
json.dump(cluster, cluster_file, indent=4)

91
tools/tidal-api/create_groups.py
View File

@ -1,91 +0,0 @@
from api import TidalAPI
import json
VERSION = 1
GALAXY_PATH = "../../galaxies/"
CLUSTER_PATH = "../../clusters/"
GALAXY_UUID = "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6"
def create_galaxy():
galaxy = {}
galaxy["description"] = "Tidal Threat Group Galaxy"
galaxy["name"] = "Tidal Threat Group"
galaxy["namespace"] = "tidal"
galaxy["type"] = "threat-group"
galaxy["uuid"] = GALAXY_UUID
galaxy["version"] = VERSION
return galaxy
def create_cluster(galaxy, data):
cluster = {}
values = []
for group in data["data"]:
value = {}
relations = []
# TODO check for id and associated_group_id and add to relations
for entry in group["associated_groups"]:
relation = {}
relation["dest-uuid"] = entry["id"]
relation["type"] = "related-to"
relations.append(relation)
value["description"] = group["description"]
# Metadata fields
source = group["source"]
group_attack_id = group["group_attack_id"]
country = [country["country_name"] for country in group["country"]]
observed_country = [country["country_code"] for country in group["observed_country"]]
motive = [motive["name"] for motive in group["observed_motivation"]]
target_category = [sector["name"] for sector in group["observed_sector"]]
tags = group["tags"]
owner = group["owner_name"]
value["meta"] = {}
if source:
value["meta"]["source"] = source
if group_attack_id:
value["meta"]["group-attack-id"] = group_attack_id
if country:
value["meta"]["country"] = country
if observed_country:
value["meta"]["observed_country"] = observed_country
if motive:
value["meta"]["motive"] = motive
if target_category:
value["meta"]["target-category"] = target_category
if tags:
value["meta"]["tags"] = tags
if owner:
value["meta"]["owner"] = owner
value["related"] = relations
value["uuid"] = group["id"]
value["value"] = group["name"]
values.append(value)
cluster["authors"] = ["Tidal"]
cluster["category"] = "Threat Group"
cluster["description"] = "Tidal Threat Groups"
cluster["name"] = "Tidal Threat Group"
cluster["source"] = "https://app-api.tidalcyber.com/api/v1/groups"
cluster["type"] = "threat-group"
cluster["uuid"] = galaxy["uuid"]
cluster["values"] = values
return cluster
if __name__ == "__main__":
api = TidalAPI()
data = api.get_data("groups")
galaxy = create_galaxy()
cluster = create_cluster(galaxy, data)
with open(GALAXY_PATH + "tidal-threat-groups.json", "w") as galaxy_file:
json.dump(galaxy, galaxy_file, indent=4)
with open(CLUSTER_PATH + "tidal-threat-groups.json", "w") as cluster_file:
json.dump(cluster, cluster_file, indent=4)

106
tools/tidal-api/create_software.py
View File

@ -1,106 +0,0 @@
from api import TidalAPI
import json
import re
VERSION = 1
GALAXY_PATH = "../../galaxies/"
CLUSTER_PATH = "../../clusters/"
GALAXY_UUID = "38d62d8b-4c49-489a-9bc4-8e294c4f04f7"
def create_galaxy():
galaxy = {}
galaxy["description"] = "Tidal Software Galaxy"
galaxy["name"] = "Tidal Software"
galaxy["namespace"] = "tidal"
galaxy["type"] = "software"
galaxy["uuid"] = GALAXY_UUID
galaxy["version"] = VERSION
return galaxy
def create_cluster(galaxy, data):
cluster = {}
values = []
for software in data["data"]:
value = {}
relations = []
# TODO check for relations etc.
for entry in software["groups"]:
relation = {}
relation["dest-uuid"] = entry["id"]
relation["type"] = "used-by"
relations.append(relation)
for entry in software["associated_software"]:
relation = {}
relation["dest-uuid"] = entry["id"]
relation["type"] = "related-to"
relations.append(relation)
value["description"] = software["description"]
# Metadata fields
links = extract_links(software["description"])
source = software["source"]
type = software["type"]
software_attack_id = software["software_attack_id"]
platforms = software["platforms"]
tags = software["tags"]
owner = software["owner_name"]
value["meta"] = {}
if links:
value["meta"]["refs"] = list(links)
if source:
value["meta"]["source"] = source
if type:
value["meta"]["type"] = type
if software_attack_id:
value["meta"]["software-attack-id"] = software_attack_id
if platforms:
value["meta"]["platforms"] = platforms
if tags:
value["meta"]["tags"] = tags
if owner:
value["meta"]["owner"] = owner
value["related"] = relations
value["uuid"] = software["id"]
value["value"] = software["name"]
values.append(value)
cluster["authors"] = ["Tidal"]
cluster["category"] = "Threat software"
cluster["description"] = "Tidal Threat Groups"
cluster["name"] = "Tidal Threat software"
cluster["source"] = "https://app-api.tidalcyber.com/api/v1/software"
cluster["type"] = "threat-software"
cluster["uuid"] = galaxy["uuid"]
cluster["values"] = values
return cluster
def extract_links(text):
# extract markdown links and return text without links and the links
# urls = re.findall(r'https?://[^\s\)]+', text)
regular_links = re.findall(r'\[([^\]]+)\]\((https?://[^\s\)]+)\)', text)
# sup_links = re.findall(r'<sup>\[\[([^\]]+)\]\((https?://[^\s\)]+)\)\]</sup>', text)
# Extracting URLs from the tuples
regular_links_urls = set([url for text, url in regular_links])
# sup_links_urls = [url for text, url in sup_links]
# text_without_links = re.sub(r'\[([^\]]+)\]\(https?://[^\s\)]+\)', r'\1', text)
# text_without_sup = re.sub(r'<sup>.*<\/sup>', '', text_without_links)
return regular_links_urls
if __name__ == "__main__":
api = TidalAPI()
data = api.get_data('software')
galaxy = create_galaxy()
cluster = create_cluster(galaxy, data)
with open(GALAXY_PATH + "tidal-software.json", "w") as galaxy_file:
json.dump(galaxy, galaxy_file, indent=4)
with open(CLUSTER_PATH + "tidal-software.json", "w") as cluster_file:
json.dump(cluster, cluster_file, indent=4)

133
tools/tidal-api/main.py
View File

@ -2,135 +2,18 @@ from api.api import TidalAPI
from models.galaxy import Galaxy
from models.cluster import Cluster
from utils.extractor import extract_links
from utils.config import load_config
import argparse
CLUSTER_PATH = "../../clusters/"
GALAXY_PATH = "../../galaxies/"
UUIDS = {
"software": "38d62d8b-4c49-489a-9bc4-8e294c4f04f7",
"groups": "41c3e5c0-de5c-4edb-b48b-48cd8e7519e6",
"campaigns": "43a8fce6-08d3-46c2-957d-53606efe2c48",
}
config = load_config('./config.json')
GALAXY_CONFIGS = {
"software": {
"name": "Tidal Software",
"namespace": "tidal",
"description": "Tidal Software Galaxy",
"type": "software",
"uuid": UUIDS["software"],
},
"groups": {
"name": "Tidal Groups",
"namespace": "tidal",
"description": "Tidal Groups Galaxy",
"type": "groups",
"uuid": UUIDS["groups"],
},
"campaigns": {
"name": "Tidal Campaigns",
"namespace": "tidal",
"description": "Tidal Campaigns Galaxy",
"type": "campaigns",
"uuid": UUIDS["campaigns"],
}
}
CLUSTER_CONFIGS = {
"software": {
"authors": "Tidal",
"category": "Software",
"description": "Tidal Software Cluster",
"name": "Tidal Software",
"source": "Tidal",
"type": "software",
"uuid": UUIDS["software"],
"values": []
},
"groups": {
"authors": "Tidal",
"category": "Threat Groups",
"description": "Tidal Threat Groups Cluster",
"name": "Tidal Threat Groups",
"source": "Tidal",
"type": "groups",
"uuid": UUIDS["groups"],
"values": []
},
"campaigns": {
"authors": "Tidal",
"category": "Campaigns",
"description": "Tidal Campaigns Cluster",
"name": "Tidal Campaigns",
"source": "Tidal",
"type": "campaigns",
"uuid": UUIDS["campaigns"],
"values": []
}
}
VALUE_FIELDS = {
"software": {
"description": "description",
"meta": {
"source": "source",
"type": "type",
"software-attack-id": "software_attack_id",
"platforms": "platforms",
"tags": "tags",
"owner": "owner_name"
},
"related": {
"groups": {
"dest-uuid": "group_id",
"type": "used-by"
},
"associated_software": {
"dest-uuid": "id",
"type": "related-to"
}
},
"uuid": "id",
"value": "name"
},
"groups": {
"description": "description",
"meta": {
"source": "source",
"group-attack-id": "group_attack_id",
"country": {"extract": "single", "key": "country", "subkey": "country_code"},
"observed_country": {"extract": "multiple", "key": "observed_country", "subkey": "country_code"},
"observed_motivation": {"extract": "multiple", "key": "observed_motivation", "subkey": "name"},
"target-category": {"extract": "multiple", "key": "observed_sector", "subkey": "name"},
"tags": "tags",
"owner": "owner_name"
},
"related": {
"associated_groups": {
"dest-uuid": "id",
"type": "related-to"
}
},
"uuid": "id",
"value": "name"
},
"campaigns": {
"description": "description",
"meta": {
"source": "source",
"campaign-attack-id": "campaign_attack_id",
"first_seen": "first_seen",
"last_seen": "last_seen",
"tags": "tags",
"owner": "owner_name"
},
"related": {},
"uuid": "id",
"value": "name"
}
}
UUIDS = config['UUIDS']
GALAXY_CONFIGS = config['GALAXY_CONFIGS']
CLUSTER_CONFIGS = config['CLUSTER_CONFIGS']
VALUE_FIELDS = config['VALUE_FIELDS']
def create_cluster_values(data, cluster):
value_fields = VALUE_FIELDS[cluster.internal_type]
@ -162,6 +45,8 @@ def create_metadata(data, format):
metadata[meta_key] = data.get(meta_value["key"])[0].get(meta_value["subkey"])
elif meta_value.get("extract") == "multiple" and data.get(meta_value["key"]):
metadata[meta_key] = [entry.get(meta_value["subkey"]) for entry in data.get(meta_value["key"])]
elif meta_value.get("extract") == "reverse" and data.get(meta_value["key"]):
metadata[meta_key] = [data.get(meta_value["key"])]
elif data.get(meta_value):
metadata[meta_key] = data.get(meta_value)
return metadata
@ -191,7 +76,7 @@ def create_galaxy_and_cluster(galaxy_type, version):
create_cluster_values(data, cluster)
cluster.save_to_file(f"{CLUSTER_PATH}/tidal-{galaxy_type}.json")
print(f"Galaxy {galaxy_type} created")
print(f"Galaxy tidal-{galaxy_type} created")
def create_galaxy(args):
if args.all:

14
tools/tidal-api/utils/config.py Normal file
View File

@ -0,0 +1,14 @@
import json
def load_config(file_path):
with open(file_path, 'r') as file:
config = json.load(file)
return link_uuids(config)
def link_uuids(config):
uuids = config["UUIDS"]
for key, galaxy_config in config["GALAXY_CONFIGS"].items():
galaxy_config["uuid"] = uuids[key]
for key, cluster_config in config["CLUSTER_CONFIGS"].items():
cluster_config["uuid"] = uuids[key]
return config