new: [mitre] New MITRE ATLAS Galaxy

pull/908/head
Christophe Vandeplas 2023-12-18 12:28:30 +01:00
parent 8ec38b97e4
commit adb9c2a052
No known key found for this signature in database
GPG Key ID: BDC48619FFDC5A5B
6 changed files with 2600 additions and 1 deletions

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,723 @@
{
"authors": [
"MITRE"
],
"category": "course-of-action",
"description": "MITRE ATLAS Mitigation - Adversarial Threat Landscape for Artificial-Intelligence Systems",
"name": "MITRE ATLAS Course of Action",
"source": "https://github.com/mitre-atlas/atlas-navigator-data",
"type": "mitre-atlas-course-of-action",
"uuid": "951d5a45-43c2-422b-90af-059014f15714",
"values": [
{
"description": "Establish access controls on internal model registries and limit internal access to production models. Limit access to training data only to approved users.\n",
"meta": {
"external_id": "AML.M0005",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0005"
]
},
"related": [
{
"dest-uuid": "0d002b6b-d006-4aab-a7f9-fa69f4a1e675",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "2792e1f0-3132-4876-878d-a900b8a40e7d",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "481486ed-846c-43ce-931b-86b8a18556b0",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "666f4d33-1a62-4ad7-9bf9-6387cd3f1fd7",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "68034561-a079-4052-9b64-427bfcff76ff",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "6945b742-f1d5-4a83-ba4a-d0e0de6620c3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "822cb1e2-f35f-4b35-a650-59b7770d4abc",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "da785068-ece5-4c52-b77d-39e1b24cb6d7",
"value": "Control Access to ML Models and Data at Rest - AML.M0005"
},
{
"description": "Limit the total number and rate of queries a user can perform.\n",
"meta": {
"external_id": "AML.M0004",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0004"
]
},
"related": [
{
"dest-uuid": "1cc7f877-cb60-419a-bd1e-32b704b534d0",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "3247b43f-1888-4158-b3da-5b7c7dfaa4e2",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "3b829988-8bdb-4c4e-a4dd-500a3d3fd3e4",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "569d6edd-0140-4ab2-97b1-3635d62f40cc",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "65c5e3b8-9296-46a2-ae7d-1b68a79cbe54",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "79cdc11c-2ca9-4a6a-96a0-18bd84943086",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "83c5ba15-5312-4c7d-bbb4-f9c4f2c6ffca",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8a115a02-2b88-4a3e-9212-a39dc086320b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "b5d1fd4f-861f-43e0-b1ca-ee8a3b47f7e1",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "ba5645e5-d1ab-4f1f-8b82-cb0792543fa8",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "4a048bfe-dab5-434b-86cc-f4586951ec0d",
"value": "Restrict Number of ML Model Queries - AML.M0004"
},
{
"description": "Limit the public release of technical information about the machine learning stack used in an organization's products or services. Technical knowledge of how machine learning is used can be leveraged by adversaries to perform targeting and tailor attacks to the target system. Additionally, consider limiting the release of organizational information - including physical locations, researcher names, and department structures - from which technical details such as machine learning techniques, model architectures, or datasets may be inferred.\n",
"meta": {
"external_id": "AML.M0000",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0000"
]
},
"related": [
{
"dest-uuid": "229ead06-da1e-443c-8ff1-e57a3ae0eb61",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "d93b2175-90a8-4250-821f-dcc3bbbe194c",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "0b016f6f-2f61-493c-bf9d-02cad4c027df",
"value": "Limit Release of Public Information - AML.M0000"
},
{
"description": "Limit public release of technical project details including data, algorithms, model architectures, and model checkpoints that are used in production, or that are representative of those used in production.\n",
"meta": {
"external_id": "AML.M0001",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0001"
]
},
"related": [
{
"dest-uuid": "292ebe33-addc-4fe7-b2a9-4856293c4c96",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "6945b742-f1d5-4a83-ba4a-d0e0de6620c3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "6a7f4fc2-272b-4f86-b137-70fa3e239f58",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "c0f65fa8-8e05-4481-b934-ff2c452ae8c3",
"value": "Limit Model Artifact Release - AML.M0001"
},
{
"description": "Decreasing the fidelity of model outputs provided to the end user can reduce an adversaries ability to extract information about the model and optimize attacks for the model.\n",
"meta": {
"external_id": "AML.M0002",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0002"
]
},
"related": [
{
"dest-uuid": "569d6edd-0140-4ab2-97b1-3635d62f40cc",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "65c5e3b8-9296-46a2-ae7d-1b68a79cbe54",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "79cdc11c-2ca9-4a6a-96a0-18bd84943086",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "83c5ba15-5312-4c7d-bbb4-f9c4f2c6ffca",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8a115a02-2b88-4a3e-9212-a39dc086320b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "b5d1fd4f-861f-43e0-b1ca-ee8a3b47f7e1",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "6b53cb14-eade-4760-8dae-75164e62cb7e",
"value": "Passive ML Output Obfuscation - AML.M0002"
},
{
"description": "Incorporate multiple sensors to integrate varying perspectives and modalities to avoid a single point of failure susceptible to physical attacks.\n",
"meta": {
"external_id": "AML.M0009",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0009"
]
},
"related": [
{
"dest-uuid": "bb747632-d988-45ff-9cb3-97d827b4d9db",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "e0958449-a880-4410-bbb1-fa102030a883",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "532918ce-83cf-4f6f-86fa-8ad4024e91ab",
"value": "Use Multi-Modal Sensors - AML.M0009"
},
{
"description": "Use an ensemble of models for inference to increase robustness to adversarial inputs. Some attacks may effectively evade one model or model family but be ineffective against others.\n",
"meta": {
"external_id": "AML.M0006",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0006"
]
},
"related": [
{
"dest-uuid": "2792e1f0-3132-4876-878d-a900b8a40e7d",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "4627c4e6-fb06-4bfa-add5-dc46e0043aff",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8a115a02-2b88-4a3e-9212-a39dc086320b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8bcf7648-2683-421d-b623-bc539de59cb3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "bb747632-d988-45ff-9cb3-97d827b4d9db",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "de7a696b-f688-454c-bf61-476a68b50e9f",
"value": "Use Ensemble Methods - AML.M0006"
},
{
"description": "Detect and remove or remediate poisoned training data. Training data should be sanitized prior to model training and recurrently for an active learning model.\n\nImplement a filter to limit ingested training data. Establish a content policy that would remove unwanted content such as certain explicit or offensive language from being used.\n",
"meta": {
"external_id": "AML.M0007",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0007"
]
},
"related": [
{
"dest-uuid": "666f4d33-1a62-4ad7-9bf9-6387cd3f1fd7",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "6945b742-f1d5-4a83-ba4a-d0e0de6620c3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "822cb1e2-f35f-4b35-a650-59b7770d4abc",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "7e20b527-6299-4ee3-863e-59fee7cdaa9a",
"value": "Sanitize Training Data - AML.M0007"
},
{
"description": "Validate that machine learning models perform as intended by testing for backdoor triggers or adversarial bias.\n",
"meta": {
"external_id": "AML.M0008",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0008"
]
},
"related": [
{
"dest-uuid": "2792e1f0-3132-4876-878d-a900b8a40e7d",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "68034561-a079-4052-9b64-427bfcff76ff",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "822cb1e2-f35f-4b35-a650-59b7770d4abc",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "32bd077a-90ce-4e97-ad40-8f130a1a7dab",
"value": "Validate ML Model - AML.M0008"
},
{
"description": "Prevent abuse of library loading mechanisms in the operating system and software to load untrusted code by configuring appropriate library loading mechanisms and investigating potential vulnerable software.\n\nFile formats such as pickle files that are commonly used to store machine learning models can contain exploits that allow for loading of malicious libraries.\n",
"meta": {
"external_id": "AML.M0011",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0011"
]
},
"related": [
{
"dest-uuid": "d52b913b-808c-461d-8969-94cd5c9fe07b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "6cd8c9ca-bd46-489f-9ccb-5b76b8ef580e",
"value": "Restrict Library Loading - AML.M0011"
},
{
"description": "Encrypt sensitive data such as ML models to protect against adversaries attempting to access sensitive data.\n",
"meta": {
"external_id": "AML.M0012",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0012"
]
},
"related": [
{
"dest-uuid": "0d002b6b-d006-4aab-a7f9-fa69f4a1e675",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "529fac49-5f88-4a3c-829f-eb50cb90bcf1",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "b67fc223-fecf-4ee6-9de7-9392d9f04060",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "8bba19a7-fc6f-4381-8b34-2d43cdc14627",
"value": "Encrypt Sensitive Information - AML.M0012"
},
{
"description": "Verify the cryptographic checksum of all machine learning artifacts to verify that the file was not modified by an attacker.\n",
"meta": {
"external_id": "AML.M0014",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0014"
]
},
"related": [
{
"dest-uuid": "0799f2f2-1038-4391-ba1f-4117595db45a",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "b6697dbf-3e3f-41ce-a212-361d1c0ca0e9",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "d52b913b-808c-461d-8969-94cd5c9fe07b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "a861f658-4203-48ba-bdca-fe068518eefb",
"value": "Verify ML Artifacts - AML.M0014"
},
{
"description": "Detect and block adversarial inputs or atypical queries that deviate from known benign behavior, exhibit behavior patterns observed in previous attacks or that come from potentially malicious IPs.\nIncorporate adversarial detection algorithms into the ML system prior to the ML model.\n",
"meta": {
"external_id": "AML.M0015",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0015"
]
},
"related": [
{
"dest-uuid": "1cc7f877-cb60-419a-bd1e-32b704b534d0",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "79cdc11c-2ca9-4a6a-96a0-18bd84943086",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8bcf7648-2683-421d-b623-bc539de59cb3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "bb747632-d988-45ff-9cb3-97d827b4d9db",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "825f21ab-f3c9-46ce-b539-28f295f519f8",
"value": "Adversarial Input Detection - AML.M0015"
},
{
"description": "Deploying ML models to edge devices can increase the attack surface of the system. Consider serving models in the cloud to reduce the level of access the adversary has to the model.\n",
"meta": {
"external_id": "AML.M0017",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0017"
]
},
"related": [
{
"dest-uuid": "2792e1f0-3132-4876-878d-a900b8a40e7d",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "51c95da5-d7f1-4b57-9229-869b80305b37",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "afcd723a-e5ff-4c09-8f72-fe16f7345af7",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "79316871-3bf9-4a59-b517-b0156e84fcb4",
"value": "Model Distribution Methods - AML.M0017"
},
{
"description": "Preprocess all inference data to nullify or reverse potential adversarial perturbations.\n",
"meta": {
"external_id": "AML.M0010",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0010"
]
},
"related": [
{
"dest-uuid": "79cdc11c-2ca9-4a6a-96a0-18bd84943086",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "8bcf7648-2683-421d-b623-bc539de59cb3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "bb747632-d988-45ff-9cb3-97d827b4d9db",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "88aea80f-498f-403d-b82f-e76c44f9da94",
"value": "Input Restoration - AML.M0010"
},
{
"description": "Use techniques to make machine learning models robust to adversarial inputs such as adversarial training or network distillation.\n",
"meta": {
"external_id": "AML.M0003",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0003"
]
},
"related": [
{
"dest-uuid": "8bcf7648-2683-421d-b623-bc539de59cb3",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "bb747632-d988-45ff-9cb3-97d827b4d9db",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "04e9bb75-1b7e-4825-bc3f-774850d3c1ef",
"value": "Model Hardening - AML.M0003"
},
{
"description": "Enforce binary and application integrity with digital signature verification to prevent untrusted code from executing. Adversaries can embed malicious code in ML software or models. Enforcement of code signing can prevent the compromise of the machine learning supply chain and prevent execution of malicious code.\n",
"meta": {
"external_id": "AML.M0013",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0013"
]
},
"related": [
{
"dest-uuid": "2792e1f0-3132-4876-878d-a900b8a40e7d",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "4627c4e6-fb06-4bfa-add5-dc46e0043aff",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "d52b913b-808c-461d-8969-94cd5c9fe07b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "c55ed072-eca7-41d6-b5e0-68c10753544d",
"value": "Code Signing - AML.M0013"
},
{
"description": "Vulnerability scanning is used to find potentially exploitable software vulnerabilities to remediate them.\n\nFile formats such as pickle files that are commonly used to store machine learning models can contain exploits that allow for arbitrary code execution.\n",
"meta": {
"external_id": "AML.M0016",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0016"
]
},
"related": [
{
"dest-uuid": "d52b913b-808c-461d-8969-94cd5c9fe07b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "e2cb599d-2714-4673-bc1a-976c471d7c58",
"value": "Vulnerability Scanning - AML.M0016"
},
{
"description": "Educate ML model developers on secure coding practices and ML vulnerabilities.\n",
"meta": {
"external_id": "AML.M0018",
"refs": [
"https://atlas.mitre.org/mitigations/AML.M0018"
]
},
"related": [
{
"dest-uuid": "5e8e4108-beb6-479a-a617-323d425e5d03",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
},
{
"dest-uuid": "d52b913b-808c-461d-8969-94cd5c9fe07b",
"tags": [
"estimative-language:likelihood-probability=\"almost-certain\""
],
"type": "mitigates"
}
],
"uuid": "8c2cb25a-46b0-4551-beeb-21e8425a48bd",
"value": "User Training - AML.M0018"
}
],
"version": 5
}

View File

@ -0,0 +1,27 @@
{
"description": "MITRE ATLAS Attack Pattern - Adversarial Threat Landscape for Artificial-Intelligence Systems",
"icon": "map",
"kill_chain_order": {
"mitre-atlas": [
"collection",
"credential-access",
"defense-evasion",
"discovery",
"execution",
"exfiltration",
"impact",
"initial-access",
"ml-attack-staging",
"ml-model-access",
"persistence",
"privilege-escalation",
"reconnaissance",
"resource-development"
]
},
"name": "MITRE ATLAS Attack Pattern",
"namespace": "mitre-atlas",
"type": "mitre-atlas-attack-pattern",
"uuid": "3f3d21aa-d8a1-4f8f-b31e-fc5425eec821",
"version": 1
}

View File

@ -0,0 +1,9 @@
{
"description": "MITRE ATLAS Mitigation - Adversarial Threat Landscape for Artificial-Intelligence Systems",
"icon": "link",
"name": "MITRE ATLAS Course of Action",
"namespace": "mitre-atlas",
"type": "mitre-atlas-course-of-action",
"uuid": "29d13ede-9667-415c-bb75-b34a4bd89a81",
"version": 1
}

View File

@ -4,7 +4,7 @@ import re
import os
import argparse
parser = argparse.ArgumentParser(description='Create a couple galaxy/cluster with cti\'s intrusion-sets\nMust be in the mitre/cti/enterprise-attack/intrusion-set folder')
parser = argparse.ArgumentParser(description='Create a couple galaxy/cluster with cti\'s intrusion-sets\nMust be in the tools folder')
parser.add_argument("-p", "--path", required=True, help="Path of the mitre/cti folder")
args = parser.parse_args()

198
tools/gen_mitre_atlas.py Executable file
View File

@ -0,0 +1,198 @@
#!/usr/bin/env python3
import json
import re
import os
import argparse
parser = argparse.ArgumentParser(description='Create a couple galaxy/cluster with MITRE ATLAS - Adversarial Threat Landscape for Artificial-Intelligence Systems\nMust be in the tools folder')
parser.add_argument("-p", "--path", required=True, help="Path of the mitre atlas-navigator-data folder")
args = parser.parse_args()
values = []
misp_dir = '../'
# domains = ['enterprise-attack', 'mobile-attack', 'pre-attack']
types = ['attack-pattern', 'course-of-action']
mitre_sources = ['mitre-atlas']
all_data = {} # variable that will contain everything
# read in the non-MITRE data
# we need this to be able to build a list of non-MITRE-UUIDs which we will use later on
# to remove relations that are from MITRE.
# the reasoning is that the new MITRE export might contain less relationships than it did before
# so we cannot migrate all existing relationships as such
non_mitre_uuids = set()
for fname in os.listdir(os.path.join(misp_dir, 'clusters')):
if 'mitre' in fname:
continue
if '.json' in fname:
# print(fname)
with open(os.path.join(misp_dir, 'clusters', fname)) as f_in:
cluster_data = json.load(f_in)
for cluster in cluster_data['values']:
non_mitre_uuids.add(cluster['uuid'])
# read in existing MITRE data
# first build a data set of the MISP Galaxy ATT&CK elements by using the UUID as reference, this speeds up lookups later on.
# at the end we will convert everything again to separate datasets
all_data_uuid = {}
for t in types:
fname = os.path.join(misp_dir, 'clusters', 'mitre-atlas-{}.json'.format(t))
if os.path.exists(fname):
# print("##### {}".format(fname))
with open(fname) as f:
file_data = json.load(f)
# print(file_data)
for value in file_data['values']:
# remove (old)MITRE relations, and keep non-MITRE relations
if 'related' in value:
related_original = value['related']
related_new = []
for rel in related_original:
if rel['dest-uuid'] in non_mitre_uuids:
related_new.append(rel)
value['related'] = related_new
# find and handle duplicate uuids
if value['uuid'] in all_data_uuid:
# exit("ERROR: Something is really wrong, we seem to have duplicates.")
# if it already exists we need to copy over all the data manually to merge it
# on the other hand, from a manual analysis it looks like it's mostly the relations that are different
# so now we will just copy over the relationships
# actually, at time of writing the code below results in no change as the new items always contained more than the previously seen items
value_orig = all_data_uuid[value['uuid']]
if 'related' in value_orig:
for related_item in value_orig['related']:
if related_item not in value['related']:
value['related'].append(related_item)
all_data_uuid[value['uuid']] = value
# now load the MITRE ATT&CK
attack_dir = os.path.join(args.path, 'dist')
if not os.path.exists(attack_dir):
exit("ERROR: MITRE ATT&CK folder incorrect")
with open(os.path.join(attack_dir, 'stix-atlas.json')) as f:
attack_data = json.load(f)
for item in attack_data['objects']:
if item['type'] not in types:
continue
# print(json.dumps(item, indent=2, sort_keys=True, ensure_ascii=False))
try:
# build the new data structure
value = {}
uuid = re.search('--(.*)$', item['id']).group(0)[2:]
# item exist already in the all_data set
update = False
if uuid in all_data_uuid:
value = all_data_uuid[uuid]
if 'description' in item:
value['description'] = item['description']
value['value'] = item['name'] + ' - ' + item['external_references'][0]['external_id']
value['meta'] = {}
value['meta']['refs'] = []
value['uuid'] = re.search('--(.*)$', item['id']).group(0)[2:]
for reference in item['external_references']:
if 'url' in reference and reference['url'] not in value['meta']['refs']:
value['meta']['refs'].append(reference['url'])
# Find Mitre external IDs from allowed sources
if 'external_id' in reference and reference.get("source_name", None) in mitre_sources:
value['meta']['external_id'] = reference['external_id']
if not value['meta'].get('external_id', None):
exit("Entry is missing an external ID, please update mitre_sources. Available references: {}".format(
json.dumps(item['external_references'])
))
if 'kill_chain_phases' in item: # many (but not all) attack-patterns have this
value['meta']['kill_chain'] = []
for killchain in item['kill_chain_phases']:
value['meta']['kill_chain'].append(killchain['kill_chain_name'] + ':' + killchain['phase_name'])
if 'x_mitre_data_sources' in item:
value['meta']['mitre_data_sources'] = item['x_mitre_data_sources']
if 'x_mitre_platforms' in item:
value['meta']['mitre_platforms'] = item['x_mitre_platforms']
# TODO add the other x_mitre elements dynamically
# relationships will be build separately afterwards
value['type'] = item['type'] # remove this before dump to json
# print(json.dumps(value, sort_keys=True, indent=2))
all_data_uuid[uuid] = value
except Exception as e:
print(json.dumps(item, sort_keys=True, indent=2))
import traceback
traceback.print_exc()
# process the 'relationship' type as we now know the existence of all ATT&CK uuids
for item in attack_data['objects']:
if item['type'] != 'relationship':
continue
# print(json.dumps(item, indent=2, sort_keys=True, ensure_ascii=False))
rel_type = item['relationship_type']
dest_uuid = re.findall(r'--([0-9a-f-]+)', item['target_ref']).pop()
source_uuid = re.findall(r'--([0-9a-f-]+)', item['source_ref']).pop()
tags = []
# add the relation in the defined way
rel_source = {
"dest-uuid": dest_uuid,
"type": rel_type
}
if rel_type != 'subtechnique-of':
rel_source['tags'] = [
"estimative-language:likelihood-probability=\"almost-certain\""
]
try:
if 'related' not in all_data_uuid[source_uuid]:
all_data_uuid[source_uuid]['related'] = []
if rel_source not in all_data_uuid[source_uuid]['related']:
all_data_uuid[source_uuid]['related'].append(rel_source)
except KeyError:
pass # ignore relations from which we do not know the source
# LATER find the opposite word of "rel_type" and build the relation in the opposite direction
# dump all_data to their respective file
for t in types:
fname = os.path.join(misp_dir, 'clusters', 'mitre-atlas-{}.json'.format(t))
if not os.path.exists(fname):
exit("File {} does not exist, this is unexpected.".format(fname))
with open(fname) as f:
file_data = json.load(f)
file_data['values'] = []
for item in all_data_uuid.values():
# print(json.dumps(item, sort_keys=True, indent=2))
if 'type' not in item or item['type'] != t: # drop old data or not from the right type
continue
item_2 = item.copy()
item_2.pop('type', None)
file_data['values'].append(item_2)
# FIXME the sort algo needs to be further improved, potentially with a recursive deep sort
file_data['values'] = sorted(file_data['values'], key=lambda x: sorted(x['value']))
for item in file_data['values']:
if 'related' in item:
item['related'] = sorted(item['related'], key=lambda x: x['dest-uuid'])
if 'meta' in item:
if 'refs' in item['meta']:
item['meta']['refs'] = sorted(item['meta']['refs'])
if 'mitre_data_sources' in item['meta']:
item['meta']['mitre_data_sources'] = sorted(item['meta']['mitre_data_sources'])
file_data['version'] += 1
with open(fname, 'w') as f:
json.dump(file_data, f, indent=2, sort_keys=True, ensure_ascii=False)
f.write('\n') # only needed for the beauty and to be compliant with jq_all_the_things
print("All done, please don't forget to ./jq_all_the_things.sh, commit, and then ./validate_all.sh.")