Merge branch 'master' of github.com:MISP/misp-taxonomies

2017-08-16 12:06:06 +02:00 · 2017-08-16 12:06:06 +02:00 · c062f77fea
parent b9e65d55fe 05ce75f7fe
commit c062f77fea
16 changed files with 1230 additions and 5723 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -4,12 +4,7 @@ cache: pip

 sudo: required

-dist: trusty
-
 python:
-    - "3.4"
-    - "3.5"
-    - "3.5-dev"
    - "3.6"
    - "3.6-dev"
    - "nightly"
@ -18,10 +13,7 @@ install:
    - sudo apt-get update -qq
    - sudo apt-get install -y -qq jq moreutils
    - pip install jsonschema
-    - git clone https://github.com/MISP/PyTaxonomies.git
-    - pushd PyTaxonomies
-    - pip install .
-    - popd
+    - pip install git+https://github.com/MISP/PyTaxonomies.git

 script:
    - ./validate_all.sh
--- a/MANIFEST.json
+++ b/MANIFEST.json
@ -1,5 +1,10 @@
 {
  "taxonomies": [
+    {
+      "version": 1,
+      "name": "accessnow",
+      "description": "Access Now"
+    },
    {
      "version": 1,
      "name": "admiralty-scale",
@ -10,6 +15,16 @@
      "name": "adversary",
      "description": "An overview and description of the adversary infrastructure."
    },
+    {
+      "version": 1,
+      "name": "analyst-assessment",
+      "description": ""
+    },
+    {
+      "version": 1,
+      "name": "binary-class",
+      "description": ""
+    },
    {
      "version": 1,
      "name": "circl",
@ -20,6 +35,11 @@
      "name": "csirt_case_classification",
      "description": "FIRST CSIRT Case Classification."
    },
+    {
+      "version": 1,
+      "name": "cssa",
+      "description": ""
+    },
    {
      "version": 1,
      "name": "de-vs",
--- a/README.md
+++ b/README.md
@ -13,6 +13,7 @@ The following taxonomies are described:
 - [Admiralty Scale](./admiralty-scale)
 - [adversary](./adversary) - description of an adversary infrastructure
 - CIRCL [Taxonomy - Schemes of Classification in Incident Response and Detection](./circl)
+- [The CSSA agreed sharing taxonomy](./cssa)
 - [Cyber Kill Chain](./kill-chain) from Lockheed Martin
 - DE German (DE) [Government classification markings (VS)](./de-vs)
 - [DHS CIIP Sectors](./dhs-ciip-sectors)
--- a/adversary/machinetag.json
+++ b/adversary/machinetag.json
@ -8,16 +8,16 @@
      "expanded": "Infrastructure Status"
    },
    {
-      "value": "infrastructure-type",
-      "expanded": "Infrastructure Type"
+      "value": "infrastructure-action",
+      "expanded": "Infrastructure Action"
    },
    {
      "value": "infrastructure-state",
      "expanded": "Infrastructure State"
    },
    {
-      "value": "infrastructure-action",
-      "expanded": "Infrastructure Action"
+      "value": "infrastructure-type",
+      "expanded": "Infrastructure Type"
    }
  ],
  "values": [
--- a/cssa/machinetag.json
+++ b/cssa/machinetag.json
@ -0,0 +1,77 @@
+{
+  "namespace": "cssa",
+  "description": "The CSSA agreed sharing taxonomy.",
+  "version": 3,
+  "predicates": [
+    {
+      "value": "sharing-class",
+      "expanded": "Sharing Class"
+    },
+    {
+      "value": "origin",
+      "expanded": "Origin"
+    }
+  ],
+  "values": [
+    {
+      "predicate": "sharing-class",
+      "entry": [
+        {
+          "value": "high_profile",
+          "expanded": "Generated within the company during incident/case related investigations or forensic analysis or via malware reversing, validated by humans and highly contextualized.",
+          "colour": "#007695"
+        },
+        {
+          "value": "vetted",
+          "expanded": "Generated within the company, validated by a human prior to sharing, data points have been contextualized (to a degree) e.g. IPs are related to C2 or drop site.",
+          "colour": "#008aaf"
+        },
+        {
+          "value": "unvetted",
+          "expanded": "Generated within the company by automated means without human interaction e.g., by malware sandbox, honeypots, IDS, etc.",
+          "colour": "#00b3e2"
+        }
+      ]
+    },
+    {
+      "predicate": "origin",
+      "entry": [
+        {
+          "value": "manual_investigation",
+          "expanded": "Information gathered by an analyst/incident responder/forensic expert/etc.",
+          "colour": "#29775d"
+        },
+        {
+          "value": "honeypot",
+          "expanded": "Information coming out of honeypots.",
+          "colour": "#2f8a6c"
+        },
+        {
+          "value": "sandbox",
+          "expanded": "Information coming out of sandboxes.",
+          "colour": "#369d7b"
+        },
+        {
+          "value": "email",
+          "expanded": "Information coming out of email infrastructure.",
+          "colour": "#3cb08a"
+        },
+        {
+          "value": "3rd-party",
+          "expanded": "Information from outside the company.",
+          "colour": "#46c098"
+        },
+        {
+          "value": "other",
+          "expanded": "If none of the other origins applies.",
+          "colour": "#59c6a2"
+        },
+        {
+          "value": "unknown",
+          "expanded": "Origin of the data unknown.",
+          "colour": "#6ccdad"
+        }
+      ]
+    }
+  ]
+}
--- a/dni-ism/machinetag.json
+++ b/dni-ism/machinetag.json
@ -11,13 +11,21 @@
      "value": "classification:us",
      "expanded": "ClassificationUS"
    },
+    {
+      "value": "scicontrols",
+      "expanded": "SCIControls"
+    },
    {
      "value": "complies:with",
      "expanded": "CompliesWith"
    },
    {
-      "value": "dissem",
-      "expanded": "Dissem"
+      "value": "atomicenergymarkings",
+      "expanded": "atomicEnergyMarkings"
+    },
+    {
+      "value": "notice",
+      "expanded": "Notice"
    },
    {
      "value": "nonic",
@ -28,16 +36,8 @@
      "expanded": "NonUSControls"
    },
    {
-      "value": "notice",
-      "expanded": "Notice"
-    },
-    {
-      "value": "scicontrols",
-      "expanded": "SCIControls"
-    },
-    {
-      "value": "atomicenergymarkings",
-      "expanded": "atomicEnergyMarkings"
+      "value": "dissem",
+      "expanded": "Dissem"
    }
  ],
  "values": [
@ -170,6 +170,7 @@
      ]
    },
    {
+      "predicate": "atomicenergymarkings",
      "entry": [
        {
          "expanded": "RESTRICTED DATA",
@ -195,10 +196,10 @@
          "expanded": "TRANSCLASSIFIED FOREIGN NUCLEAR INFORMATION",
          "value": "TFNI"
        }
-      ],
-      "predicate": "atomicenergymarkings"
+      ]
    },
    {
+      "predicate": "notice",
      "entry": [
        {
          "expanded": "FISA Warning statement",
@ -280,10 +281,10 @@
          "expanded": "COMSEC Notice",
          "value": "COMSEC"
        }
-      ],
-      "predicate": "notice"
+      ]
    },
    {
+      "predicate": "nonic",
      "entry": [
        {
          "expanded": "NAVAL NUCLEAR PROPULSION INFORMATION",
@ -321,8 +322,7 @@
          "expanded": "SENSITIVE SECURITY INFORMATION",
          "value": "SSI"
        }
-      ],
-      "predicate": "nonic"
+      ]
    },
    {
      "predicate": "nonuscontrols",
--- a/domain-abuse/machinetag.json
+++ b/domain-abuse/machinetag.json
@ -4,15 +4,15 @@
  "description": "Domain Name Abuse - taxonomy to tag domain names used for cybercrime. Use europol-incident to tag abuse-activity",
  "version": 1,
  "predicates": [
-    {
-      "value": "domain-access-method",
-      "description": "Domain Access - describes how the adversary has gained access to the domain name",
-      "expanded": "Domain access method"
-    },
    {
      "value": "domain-status",
      "description": "Domain status - describes the registration status of the domain name",
      "expanded": "Domain status"
+    },
+    {
+      "value": "domain-access-method",
+      "description": "Domain Access - describes how the adversary has gained access to the domain name",
+      "expanded": "Domain access method"
    }
  ],
  "values": [
--- a/ecsirt/machinetag.json
+++ b/ecsirt/machinetag.json
@ -137,18 +137,30 @@
    }
  ],
  "predicates": [
+    {
+      "expanded": "Fraud",
+      "value": "fraud"
+    },
+    {
+      "expanded": "Availability",
+      "value": "availability"
+    },
    {
      "expanded": "Abusive Content",
      "value": "abusive-content"
    },
-    {
-      "expanded": "Malicious Code",
-      "value": "malicious-code"
-    },
    {
      "expanded": "Information Gathering",
      "value": "information-gathering"
    },
+    {
+      "expanded": "Information Content Security",
+      "value": "information-content-security"
+    },
+    {
+      "expanded": "Malicious Code",
+      "value": "malicious-code"
+    },
    {
      "expanded": "Intrusion Attempts",
      "value": "intrusion-attempts"
@ -157,26 +169,14 @@
      "expanded": "Intrusions",
      "value": "intrusions"
    },
-    {
-      "expanded": "Availability",
-      "value": "availability"
-    },
    {
      "expanded": "Information Security",
      "value": "information-security"
    },
-    {
-      "expanded": "Information Content Security",
-      "value": "information-content-security"
-    },
    {
      "expanded": "Vulnerable",
      "value": "vulnerable"
    },
-    {
-      "expanded": "Fraud",
-      "value": "fraud"
-    },
    {
      "expanded": "Other",
      "value": "other"
--- a/enisa/machinetag.json
+++ b/enisa/machinetag.json
@ -848,13 +848,13 @@
          "description": "Threat of sophisticated, targeted attack which combine many attack techniques."
        },
        {
-          "value": "mobile-malware",
-          "expanded": "Mobile malware",
+          "value": "mobile-malware-exfiltration",
+          "expanded": "Mobile malware (exfiltration)",
          "description": "Threat of mobile software that aims to gather information about a person or organization without their knowledge."
        },
        {
-          "value": "spear-phishing-attacks",
-          "expanded": "Spear phishing attacks",
+          "value": "spear-phishing-attacks-targeted",
+          "expanded": "Spear phishing attacks (targeted)",
          "description": "Threat of attack focused on a single user or department within an organization, coming from someone within the company in a position of trust and requesting information such as login, IDs and passwords."
        },
        {
@ -916,18 +916,18 @@
      "expanded": "Eavesdropping/ Interception/ Hijacking",
      "value": "eavesdropping-interception-hijacking"
    },
-    {
-      "description": "Threats of nefarious activities that require use of tools by the attacker. These attacks require installation of additional tools/software or performing additional steps on the victim's IT infrastructure/software.",
-      "expanded": "Nefarious Activity/ Abuse",
-      "value": "nefarious-activity-abuse"
-    },
    {
      "description": "Threat of financial or legal penalty or loss of trust of customers and collaborators due to legislation.",
      "expanded": "Legal",
      "value": "legal"
+    },
+    {
+      "description": "Threats of nefarious activities that require use of tools by the attacker. These attacks require installation of additional tools/software or performing additional steps on the victim's IT infrastructure/software.",
+      "expanded": "Nefarious Activity/ Abuse",
+      "value": "nefarious-activity-abuse"
    }
  ],
-  "version": 201601,
+  "version": 20170725,
  "description": "The present threat taxonomy is an initial version that has been developed on the basis of available ENISA material. This material has been used as an ENISA-internal structuring aid for information collection and threat consolidation purposes. It emerged in the time period 2012-2015.",
  "expanded": "ENISA Threat Taxonomy",
  "namespace": "enisa"
--- a/iep/machinetag.json
+++ b/iep/machinetag.json
@ -3,36 +3,6 @@
  "description": "Forum of Incident Response and Security Teams (FIRST) Information Exchange Policy (IEP) framework",
  "version": 2,
  "predicates": [
-    {
-      "value": "id",
-      "expanded": "POLICY ID",
-      "description": "Provides a unique ID to identify a specific IEP implementation."
-    },
-    {
-      "value": "version",
-      "expanded": "POLICY VERSION",
-      "description": "States the version of the IEP framework that has been used."
-    },
-    {
-      "value": "name",
-      "expanded": "POLICY NAME",
-      "description": "This statement can be used to provide a name for an IEP implementation."
-    },
-    {
-      "value": "start-date",
-      "expanded": "POLICY START DATE",
-      "description": "States the UTC date that the IEP is effective from."
-    },
-    {
-      "value": "end-date",
-      "expanded": "POLICY END DATE",
-      "description": "States the UTC date that the IEP is effective until."
-    },
-    {
-      "value": "reference",
-      "expanded": "POLICY REFERENCE",
-      "description": "This statement can be used to provide a URL reference to the specific IEP implementation."
-    },
    {
      "value": "commercial-use",
      "expanded": "COMMERCIAL USE",
@ -82,6 +52,36 @@
      "value": "unmodified-resale",
      "expanded": "UNMODIFIED RESALE",
      "description": "States whether the recipient MAY or MUST NOT resell the information received unmodified or in a semantically equivalent format."
+    },
+    {
+      "value": "start-date",
+      "expanded": "POLICY START DATE",
+      "description": "States the UTC date that the IEP is effective from."
+    },
+    {
+      "value": "end-date",
+      "expanded": "POLICY END DATE",
+      "description": "States the UTC date that the IEP is effective until."
+    },
+    {
+      "value": "reference",
+      "expanded": "POLICY REFERENCE",
+      "description": "This statement can be used to provide a URL reference to the specific IEP implementation."
+    },
+    {
+      "value": "name",
+      "expanded": "POLICY NAME",
+      "description": "This statement can be used to provide a name for an IEP implementation."
+    },
+    {
+      "value": "version",
+      "expanded": "POLICY VERSION",
+      "description": "States the version of the IEP framework that has been used."
+    },
+    {
+      "value": "id",
+      "expanded": "POLICY ID",
+      "description": "Provides a unique ID to identify a specific IEP implementation."
    }
  ],
  "values": [
--- a/misp/machinetag.json
+++ b/misp/machinetag.json
@ -109,6 +109,11 @@
      "expanded": "API related tag influencing the MISP behavior of the API.",
      "value": "api"
    },
+    {
+      "description": "Expansion tag incluencing the MISP behavior using expansion modules",
+      "expanded": "Expansion",
+      "value": "expansion"
+    },
    {
      "expanded": "Information related to the contributor.",
      "value": "contributor"
@ -125,11 +130,6 @@
      "description": "Event with this tag should not be synced to other MISP instances",
      "expanded": "Should not sync",
      "value": "should-not-sync"
-    },
-    {
-      "description": "Expansion tag incluencing the MISP behavior using expansion modules",
-      "expanded": "Expansion",
-      "value": "expansion"
    }
  ],
  "version": 4,
--- a/ms-caro-malware-full/machinetag.json
+++ b/ms-caro-malware-full/machinetag.json
@ -1,7 +1,7 @@
 {
  "namespace": "ms-caro-malware-full",
  "description": "Malware Type and Platform classification based on Microsoft's implementation of the Computer Antivirus Research Organization (CARO) Naming Scheme and Malware Terminology. Based on https://www.microsoft.com/en-us/security/portal/mmpc/shared/malwarenaming.aspx, https://www.microsoft.com/security/portal/mmpc/shared/glossary.aspx, https://www.microsoft.com/security/portal/mmpc/shared/objectivecriteria.aspx, and http://www.caro.org/definitions/index.html. Malware families are extracted from Microsoft SIRs since 2008 based on https://www.microsoft.com/security/sir/archive/default.aspx and https://www.microsoft.com/en-us/security/portal/threat/threats.aspx. Note that SIRs do NOT include all Microsoft malware families.",
-  "version": 1,
+  "version": 2,
  "predicates": [
    {
      "value": "malware-type",
@ -687,7 +687,7 @@
          "expanded": "2008 - A detection for the DameWare Mini Remote Control tools. This program was detected by definitions prior to 1.147.1889.0 as it violated the guidelines by which Microsoft identified unwanted software. Based on analysis using current guidelines, the program does not have unwanted behaviors. Microsoft has released definition 1.147.1889.0 which no longer detects this program."
        },
        {
-          "value": "SeekmoSearchAssistant",
+          "value": "SeekmoSearchAssistant_Repack",
          "expanded": "2008 - A detection that is triggered by modified (that is, edited and re-packed) remote control programs based on DameWare Mini Remote Control, a commercial software product"
        },
        {
@ -1611,7 +1611,7 @@
          "expanded": "2012 VOL13 - A malicious program that affects mobile devices running the Android operating system. It may be bundled with clean applications, and is capable of allowing a remote attacker to gain access to the mobile device."
        },
        {
-          "value": "Mult",
+          "value": "Mult_JS",
          "expanded": "2012 VOL13 - A generic detection for various exploits written in the JavaScript language."
        },
        {
@ -2107,7 +2107,7 @@
          "expanded": "2015 VOL19 - A detection for the Superfish VisualDiscovery advertising program that was preinstalled on some Lenovo laptops sold in 2014 and 2015. It installs a compromised trusted root certificate on the computer, which can be used to conduct man-in-the-middle attacks on the computer."
        },
        {
-          "value": "CouponRuc",
+          "value": "CouponRuc_new",
          "expanded": "2015 VOL19 - A browser modifier that changes browser settings and may also modify some computer and Internet settings."
        },
        {
--- a/passivetotal/machinetag.json
+++ b/passivetotal/machinetag.json
@ -12,13 +12,13 @@
      "value": "ever-comprimised",
      "expanded": "Ever Comprimised?"
    },
-    {
-      "value": "class",
-      "expanded": "Classification"
-    },
    {
      "value": "dynamic-dns",
      "expanded": "Dynamic DNS"
+    },
+    {
+      "value": "class",
+      "expanded": "Classification"
    }
  ],
  "values": [
--- a/schema.json
+++ b/schema.json
@ -1,42 +1,87 @@
 {
-  "required": [
-    "namespace",
-    "description",
-    "version",
-    "predicates"
-  ],
-  "properties": {
-    "type": {
-      "minItems": 1,
-      "items": {
-        "type": "string"
-      },
+  "$schema": "http://json-schema.org/schema#",
+  "title": "Validator for misp-taxonomies",
+  "id": "https://www.github.com/MISP/misp-taxonomies/schema.json",
+  "defs": {
+    "entry": {
+      "type": "array",
      "uniqueItems": true,
-      "type": "array"
-    },
-    "refs": {
      "items": {
-        "type": "string"
-      },
-      "uniqueItems": true,
-      "type": "array"
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "numerical_value": {
+            "type": "number"
+          },
+          "expanded": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "colour": {
+            "type": "string"
+          },
+          "value": {
+            "type": "string"
+          },
+          "required": [
+            "value"
+          ]
+        }
+      }
    },
    "values": {
-      "items": {
-        "$ref": "#/defs/entry",
-        "type": "object"
-      },
+      "type": "array",
      "uniqueItems": true,
-      "type": "array"
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "entry": {
+            "$ref": "#/defs/entry"
+          },
+          "predicate": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "predicate"
+        ]
+      }
    },
    "predicates": {
-      "items": {
-        "$ref": "#/defs/predicate",
-        "type": "object"
-      },
+      "type": "array",
      "uniqueItems": true,
-      "type": "array"
-    },
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "numerical_value": {
+            "type": "number"
+          },
+          "colour": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "expanded": {
+            "type": "string"
+          },
+          "value": {
+            "type": "string"
+          },
+          "required": [
+            "value"
+          ]
+        }
+      }
+    }
+  },
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
    "version": {
      "type": "integer"
    },
@ -48,74 +93,38 @@
    },
    "namespace": {
      "type": "string"
-    }
-  },
-  "additionalProperties": false,
-  "type": "object",
-  "defs": {
-    "required": [
-      "predicate"
-    ],
-    "entry": {
-      "properties": {
-        "entry": {
-          "items": {
-            "required": [
-              "value"
-            ],
-            "properties": {
-              "numerical_value": {
-                "type": "number"
-              },
-              "expanded": {
-                "type": "string"
-              },
-              "description": {
-                "type": "string"
-              },
-              "value": {
-                "type": "string"
-              }
-            },
-            "additionalProperties": false,
-            "type": "object"
-          },
-          "uniqueItems": true,
-          "type": "array"
-        },
-        "predicate": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
    },
-    "predicate": {
-      "required": [
-        "value"
-      ],
-      "properties": {
-        "expanded": {
-          "type": "string"
-        },
-        "numerical_value": {
-          "type": "number"
-        },
-        "description": {
-          "type": "string"
-        },
-        "colour": {
-          "type": "string"
-        },
-        "value": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
+    "type": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "enum": [
+          "org",
+          "user",
+          "attribute",
+          "event"
+        ]
+      }
+    },
+    "refs": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "type": "string"
+      }
+    },
+    "predicates": {
+      "$ref": "#/defs/predicates"
+    },
+    "values": {
+      "$ref": "#/defs/values"
    }
  },
-  "id": "https://www.github.com/MISP/misp-taxonomies/schema.json",
-  "title": "Validator for misp-taxonomies",
-  "$schema": "http://json-schema.org/schema#"
+  "required": [
+    "namespace",
+    "description",
+    "version",
+    "predicates"
+  ]
 }
--- a/validate_all.sh
+++ b/validate_all.sh
@ -7,11 +7,19 @@ set -x

 diffs=`git status --porcelain | wc -l`

-if ! [ $diffs -eq 1 ]; then
+if ! [ $diffs -eq 0 ]; then
 	echo "Please make sure you run ./jq_all_the_things.sh before commiting."
 	exit 1
 fi

+directories=`ls -d */ | wc -w`
+manifest_entries=`cat MANIFEST.json | jq '.taxonomies | length'`
+
+if ! [ $directories -eq $manifest_entries ]; then
+    echo "MANIFEST isn't up-to-date."
+    exit 1
+fi
+
 for dir in */machinetag.json
 do
  echo -n "${dir}: "
--- a/veris/machinetag.json
+++ b/veris/machinetag.json