#!/usr/bin/env python # -*- coding: utf-8 -*- # # Python script parsing the MISP taxonomies expressed in Machine Tags (Triple # Tags) to list all valid tags from a specific taxonomy. # # Copyright (c) 2015-2022 Alexandre Dulaunoy - a@foo.be # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED # OF THE POSSIBILITY OF SUCH DAMAGE. import json import os.path import argparse import os import sys skip_list = ["death-possibilities", "poison-taxonomy", "doping-substances"] taxonomies = [] # Get our current directory from file location thisDir = os.path.dirname(__file__) argParser = argparse.ArgumentParser( description="Dump Machine Tags (Triple Tags) from MISP taxonomies", epilog="Available taxonomies are {0}".format(taxonomies), ) argParser.add_argument("-e", action="store_true", help="Include expanded tags") argParser.add_argument( "-a", action="store_true", help="Generate asciidoctor document from MISP taxonomies" ) argParser.add_argument("-v", action="store_true", help="Include descriptions") argParser.add_argument("-n", default=False, help="Show only the specified namespace") argParser.add_argument( "--disable-skip-list", default=False, action="store_true", help="disable default skip list", ) args = argParser.parse_args() if args.disable_skip_list: skip_list = "" for folder in os.listdir(os.path.join(thisDir, "../")): if os.path.isfile(os.path.join(thisDir, "../", folder, "machinetag.json")): if folder in skip_list: continue taxonomies.append(folder) taxonomies.sort() doc = "" if args.a: dedication = "\n[dedication]\n== Funding and Support\nThe MISP project is financially and resource supported by https://www.circl.lu/[CIRCL Computer Incident Response Center Luxembourg ].\n\nimage:{images-misp}logo.png[CIRCL logo]\n\nA CEF (Connecting Europe Facility) funding under CEF-TC-2016-3 - Cyber Security has been granted from 1st September 2017 until 31th August 2019 as ***Improving MISP as building blocks for next-generation information sharing***.\n\nimage:{images-misp}en_cef.png[CEF funding]\n\nIf you are interested to co-fund projects around MISP, feel free to get in touch with us.\n\n" doc = doc + ":toc: right\n" doc = doc + ":toclevels: 1\n" doc = doc + ":icons: font\n" doc = ( doc + ":images-cdn: https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/logos/\n" ) doc = doc + ":images-misp: https://www.misp-project.org/assets/images/\n" doc = doc + "= MISP taxonomies and classification as machine tags\n\n" doc = doc + "= Introduction\n" doc = doc + "\nimage::{images-cdn}misp-logo.png[MISP logo]\n" doc = ( doc + "The MISP threat sharing platform is a free and open source software helping information sharing of threat intelligence including cyber security indicators, financial fraud or counter-terrorism information. The MISP project includes multiple sub-projects to support the operational requirements of analysts and improve the overall quality of information shared.\n\n" ) doc = doc + "" doc = "{} {} {} {}".format( doc, "\nTaxonomies that can be used in MISP (2.4) and other information sharing tool and expressed in Machine Tags (Triple Tags).", "A machine tag is composed of a namespace (MUST), a predicate (MUST) and an (OPTIONAL) value.", "Machine tags are often called triple tag due to their format.\n", ) doc = ( doc + "The following document is generated from the machine-readable JSON describing the https://github.com/MISP/misp-taxonomies[MISP taxonomies]." ) doc = doc + "\n\n" doc = doc + "<<<\n" doc = doc + dedication doc = doc + "<<<\n" doc = doc + "= MISP taxonomies\n" doc = doc + "\n\n" if args.n: del taxonomies[:] taxonomies.append(args.n) def asciidoc(content=False, adoc=doc, t="title", toplevel=False): if not args.a: return False adoc = adoc + "\n" if t == "title": content = "==== " + content elif t == "predicate": content = "=== " + content elif t == "namespace": content = "== " + content + "\n" content = "{}\n{}{} {}{}{} {}".format( content, "NOTE: ", namespace, "namespace available in JSON format at https://github.com/MISP/misp-taxonomies/blob/main/", namespace, "/machinetag.json[*this location*]. The JSON format can be freely reused in your application", "or automatically enabled in https://www.github.com/MISP/MISP[MISP] taxonomy.", ) elif t == "description" and toplevel is True: content = "\n{} \n".format(content) elif t == "description" and toplevel is False: try: (n, value) = content.split(":", 1) content = "\n{} \n".format(value) except: content = "\n{} \n".format(content) elif t == "numerical_value": (n, value) = content.split(":", 1) content = '\nAssociated numerical value="{}" \n'.format(value) elif t == "exclusive": (n, value) = content.split(":", 1) if n: content = "\nIMPORTANT: Exclusive flag set which means the values or predicate below must be set exclusively.\n" adoc = adoc + content return adoc def machineTag(namespace=False, predicate=False, value=None): if namespace is False or predicate is False: return None if value is None: return "{0}:{1}".format(namespace, predicate) else: return '{0}:{1}="{2}"'.format(namespace, predicate, value) for taxonomy in taxonomies: if taxonomy in skip_list: sys.stderr.write(f"Skip {taxonomy}") continue filename = os.path.join(thisDir, "../", taxonomy, "machinetag.json") with open(filename) as fp: t = json.load(fp) namespace = t["namespace"] if t.get("expanded"): expanded_namespace = t["expanded"] else: expanded_namespace = namespace if args.a: doc = asciidoc(content=t["namespace"], adoc=doc, t="namespace") doc = asciidoc( content=t["description"], adoc=doc, t="description", toplevel=True ) if t.get("exclusive"): doc = asciidoc( content=machineTag(namespace=namespace, predicate=t["exclusive"]), adoc=doc, t="exclusive", ) if args.v: print("{0}".format(t["description"])) for predicate in t["predicates"]: if args.a: doc = asciidoc(content=predicate["value"], adoc=doc, t="predicate") if predicate.get("description"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["description"] ), adoc=doc, t="description", ) if predicate.get("exclusive"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["exclusive"] ), adoc=doc, t="exclusive", ) if t.get("values") is None: if args.a: doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["value"] ), adoc=doc, ) doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["expanded"] ), adoc=doc, t="description", ) if predicate.get("description"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["description"] ), adoc=doc, t="description", ) if predicate.get("numerical_value"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["numerical_value"] ), adoc=doc, t="description", ) if predicate.get("exclusive"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=predicate["exclusive"] ), adoc=doc, t="exclusive", ) else: print(machineTag(namespace=namespace, predicate=predicate["value"])) if args.e: print( "--> " + machineTag( namespace=expanded_namespace, predicate=predicate["expanded"] ) ) if predicate.get("description"): print("--> " + predicate["description"]) else: for e in t["values"]: if e["predicate"] == predicate["value"]: if "expanded" in predicate: expanded = predicate["expanded"] for v in e["entry"]: if args.a and "expanded" in v: doc = asciidoc( content=machineTag( namespace=namespace, predicate=e["predicate"], value=v["value"], ), adoc=doc, ) doc = asciidoc( content=machineTag( namespace=namespace, predicate=v["expanded"] ), adoc=doc, t="description", ) if "description" in v: doc = asciidoc( content=machineTag( namespace=namespace, predicate=v["description"] ), adoc=doc, t="description", ) if v.get("numerical_value"): doc = asciidoc( content=machineTag( namespace=namespace, predicate=v["numerical_value"], ), adoc=doc, t="numerical_value", ) else: print( machineTag( namespace=namespace, predicate=e["predicate"], value=v["value"], ) ) if args.e: if "expanded" in v: print( "--> " + machineTag( namespace=namespace, predicate=expanded, value=v["expanded"], ) ) with open("../mapping/mapping.json") as mapping: m = json.load(mapping) output = "\n= Mapping of taxonomies\n" output = "{}{}".format( output, "Analysts relying on taxonomies don't always know the appropriate namespace to use but know which value to use for classification. The MISP mapping taxonomy allows to map a single classification into a series of machine-tag synonyms.\n", ) for value in sorted(m.keys()): output = "{}{} **{}**{}{}\n".format( output, "\n.Mapping table - ", value, "\n|===\n|", value ) for mapped in m[value]["values"]: output = "{}|{}\n".format(output, mapped) output = "{}|===\n".format(output) doc = doc + output if args.a: print(doc)