new: [MITRE CTI] first version of the CTI crawler

crawler/bin/markdown2json.py

@@ -0,0 +1,17 @@
+import sys
+
+md_table = sys.stdin.readlines()
+print(md_table)
+
+result = []
+for n, line in enumerate(md_table[1:-1]):
+    data = {}
+    if n == 0:
+        header = [t.strip() for t in line.split('|')[1:-1]]
+    if n > 1:
+        values = [t.strip() for t in line.split('|')[1:-1]]
+        for col, value in zip(header, values):
+            data[col] = value
+        result.append(data)
+
+print(result)

crawler/mitre-cti/cti-importer.py

@@ -0,0 +1,61 @@
+import argparse
+import json
+import redis
+import os
+import requests
+import uuid
+parser = argparse.ArgumentParser(description='MITRE CTI (ATT&CK) import for CyCAT')
+parser.add_argument('-p', '--path', help='Path to the CTI git repository')
+args = parser.parse_args()
+rdb = redis.Redis(host='127.0.0.1', port='3033')
+
+# CTI parent c7001e65-fefe-55cb-84a3-97ec2620137
+
+projectuuid='c7001e65-fefe-55cb-84a3-97ec2620137a'
+
+if not args.path:
+    parser.print_usage()
+    os.sys.exit(1)
+
+def additem(uuidref=None, data=None, project=None):
+    if uuidref is None or data is None:
+        return None
+    rdb.set("u:{}".format(uuidref), 3)
+    d = {"{}".format(uuidref): 1}
+    k = "t:{}".format(3)
+    rdb.zadd(k, d, nx=False)
+    rdb.hmset("{}:{}".format(3, uuidref), data)
+    if project is not None:
+        rdb.sadd("parent:{}".format(uuidref), project)
+        rdb.sadd("child:{}".format(project), uuidref)
+    return True
+
+def addrelationship(uuidsource=None, uuiddest=None, data=None):
+    if uuidsource is None or uuiddest is None:
+        return None
+    rdb.sadd("r:{}".format(uuidsource), uuiddest)
+    rdb.sadd("rd:{}:{}".format(uuidsource, uuiddest), data)
+    return True
+
+models = ['enterprise-attack', 'mobile-attack', 'ics-attack', 'pre-attack']
+
+for model in models:
+    path = "{}/{}/{}.json".format(args.path, model, model)
+    f = open(path, mode='r')
+    m = json.loads(f.read())
+    for obj in m['objects']:
+        (obj_type, obj_id) = obj['id'].split('--')
+        if obj_type != 'relationship':
+            data = {}
+            data['raw'] = str(obj)
+            data['mitre-cti:type'] = obj['type']
+            if 'description' in obj:
+                data['mitre-cti:description'] = obj['description']
+            if 'name' in obj:
+                data['mitre-cti:name'] = obj['name']
+            additem(uuidref=obj_id, project=projectuuid, data=data)
+        elif obj_type == 'relationship':
+            (source_type, source_id) = obj['source_ref'].split('--')
+            (destination_type, destination_id) = obj['target_ref'].split('--')
+            addrelationship(uuidsource=source_id, uuiddest=destination_id, data=str(obj))
+