From 285ed78f2913317a0580a739f784fc9ca4c165cc Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 31 May 2021 22:58:04 +0200
Subject: [PATCH] new: [crawler] misp-galaxy crawler added

---
 README.md                                   |  1 +
 crawler/misp-galaxy/README.md               | 16 +++++
 crawler/misp-galaxy/import.sh               |  3 +
 crawler/misp-galaxy/misp_galaxy_importer.py | 65 +++++++++++++++++++++
 4 files changed, 85 insertions(+)
 create mode 100644 crawler/misp-galaxy/README.md
 create mode 100644 crawler/misp-galaxy/import.sh
 create mode 100644 crawler/misp-galaxy/misp_galaxy_importer.py

diff --git a/README.md b/README.md
index 47cdfb3..465d37a 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,7 @@
 
 - [GitHub](./crawler/github/)
 - [MISP Feeds](./crawler/misp-feeds/)
+- [MISP Galaxy](./crawler/misp-galaxy/)
 - [MITRE CTI - ATT&CK](./crawler/mitre-cti)
 - [Sigma](./crawler/sigma/)
 
diff --git a/crawler/misp-galaxy/README.md b/crawler/misp-galaxy/README.md
new file mode 100644
index 0000000..92bb485
--- /dev/null
+++ b/crawler/misp-galaxy/README.md
@@ -0,0 +1,16 @@
+# MISP Galaxy Cluster importer for CyCAT
+
+## Usage
+
+`python3 misp_galaxy_importer.py -u https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/threat-actor.json`
+
+~~~
+usage: misp_galaxy_importer.py [-h] [-u URL]
+
+MISP galaxy importer for CyCAT
+
+optional arguments:
+  -h, --help         show this help message and exit
+    -u URL, --url URL  MISP JSON galaxy cluster url
+~~~
+
diff --git a/crawler/misp-galaxy/import.sh b/crawler/misp-galaxy/import.sh
new file mode 100644
index 0000000..449a967
--- /dev/null
+++ b/crawler/misp-galaxy/import.sh
@@ -0,0 +1,3 @@
+python3 misp_galaxy_importer.py -u https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/threat-actor.json
+python3 misp_galaxy_importer.py -u https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/tool.json
+python3 misp_galaxy_importer.py -u https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json
diff --git a/crawler/misp-galaxy/misp_galaxy_importer.py b/crawler/misp-galaxy/misp_galaxy_importer.py
new file mode 100644
index 0000000..ba9c6ea
--- /dev/null
+++ b/crawler/misp-galaxy/misp_galaxy_importer.py
@@ -0,0 +1,65 @@
+
+import argparse
+import json
+import redis
+import os
+import requests
+import uuid
+import re
+import yaml
+
+parser = argparse.ArgumentParser(description='MISP galaxy importer for CyCAT')
+parser.add_argument('-u', '--url', help='MISP JSON galaxy cluster url')
+args = parser.parse_args()
+rdb = redis.Redis(host='127.0.0.1', port='3033')
+
+if not args.url:
+    parser.print_usage()
+    os.sys.exit(1)
+
+def additem(uuidref=None, data=None, project=None):
+    if uuidref is None or data is None:
+        return None
+    rdb.set("u:{}".format(uuidref), 3)
+    d = {"{}".format(uuidref): 1}
+    k = "t:{}".format(3)
+    rdb.zadd(k, d, nx=False)
+    rdb.hmset("{}:{}".format(3, uuidref), data)
+    if project is not None:
+        rdb.sadd("parent:{}".format(uuidref), project)
+        rdb.sadd("child:{}".format(project), uuidref)
+    if 'capec' in data:
+        addexternalid(uuidsource=uuidref, namespace='capec', namespaceid=data['capec'])
+    if 'mitre-attack-id' in data:
+        addexternalid(uuidsource=uuidref, namespace='mitre-attack-id', namespaceid=data['mitre-attack-id'])
+    return True
+
+def addrelationship(uuidsource=None, uuiddest=None, data=None):
+    if uuidsource is None or uuiddest is None:
+        return None
+    rdb.sadd("r:{}".format(uuidsource), uuiddest)
+    rdb.sadd("rd:{}:{}".format(uuidsource, uuiddest), data)
+    return True
+
+def addexternalid(uuidsource=None, namespace=None, namespaceid=None):
+    if uuidsource is None or namespace is None or namespaceid is None:
+        return None
+    k = "id:{}:{}".format(namespace.lower(), namespaceid)
+    rdb.sadd(k, uuidsource)
+    k = "idk:{}".format(namespace)
+    rdb.sadd(k, namespaceid)
+    rdb.sadd("idnamespace", namespace)
+
+r = requests.get("{}".format(args.url))
+cluster = r.json()
+for element in cluster['values']:
+    data = {}
+    data['uuid'] = element['uuid']
+    if 'description' in element:
+        data['description'] = element['description']
+    data['misp-galaxy:value'] = element['value']
+    data['raw'] = json.dumps(element)
+    if 'related' in element:
+        for rel in element['related']:
+            addrelationship(uuidsource=data['uuid'], uuiddest=rel['dest-uuid'], data=json.dumps(rel))
+    additem(uuidref=data['uuid'], data=data)