new: [crawler] MISP feed importer into CyCAT including misp-tag and mitre-attack-id

README.md

@@ -11,6 +11,7 @@
 ## Crawler
 
 - [GitHub](./crawler/github/)
+- [MISP Feeds](./crawler/misp-feeds/)
 - [MITRE CTI - ATT&CK](./crawler/mitre-cti)
 - [Sigma](./crawler/sigma/)
 

crawler/misp-feeds/README.md

@@ -0,0 +1,16 @@
+# MISP feed importer in CyCAT
+
+MISP event UUID are imported in CyCAT along with associated namespace `misp-tag` and `mitre-attack-id` (if presents).
+
+# Usage
+
+~~~
+usage: misp_feed_importer.py [-h] [-u URL]
+
+MISP feed importer for CyCAT
+
+optional arguments:
+  -h, --help         show this help message and exit
+    -u URL, --url URL  MISP feed url
+~~~
+

crawler/misp-feeds/importer.sh

@@ -0,0 +1,5 @@
+python3 misp_feed_importer.py -u https://www.circl.lu/doc/misp/feed-osint
+python3 misp_feed_importer.py -u http://www.botvrij.eu/data/feed-osint
+python3 misp_feed_importer.py -u https://threatfox.abuse.ch/downloads/misp/
+python3 misp_feed_importer.py -u https://bazaar.abuse.ch/downloads/misp/
+python3 misp_feed_importer.py -u https://urlhaus.abuse.ch/downloads/misp/

crawler/misp-feeds/misp_feed_importer.py

@@ -0,0 +1,74 @@
+
+import argparse
+import json
+import redis
+import os
+import requests
+import uuid
+import re
+import yaml
+
+parser = argparse.ArgumentParser(description='MISP feed importer for CyCAT')
+parser.add_argument('-u', '--url', help='MISP feed url')
+args = parser.parse_args()
+rdb = redis.Redis(host='127.0.0.1', port='3033')
+
+if not args.url:
+    parser.print_usage()
+    os.sys.exit(1)
+
+def additem(uuidref=None, data=None, project=None):
+    if uuidref is None or data is None:
+        return None
+    rdb.set("u:{}".format(uuidref), 3)
+    d = {"{}".format(uuidref): 1}
+    k = "t:{}".format(3)
+    rdb.zadd(k, d, nx=False)
+    rdb.hmset("{}:{}".format(3, uuidref), data)
+    if project is not None:
+        rdb.sadd("parent:{}".format(uuidref), project)
+        rdb.sadd("child:{}".format(project), uuidref)
+    if 'capec' in data:
+        addexternalid(uuidsource=uuidref, namespace='capec', namespaceid=data['capec'])
+    if 'mitre-attack-id' in data:
+        addexternalid(uuidsource=uuidref, namespace='mitre-attack-id', namespaceid=data['mitre-attack-id'])
+    return True
+
+def addrelationship(uuidsource=None, uuiddest=None, data=None):
+    if uuidsource is None or uuiddest is None:
+        return None
+    rdb.sadd("r:{}".format(uuidsource), uuiddest)
+    rdb.sadd("rd:{}:{}".format(uuidsource, uuiddest), data)
+    return True
+
+def addexternalid(uuidsource=None, namespace=None, namespaceid=None):
+    if uuidsource is None or namespace is None or namespaceid is None:
+        return None
+    k = "id:{}:{}".format(namespace.lower(), namespaceid)
+    rdb.sadd(k, uuidsource)
+    k = "idk:{}".format(namespace)
+    rdb.sadd(k, namespaceid)
+    rdb.sadd("idnamespace", namespace)
+
+r = requests.get("{}/manifest.json".format(args.url))
+feed = r.json()
+for event in feed:
+    data = {}
+    data['uuid'] = event
+    data['description'] = feed[event]['info']
+    data['timestamp'] = feed[event]['timestamp']
+    data['link'] = "{}/{}.json".format(args.url, event)
+    data['misp:feed'] = "{}".format(args.url)
+    if 'Tag' in feed[event]:
+        for tags in feed[event]['Tag']:
+            addexternalid(uuidsource=data['uuid'], namespace='misp-tag', namespaceid=tags['name'].lower())
+            pattern = re.compile(r"^misp-galaxy:mitre-")
+            if pattern.search(tags['name']):
+                (namespacepredicate, value) = tags['name'].split('=')
+                try:
+                    mitre = value.rsplit('-', 1)[1]
+                    mitreattackid = mitre.replace(" ", "").replace("\"","")
+                    addexternalid(uuidsource=data['uuid'], namespace='mitre-attack-id', namespaceid=mitreattackid)
+                except:
+                    continue
+    additem(uuidref=data['uuid'], data=data)