diff --git a/README.md b/README.md index 10786e3..47cdfb3 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ ## Crawler - [GitHub](./crawler/github/) +- [MISP Feeds](./crawler/misp-feeds/) - [MITRE CTI - ATT&CK](./crawler/mitre-cti) - [Sigma](./crawler/sigma/) diff --git a/crawler/misp-feeds/README.md b/crawler/misp-feeds/README.md new file mode 100644 index 0000000..09e55cc --- /dev/null +++ b/crawler/misp-feeds/README.md @@ -0,0 +1,16 @@ +# MISP feed importer in CyCAT + +MISP event UUID are imported in CyCAT along with associated namespace `misp-tag` and `mitre-attack-id` (if presents). + +# Usage + +~~~ +usage: misp_feed_importer.py [-h] [-u URL] + +MISP feed importer for CyCAT + +optional arguments: + -h, --help show this help message and exit + -u URL, --url URL MISP feed url +~~~ + diff --git a/crawler/misp-feeds/importer.sh b/crawler/misp-feeds/importer.sh new file mode 100644 index 0000000..ab28d5e --- /dev/null +++ b/crawler/misp-feeds/importer.sh @@ -0,0 +1,5 @@ +python3 misp_feed_importer.py -u https://www.circl.lu/doc/misp/feed-osint +python3 misp_feed_importer.py -u http://www.botvrij.eu/data/feed-osint +python3 misp_feed_importer.py -u https://threatfox.abuse.ch/downloads/misp/ +python3 misp_feed_importer.py -u https://bazaar.abuse.ch/downloads/misp/ +python3 misp_feed_importer.py -u https://urlhaus.abuse.ch/downloads/misp/ diff --git a/crawler/misp-feeds/misp_feed_importer.py b/crawler/misp-feeds/misp_feed_importer.py new file mode 100644 index 0000000..0c5939d --- /dev/null +++ b/crawler/misp-feeds/misp_feed_importer.py @@ -0,0 +1,74 @@ + +import argparse +import json +import redis +import os +import requests +import uuid +import re +import yaml + +parser = argparse.ArgumentParser(description='MISP feed importer for CyCAT') +parser.add_argument('-u', '--url', help='MISP feed url') +args = parser.parse_args() +rdb = redis.Redis(host='127.0.0.1', port='3033') + +if not args.url: + parser.print_usage() + os.sys.exit(1) + +def additem(uuidref=None, data=None, project=None): + if uuidref is None or data is None: + return None + rdb.set("u:{}".format(uuidref), 3) + d = {"{}".format(uuidref): 1} + k = "t:{}".format(3) + rdb.zadd(k, d, nx=False) + rdb.hmset("{}:{}".format(3, uuidref), data) + if project is not None: + rdb.sadd("parent:{}".format(uuidref), project) + rdb.sadd("child:{}".format(project), uuidref) + if 'capec' in data: + addexternalid(uuidsource=uuidref, namespace='capec', namespaceid=data['capec']) + if 'mitre-attack-id' in data: + addexternalid(uuidsource=uuidref, namespace='mitre-attack-id', namespaceid=data['mitre-attack-id']) + return True + +def addrelationship(uuidsource=None, uuiddest=None, data=None): + if uuidsource is None or uuiddest is None: + return None + rdb.sadd("r:{}".format(uuidsource), uuiddest) + rdb.sadd("rd:{}:{}".format(uuidsource, uuiddest), data) + return True + +def addexternalid(uuidsource=None, namespace=None, namespaceid=None): + if uuidsource is None or namespace is None or namespaceid is None: + return None + k = "id:{}:{}".format(namespace.lower(), namespaceid) + rdb.sadd(k, uuidsource) + k = "idk:{}".format(namespace) + rdb.sadd(k, namespaceid) + rdb.sadd("idnamespace", namespace) + +r = requests.get("{}/manifest.json".format(args.url)) +feed = r.json() +for event in feed: + data = {} + data['uuid'] = event + data['description'] = feed[event]['info'] + data['timestamp'] = feed[event]['timestamp'] + data['link'] = "{}/{}.json".format(args.url, event) + data['misp:feed'] = "{}".format(args.url) + if 'Tag' in feed[event]: + for tags in feed[event]['Tag']: + addexternalid(uuidsource=data['uuid'], namespace='misp-tag', namespaceid=tags['name'].lower()) + pattern = re.compile(r"^misp-galaxy:mitre-") + if pattern.search(tags['name']): + (namespacepredicate, value) = tags['name'].split('=') + try: + mitre = value.rsplit('-', 1)[1] + mitreattackid = mitre.replace(" ", "").replace("\"","") + addexternalid(uuidsource=data['uuid'], namespace='mitre-attack-id', namespaceid=mitreattackid) + except: + continue + additem(uuidref=data['uuid'], data=data)