diff --git a/examples/et2misp.py b/examples/et2misp.py new file mode 100644 index 0000000..dc6b79e --- /dev/null +++ b/examples/et2misp.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copy Emerging Threats Block IPs list to several MISP events +# Because of the large size of the list the first run will take about 30 mins +# Running it again will update the MISP events if changes are detected +# +# This script requires PyMISP 2.4.50 or later + +import sys, json, time, requests +from pymisp import PyMISP +from keys import misp_url, misp_key + +et_url = 'https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt' +et_str = 'Emerging Threats ' + +def init_misp(): + global mymisp + mymisp = PyMISP(misp_url, misp_key) + +def load_misp_event(eid): + global et_attr + global et_drev + global et_event + et_attr = {} + et_drev = {} + + et_event = mymisp.get(eid) + echeck(et_event) + for a in et_event['Event']['Attribute']: + if a['category'] == 'Network activity': + et_attr[a['value']] = a['id'] + continue + if a['category'] == 'Internal reference': + et_drev = a; + +def init_et(): + global et_data + global et_rev + requests.packages.urllib3.disable_warnings() + s = requests.Session() + r = s.get(et_url) + if r.status_code != 200: + raise Exception('Error getting ET data: {}'.format(r.text)) + name = '' + et_data = {} + et_rev = 0 + for line in r.text.splitlines(): + if line.startswith('# Rev '): + et_rev = int(line[6:]) + continue + if line.startswith('#'): + name = line[1:].strip() + if et_rev and not et_data.get(name): + et_data[name] = {} + continue + l = line.rstrip() + if l: + et_data[name][l] = name + +def update_et_event(name): + if et_drev and et_rev and int(et_drev['value']) < et_rev: + # Copy MISP attributes to new dict + et_ips = dict.fromkeys(et_attr.keys()) + + # Weed out attributes still in ET data + for k,v in et_data[name].items(): + et_attr.pop(k, None) + + # Delete the leftover attributes from MISP + for k,v in et_attr.items(): + r = mymisp.delete_attribute(v) + if r.get('errors'): + print "Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']) + + # Weed out ips already in the MISP event + for k,v in et_ips.items(): + et_data[name].pop(k, None) + + # Add new attributes to MISP event + for k,v in et_data[name].items(): + r = mymisp.add_ipdst(et_event, k, comment=v) + echeck(r, et_event['Event']['id']) + + # Update revision number + et_drev['value'] = et_rev + et_drev.pop('timestamp', None) + attr = [] + attr.append(et_drev) + + # Publish updated MISP event + et_event['Event']['Attribute'] = attr + et_event['Event']['published'] = False + et_event['Event']['date'] = time.strftime('%Y-%m-%d') + r = mymisp.publish(et_event) + echeck(r, et_event['Event']['id']) + +def echeck(r, eid=None): + if r.get('errors'): + if eid: + print "Processing event {} failed: {}".format(eid, r['errors']) + else: + print r['errors'] + sys.exit(1) + +if __name__ == '__main__': + init_misp() + init_et() + + for et_type in set(et_data.keys()): + info = et_str + et_type + r = mymisp.search_index(eventinfo=info) + if r['response']: + eid=r['response'][0]['id'] + else: # event not found, create it + new_event = mymisp.new_event(info=info, distribution=3, threat_level_id=4, analysis=1) + echeck(new_event) + eid=new_event['Event']['id'] + r = mymisp.add_internal_text(new_event, 1, comment='Emerging Threats revision number') + echeck(r, eid) + load_misp_event(eid) + update_et_event(et_type)