From 762b517150ac46898331e52878496752681be167 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 13:36:03 +0200 Subject: [PATCH] fix: [MISP auto export] fix MISP_TheHive_feeder --- bin/MISP_The_Hive_feeder.py | 40 +++++----- bin/Tags.py | 1 + bin/ailleakObject.py | 154 ++++++++++++++++-------------------- bin/export/MispExport.py | 8 +- bin/lib/item_basic.py | 3 + bin/packages/Item.py | 2 +- bin/packages/Tag.py | 24 +++++- 7 files changed, 118 insertions(+), 114 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 0949471c..23043788 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -20,8 +20,12 @@ from Helper import Process from packages import Paste import ailleakObject -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Tag + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import ConfigLoader +import item_basic from pymisp import PyMISP @@ -94,26 +98,22 @@ def create_the_hive_alert(source, path, tag): def feeder(message, count=0): if flag_the_hive or flag_misp: - tag, path = message.split(';') + tag, item_id = message.split(';') + ## FIXME: remove it - if PASTES_FOLDER not in path: - path = os.path.join(PASTES_FOLDER, path) - try: - paste = Paste.Paste(path) - except FileNotFoundError: + if not item_basic.exist_item(item_id): if count < 10: r_serv_db.zincrby('mess_not_saved_export', message, 1) return 0 else: r_serv_db.zrem('mess_not_saved_export', message) - print('Error: {} do not exist, tag= {}'.format(path, tag)) + print('Error: {} do not exist, tag= {}'.format(item_id, tag)) return 0 - source = '/'.join(paste.p_path.split('/')[-6:]) + source = item_basic.get_source(item_id) if HiveApi != False: if int(r_serv_db.get('hive:auto-alerts')) == 1: - whitelist_hive = r_serv_db.scard('whitelist_hive') if r_serv_db.sismember('whitelist_hive', tag): create_the_hive_alert(source, path, tag) else: @@ -121,7 +121,7 @@ def feeder(message, count=0): if flag_misp: if int(r_serv_db.get('misp:auto-events')) == 1: if r_serv_db.sismember('whitelist_misp', tag): - misp_wrapper.pushToMISP(uuid_ail, path, tag) + misp_wrapper.pushToMISP(uuid_ail, item_id, tag) else: print('misp, auto events creation disable') @@ -161,15 +161,15 @@ if __name__ == "__main__": print('Not connected to MISP') if flag_misp: - try: - misp_wrapper = ailleakObject.ObjectWrapper(pymisp) - r_serv_db.set('ail:misp', True) - print('Connected to MISP:', misp_url) - except Exception as e: - flag_misp = False - r_serv_db.set('ail:misp', False) - print(e) - print('Not connected to MISP') + #try: + misp_wrapper = ailleakObject.ObjectWrapper(pymisp) + r_serv_db.set('ail:misp', True) + print('Connected to MISP:', misp_url) + #except Exception as e: + # flag_misp = False + # r_serv_db.set('ail:misp', False) + # print(e) + # print('Not connected to MISP') # create The HIVE connection if flag_the_hive: diff --git a/bin/Tags.py b/bin/Tags.py index 88e0ef0e..b38d6309 100755 --- a/bin/Tags.py +++ b/bin/Tags.py @@ -45,4 +45,5 @@ if __name__ == '__main__': tag, item_id = message.split(';') Tag.add_tag("item", tag, item_id) + p.populate_set_out(message, 'MISP_The_Hive_feeder') diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py index 5fbf9f75..fd07e6cc 100755 --- a/bin/ailleakObject.py +++ b/bin/ailleakObject.py @@ -4,7 +4,10 @@ import os import sys +from pymisp import MISPEvent, MISPObject from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +MISPEvent + from packages import Paste import datetime import json @@ -12,28 +15,10 @@ from io import BytesIO sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import item_basic -class AilLeakObject(AbstractMISPObjectGenerator): - def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number): - super(AbstractMISPObjectGenerator, self).__init__('ail-leak') - self._uuid = uuid_ail - self._p_source = p_source - self._p_date = p_date - self._p_content = p_content - self._p_duplicate = p_duplicate - self._p_duplicate_number = p_duplicate_number - self.generate_attributes() - - def generate_attributes(self): - self.add_attribute('origin', value=self._p_source, type='text') - self.add_attribute('last-seen', value=self._p_date, type='datetime') - if self._p_duplicate_number > 0: - self.add_attribute('duplicate', value=self._p_duplicate, type='text') - self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter') - self._pseudofile = BytesIO(self._p_content.encode()) - res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag) - #res.add_shadow_attributes(tag) - self.add_attribute('sensor', value=self._uuid, type="text") +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) +import MispExport class ObjectWrapper: def __init__(self, pymisp): @@ -45,53 +30,48 @@ class ObjectWrapper: config_loader = None self.attribute_to_tag = None - def add_new_object(self, uuid_ail, path, p_source, tag): + def add_new_object(self, uuid_ail, item_id, tag): self.uuid_ail = uuid_ail - self.path = path - self.p_source = p_source - self.paste = Paste.Paste(path) - self.p_date = self.date_to_str(self.paste.p_date) - self.p_content = self.paste.get_p_content() - self.p_tag = tag - temp = self.paste._get_p_duplicate() + # self.paste = Paste.Paste(path) + # temp = self.paste._get_p_duplicate() + # + # #beautifier + # if not temp: + # temp = '' + # + # p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + # + # to_ret = "" + # for dup in temp[:10]: + # dup = dup.replace('\'','\"').replace('(','[').replace(')',']') + # dup = json.loads(dup) + # algo = dup[0] + # path = dup[1].split('/')[-6:] + # path = '/'.join(path)[:-3] # -3 removes .gz + # if algo == 'tlsh': + # perc = 100 - int(dup[2]) + # else: + # perc = dup[2] + # to_ret += "{}: {} [{}%]\n".format(path, algo, perc) + # p_duplicate = to_ret - #beautifier - if not temp: - temp = '' - - p_duplicate_number = len(temp) if len(temp) >= 0 else 0 - - to_ret = "" - for dup in temp[:10]: - dup = dup.replace('\'','\"').replace('(','[').replace(')',']') - dup = json.loads(dup) - algo = dup[0] - path = dup[1].split('/')[-6:] - path = '/'.join(path)[:-3] # -3 removes .gz - if algo == 'tlsh': - perc = 100 - int(dup[2]) - else: - perc = dup[2] - to_ret += "{}: {} [{}%]\n".format(path, algo, perc) - p_duplicate = to_ret - - self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number) + return MispExport.export_ail_item(item_id, [tag]) def date_to_str(self, date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) - def get_all_related_events(self): - to_search = "Daily AIL-leaks" - result = self.pymisp.search_all(to_search) + def get_all_related_events(self, to_search): + result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False) events = [] - for e in result['response']: - events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) + if result: + for e in result: + events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) return events def get_daily_event_id(self): to_match = "Daily AIL-leaks {}".format(datetime.date.today()) - events = self.get_all_related_events() + events = self.get_all_related_events(to_match) for dic in events: info = dic['info'] e_id = dic['id'] @@ -99,8 +79,8 @@ class ObjectWrapper: print('Found: ', info, '->', e_id) self.currentID_date = datetime.date.today() return e_id - created_event = self.create_daily_event()['Event'] - new_id = created_event['id'] + created_event = self.create_daily_event() + new_id = created_event['Event']['id'] print('New event created:', new_id) self.currentID_date = datetime.date.today() return new_id @@ -120,17 +100,20 @@ class ObjectWrapper: orgc_id = None sharing_group_id = None date = None - event = self.pymisp.new_event(distribution, threat, - analysis, info, date, - published, orgc_id, org_id, sharing_group_id) - eventUuid = event['Event']['uuid'] - self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"') - return event + + event = MISPEvent() + event.distribution = distribution + event.info = info + event.analysis = analysis + event.threat = threat + event.published = published + + event.add_tag('infoleak:output-format="ail-daily"') + existing_event = self.pymisp.add_event(event) + return existing_event # Publish object to MISP - def pushToMISP(self, uuid_ail, path, tag): - self._p_source = path.split('/')[-5:] - self._p_source = '/'.join(self._p_source)[:-3] + def pushToMISP(self, uuid_ail, item_id, tag): if self.currentID_date != datetime.date.today(): #refresh id self.eventID_to_push = self.get_daily_event_id() @@ -138,42 +121,37 @@ class ObjectWrapper: mispTYPE = 'ail-leak' # paste object already exist - if self.paste_object_exist(self.eventID_to_push, self._p_source): + if self.paste_object_exist(self.eventID_to_push, item_id): # add new tag self.tag(self.attribute_to_tag, tag) - print(self._p_source + ' tagged: ' + tag) + print(item_id + ' tagged: ' + tag) #create object else: - self.add_new_object(uuid_ail, path, self._p_source, tag) + misp_obj = self.add_new_object(uuid_ail, item_id, tag) + + # deprecated + # try: + # templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + # except IndexError: + # valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) + # print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - try: - templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] - except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) - print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject) + r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True) if 'errors' in r: print(r) else: - # tag new object - self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source) - self.tag(self.attribute_to_tag, tag) - print('Pushed:', tag, '->', self._p_source) + print('Pushed:', tag, '->', item_id) - def paste_object_exist(self, eventId, source): - res = self.pymisp.search(controller='attributes', eventid=eventId, values=source) + def paste_object_exist(self, eventId, item_id): + res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id) # object already exist - if res['response']: - self.attribute_to_tag = res['response']['Attribute'][0]['uuid'] + if res.get('Attribute', []): + self.attribute_to_tag = res['Attribute'][0]['uuid'] return True # new object else: return False - def set_attribute_to_tag_uuid(self, eventId, source): - res = self.pymisp.search(controller='attributes', eventid=eventId, values=source) - self.attribute_to_tag = res['response']['Attribute'][0]['uuid'] - def tag(self, uuid, tag): self.pymisp.tag(uuid, tag) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index fe1fc304..4ee6bf89 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -19,6 +19,7 @@ import Screenshot import Correlate_object import AILObjects +import Export # # TODO: # FIXME: REFRACTOR ME => use UI/Global config sys.path.append('../../configs/keys') @@ -59,8 +60,12 @@ def tag_misp_object_attributes(l_ref_obj_attr, tags): for tag in tags: obj_attr.add_tag(tag) -def export_ail_item(item_id): +def export_ail_item(item_id, tags=[]): dict_metadata = Item.get_item({'id': item_id, 'date':True, 'tags':True, 'raw_content':True})[0] + # force tags + for tag in tags: + if tag not in dict_metadata['tags']: + dict_metadata['tags'].append(tag) #obj = MISPObject('ail-item', standalone=True) obj = MISPObject('ail-leak', standalone=True) @@ -69,6 +74,7 @@ def export_ail_item(item_id): l_obj_attr = [] l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['date']) ) l_obj_attr.append( obj.add_attribute('raw-data', value=item_id, data=dict_metadata['raw_content']) ) + l_obj_attr.append( obj.add_attribute('sensor', value=Export.get_ail_uuid()) ) # add tags if dict_metadata['tags']: diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 98a6891c..6b606dda 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -33,6 +33,9 @@ def get_item_date(item_id, add_separator=False): else: return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) +def get_source(item_id): + return item_id.split('/')[-5] + # # TODO: add an option to check the tag def is_crawled(item_id): return item_id.startswith('crawled') diff --git a/bin/packages/Item.py b/bin/packages/Item.py index dc8fdccd..28a7349b 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -48,7 +48,7 @@ def get_item_date(item_id, add_separator=False): return item_basic.get_item_date(item_id, add_separator=add_separator) def get_source(item_id): - return item_id.split('/')[-5] + return item_basic.get_source(item_id) def get_item_basename(item_id): return os.path.basename(item_id) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 0c6a695b..f609be0b 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -8,10 +8,10 @@ import datetime sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date -import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import item_basic from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -383,8 +383,8 @@ def add_obj_tag(object_type, object_id, tag, obj_date=None): r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id) # add domain tag - if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': - domain = Item.get_item_domain(object_id) + if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': + domain = item_basic.get_item_domain(object_id) add_tag("domain", tag, domain) else: r_serv_metadata.sadd('tag:{}'.format(object_id), tag) @@ -432,7 +432,7 @@ def delete_tag(object_type, tag, object_id, obj_date=None): # # TODO: move me def get_obj_date(object_type, object_id): if object_type == "item": - return int(Item.get_item_date(object_id)) + return int(item_basic.get_item_date(object_id)) else: return None @@ -573,3 +573,19 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 l_tagged_obj = list(l_tagged_obj) return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem} + + +#### TAGS EXPORT #### +# # TODO: +def is_updated_tags_to_export(): # by type + return False + +def get_list_of_solo_tags_to_export_by_type(export_type): # by type + if export_type in ['misp', 'thehive']: + return r_serv_db.smembers('whitelist_{}'.format(export_type)) + else: + return None + #r_serv_db.smembers('whitelist_hive') + + +#### -- ####