fix: [MISP auto export] fix MISP_TheHive_feeder

bin/MISP_The_Hive_feeder.py

@@ -20,8 +20,12 @@ from Helper import Process
 from packages import Paste
 import ailleakObject
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Tag
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import ConfigLoader
+import item_basic
 
 from pymisp import PyMISP
 
@@ -94,26 +98,22 @@ def create_the_hive_alert(source, path, tag):
 def feeder(message, count=0):
 
     if flag_the_hive or flag_misp:
-        tag, path = message.split(';')
+        tag, item_id = message.split(';')
+
         ## FIXME: remove it
-        if PASTES_FOLDER not in path:
+        if not item_basic.exist_item(item_id):
-            path = os.path.join(PASTES_FOLDER, path)
-        try:
-            paste = Paste.Paste(path)
-        except FileNotFoundError:
             if count < 10:
                 r_serv_db.zincrby('mess_not_saved_export', message, 1)
                 return 0
             else:
                 r_serv_db.zrem('mess_not_saved_export', message)
-                print('Error: {} do not exist, tag= {}'.format(path, tag))
+                print('Error: {} do not exist, tag= {}'.format(item_id, tag))
                 return 0
 
-        source = '/'.join(paste.p_path.split('/')[-6:])
+        source = item_basic.get_source(item_id)
 
         if HiveApi != False:
             if int(r_serv_db.get('hive:auto-alerts')) == 1:
-                whitelist_hive = r_serv_db.scard('whitelist_hive')
                 if r_serv_db.sismember('whitelist_hive', tag):
                     create_the_hive_alert(source, path, tag)
             else:
@@ -121,7 +121,7 @@ def feeder(message, count=0):
         if flag_misp:
             if int(r_serv_db.get('misp:auto-events')) == 1:
                 if r_serv_db.sismember('whitelist_misp', tag):
-                    misp_wrapper.pushToMISP(uuid_ail, path, tag)
+                    misp_wrapper.pushToMISP(uuid_ail, item_id, tag)
             else:
                 print('misp, auto events creation disable')
 
@@ -161,15 +161,15 @@ if __name__ == "__main__":
             print('Not connected to MISP')
 
         if flag_misp:
-            try:
+            #try:
-                misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
+            misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
-                r_serv_db.set('ail:misp', True)
+            r_serv_db.set('ail:misp', True)
-                print('Connected to MISP:', misp_url)
+            print('Connected to MISP:', misp_url)
-            except Exception as e:
+            #except Exception as e:
-                flag_misp = False
+            #    flag_misp = False
-                r_serv_db.set('ail:misp', False)
+            #    r_serv_db.set('ail:misp', False)
-                print(e)
+            #    print(e)
-                print('Not connected to MISP')
+            #    print('Not connected to MISP')
 
     # create The HIVE connection
     if flag_the_hive:

bin/Tags.py

@@ -45,4 +45,5 @@ if __name__ == '__main__':
             tag, item_id = message.split(';')
 
             Tag.add_tag("item", tag, item_id)
+
             p.populate_set_out(message, 'MISP_The_Hive_feeder')

bin/ailleakObject.py

@@ -4,7 +4,10 @@
 import os
 import sys
 
+from pymisp import MISPEvent, MISPObject
 from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
+MISPEvent
+
 from packages import Paste
 import datetime
 import json
@@ -12,28 +15,10 @@ from io import BytesIO
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
+import item_basic
 
-class AilLeakObject(AbstractMISPObjectGenerator):
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export'))
-    def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
+import MispExport
-        super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
-        self._uuid = uuid_ail
-        self._p_source = p_source
-        self._p_date = p_date
-        self._p_content = p_content
-        self._p_duplicate = p_duplicate
-        self._p_duplicate_number = p_duplicate_number
-        self.generate_attributes()
-
-    def generate_attributes(self):
-        self.add_attribute('origin', value=self._p_source, type='text')
-        self.add_attribute('last-seen', value=self._p_date, type='datetime')
-        if self._p_duplicate_number > 0:
-            self.add_attribute('duplicate', value=self._p_duplicate, type='text')
-            self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
-        self._pseudofile = BytesIO(self._p_content.encode())
-        res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag)
-        #res.add_shadow_attributes(tag)
-        self.add_attribute('sensor', value=self._uuid, type="text")
 
 class ObjectWrapper:
     def __init__(self, pymisp):
@@ -45,53 +30,48 @@ class ObjectWrapper:
         config_loader = None
         self.attribute_to_tag = None
 
-    def add_new_object(self, uuid_ail, path, p_source, tag):
+    def add_new_object(self, uuid_ail, item_id, tag):
         self.uuid_ail = uuid_ail
-        self.path = path
-        self.p_source = p_source
-        self.paste = Paste.Paste(path)
-        self.p_date = self.date_to_str(self.paste.p_date)
-        self.p_content = self.paste.get_p_content()
-        self.p_tag = tag
 
-        temp = self.paste._get_p_duplicate()
+        # self.paste = Paste.Paste(path)
+        # temp = self.paste._get_p_duplicate()
+        #
+        # #beautifier
+        # if not temp:
+        #     temp = ''
+        #
+        # p_duplicate_number = len(temp) if len(temp) >= 0 else 0
+        #
+        # to_ret = ""
+        # for dup in temp[:10]:
+        #     dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
+        #     dup = json.loads(dup)
+        #     algo = dup[0]
+        #     path = dup[1].split('/')[-6:]
+        #     path = '/'.join(path)[:-3] # -3 removes .gz
+        #     if algo == 'tlsh':
+        #         perc = 100 - int(dup[2])
+        #     else:
+        #         perc = dup[2]
+        #     to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
+        # p_duplicate = to_ret
 
-        #beautifier
+        return MispExport.export_ail_item(item_id, [tag])
-        if not temp:
-            temp = ''
-
-        p_duplicate_number = len(temp) if len(temp) >= 0 else 0
-
-        to_ret = ""
-        for dup in temp[:10]:
-            dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
-            dup = json.loads(dup)
-            algo = dup[0]
-            path = dup[1].split('/')[-6:]
-            path = '/'.join(path)[:-3] # -3 removes .gz
-            if algo == 'tlsh':
-                perc = 100 - int(dup[2])
-            else:
-                perc = dup[2]
-            to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
-        p_duplicate = to_ret
-
-        self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
 
     def date_to_str(self, date):
         return "{0}-{1}-{2}".format(date.year, date.month, date.day)
 
-    def get_all_related_events(self):
+    def get_all_related_events(self, to_search):
-        to_search = "Daily AIL-leaks"
+        result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False)
-        result = self.pymisp.search_all(to_search)
         events = []
-        for e in result['response']:
+        if result:
-            events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
+            for e in result:
+                events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
         return events
 
     def get_daily_event_id(self):
         to_match = "Daily AIL-leaks {}".format(datetime.date.today())
-        events = self.get_all_related_events()
+        events = self.get_all_related_events(to_match)
         for dic in events:
             info = dic['info']
             e_id = dic['id']
@@ -99,8 +79,8 @@ class ObjectWrapper:
                 print('Found: ', info, '->', e_id)
                 self.currentID_date = datetime.date.today()
                 return e_id
-        created_event = self.create_daily_event()['Event']
+        created_event = self.create_daily_event()
-        new_id = created_event['id']
+        new_id = created_event['Event']['id']
         print('New event created:', new_id)
         self.currentID_date = datetime.date.today()
         return new_id
@@ -120,17 +100,20 @@ class ObjectWrapper:
         orgc_id = None
         sharing_group_id = None
         date = None
-        event = self.pymisp.new_event(distribution, threat,
+
-                analysis, info, date,
+        event = MISPEvent()
-                published, orgc_id, org_id, sharing_group_id)
+        event.distribution = distribution
-        eventUuid = event['Event']['uuid']
+        event.info = info
-        self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"')
+        event.analysis = analysis
-        return event
+        event.threat = threat
+        event.published = published
+
+        event.add_tag('infoleak:output-format="ail-daily"')
+        existing_event = self.pymisp.add_event(event)
+        return existing_event
 
     # Publish object to MISP
-    def pushToMISP(self, uuid_ail, path, tag):
+    def pushToMISP(self, uuid_ail, item_id, tag):
-        self._p_source = path.split('/')[-5:]
-        self._p_source = '/'.join(self._p_source)[:-3]
 
         if self.currentID_date != datetime.date.today(): #refresh id
             self.eventID_to_push = self.get_daily_event_id()
@@ -138,42 +121,37 @@ class ObjectWrapper:
         mispTYPE = 'ail-leak'
 
         # paste object already exist
-        if self.paste_object_exist(self.eventID_to_push, self._p_source):
+        if self.paste_object_exist(self.eventID_to_push, item_id):
             # add new tag
             self.tag(self.attribute_to_tag, tag)
-            print(self._p_source + ' tagged: ' + tag)
+            print(item_id + ' tagged: ' + tag)
         #create object
         else:
-            self.add_new_object(uuid_ail, path, self._p_source, tag)
+            misp_obj = self.add_new_object(uuid_ail, item_id, tag)
+
+            # deprecated
+            # try:
+            #     templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
+            # except IndexError:
+            #     valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
+            #     print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
 
 
-            try:
+            r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True)
-                templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
-            except IndexError:
-                valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
-                print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
-            r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
             if 'errors' in r:
                 print(r)
             else:
-                # tag new object
+                print('Pushed:', tag, '->', item_id)
-                self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source)
-                self.tag(self.attribute_to_tag, tag)
-                print('Pushed:', tag, '->', self._p_source)
 
-    def paste_object_exist(self, eventId, source):
+    def paste_object_exist(self, eventId, item_id):
-        res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
+        res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id)
         # object already exist
-        if res['response']:
+        if res.get('Attribute', []):
-            self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
+            self.attribute_to_tag = res['Attribute'][0]['uuid']
             return True
         # new object
         else:
             return False
 
-    def set_attribute_to_tag_uuid(self, eventId, source):
-        res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
-        self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
-
     def tag(self, uuid, tag):
         self.pymisp.tag(uuid, tag)

bin/export/MispExport.py

@@ -19,6 +19,7 @@ import Screenshot
 import Correlate_object
 
 import AILObjects
+import Export
 
 # # TODO: # FIXME: REFRACTOR ME => use UI/Global config
 sys.path.append('../../configs/keys')
@@ -59,8 +60,12 @@ def tag_misp_object_attributes(l_ref_obj_attr, tags):
         for tag in tags:
             obj_attr.add_tag(tag)
 
-def export_ail_item(item_id):
+def export_ail_item(item_id, tags=[]):
     dict_metadata = Item.get_item({'id': item_id, 'date':True, 'tags':True, 'raw_content':True})[0]
+    # force tags
+    for tag in tags:
+        if tag not in dict_metadata['tags']:
+            dict_metadata['tags'].append(tag)
 
     #obj = MISPObject('ail-item', standalone=True)
     obj = MISPObject('ail-leak', standalone=True)
@@ -69,6 +74,7 @@ def export_ail_item(item_id):
     l_obj_attr = []
     l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['date']) )
     l_obj_attr.append( obj.add_attribute('raw-data', value=item_id, data=dict_metadata['raw_content']) )
+    l_obj_attr.append( obj.add_attribute('sensor', value=Export.get_ail_uuid()) )
 
     # add tags
     if dict_metadata['tags']:

bin/lib/item_basic.py

@@ -33,6 +33,9 @@ def get_item_date(item_id, add_separator=False):
     else:
         return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
 
+def get_source(item_id):
+    return item_id.split('/')[-5]
+
 # # TODO: add an option to check the tag
 def is_crawled(item_id):
     return item_id.startswith('crawled')

bin/packages/Item.py

@@ -48,7 +48,7 @@ def get_item_date(item_id, add_separator=False):
     return item_basic.get_item_date(item_id, add_separator=add_separator)
 
 def get_source(item_id):
-    return item_id.split('/')[-5]
+    return item_basic.get_source(item_id)
 
 def get_item_basename(item_id):
     return os.path.basename(item_id)

bin/packages/Tag.py

@@ -8,10 +8,10 @@ import datetime
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Date
-import Item
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
+import item_basic
 
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
@@ -383,8 +383,8 @@ def add_obj_tag(object_type, object_id, tag, obj_date=None):
         r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id)
 
         # add domain tag
-        if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
+        if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
-            domain = Item.get_item_domain(object_id)
+            domain = item_basic.get_item_domain(object_id)
             add_tag("domain", tag, domain)
     else:
         r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
@@ -432,7 +432,7 @@ def delete_tag(object_type, tag, object_id, obj_date=None):
 # # TODO: move me
 def get_obj_date(object_type, object_id):
     if object_type == "item":
-        return int(Item.get_item_date(object_id))
+        return int(item_basic.get_item_date(object_id))
     else:
         return None
 
@@ -573,3 +573,19 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
             l_tagged_obj = list(l_tagged_obj)
 
         return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
+
+
+#### TAGS EXPORT ####
+# # TODO:
+def is_updated_tags_to_export(): # by type
+    return False
+
+def get_list_of_solo_tags_to_export_by_type(export_type): # by type
+    if export_type in ['misp', 'thehive']:
+        return r_serv_db.smembers('whitelist_{}'.format(export_type))
+    else:
+        return None
+    #r_serv_db.smembers('whitelist_hive')
+
+
+#### -- ####