diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c15311e5..1013e546 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -273,6 +273,9 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x" + sleep 0.1 + ################################## # TRACKERS MODULES # ################################## diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index cc1e3ea8..4d92cfc6 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -98,7 +98,8 @@ class FeederImporter(AbstractImporter): gzip64_content = feeder.get_gzip64_content() return obj, f'{feeder_name} {gzip64_content}' else: # Messages save on DB - return obj, f'{feeder_name}' + if obj.exists(): + return obj, f'{feeder_name}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 1a8d097d..f62500c7 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -122,7 +122,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # TODO sanitize obj type obj_type = self.get_obj_type() - print(obj_type) if obj_type == 'image': self.obj = Images.Image(self.json_data['data-sha256']) @@ -226,6 +225,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): """ # meta = self.get_json_meta() + objs = set() + if self.obj: + objs.add(self.obj) + date, timestamp = self.get_message_date_timestamp() # REPLY @@ -245,14 +248,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC): message_id = self.get_message_id() message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) message = Messages.Message(message_id) + # create empty message if message don't exists + if not message.exists(): + message.create('') + objs.add(message) + if message.exists(): obj = Images.create(self.get_message_content()) obj.add(date, message) obj.set_parent(obj_global_id=message.get_global_id()) - else: - obj = None - if obj: + for obj in objs: # TODO PERF avoid parsing metas multpile times # CHAT chat = self.process_chat(obj, date, timestamp, reply_id=reply_id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index e6808bf4..59d2e125 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -287,7 +287,7 @@ class Message(AbstractObject): # self._set_translation(translated) # return translated - def create(self, content, translation, tags): + def create(self, content, translation=None, tags=[]): self._set_field('content', content) # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) if translation: @@ -315,7 +315,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None def create(obj_id, content, translation=None, tags=[]): message = Message(obj_id) if not message.exists(): - message.create(content, translation, tags) + message.create(content, translation=translation, tags=tags) return message diff --git a/bin/modules/Exif.py b/bin/modules/Exif.py new file mode 100755 index 00000000..865cc243 --- /dev/null +++ b/bin/modules/Exif.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Exif Module +====================== + +""" + +################################## +# Import External packages +################################## +import os +import sys + +from PIL import Image, ExifTags + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule + + +class Exif(AbstractModule): + """ + CveModule for AIL framework + """ + + def __init__(self): + super(Exif, self).__init__() + + # Waiting time in seconds between to message processed + self.pending_seconds = 1 + + # Send module state to logs + self.logger.info(f'Module {self.module_name} initialized') + + def compute(self, message): + image = self.get_obj() + print(image) + img = Image.open(image.get_filepath()) + img_exif = img.getexif() + print(img_exif) + if img_exif: + for key, val in img_exif.items(): + if key in ExifTags.TAGS: + print(f'{ExifTags.TAGS[key]}:{val}') + else: + print(f'{key}:{val}') + sys.exit(0) + + # tag = 'infoleak:automatic-detection="cve"' + # Send to Tags Queue + # self.add_message_to_queue(message=tag, queue='Tags') + + +if __name__ == '__main__': + + module = Exif() + module.run() diff --git a/bin/modules/Global.py b/bin/modules/Global.py index dd3cd900..d45f4806 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -124,9 +124,11 @@ class Global(AbstractModule): else: self.logger.info(f"Empty Item: {message} not processed") - elif self.obj: + elif self.obj.type == 'message': # TODO send to specific object queue => image, ... self.add_message_to_queue(obj=self.obj, queue='Item') + elif self.obj.type == 'image': + self.add_message_to_queue(obj=self.obj, queue='Image') else: self.logger.critical(f"Empty obj: {self.obj} {message} not processed") diff --git a/configs/modules.cfg b/configs/modules.cfg index b14e5c8b..41006974 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -1,3 +1,8 @@ +######## IMPORTERS ######## + +[Crawler] +publish = Importers,Tags + [ZMQModuleImporter] publish = Importers @@ -13,8 +18,6 @@ publish = Importers [PystemonModuleImporter] publish = Importers -#################################################### - [Mixer] subscribe = Importers publish = SaveObj @@ -22,9 +25,13 @@ publish = SaveObj [Sync_importer] publish = Importers,Tags +######## OBJ SAVER ######## + [Global] subscribe = SaveObj -publish = Item +publish = Item,Image + +######## ITEM + MESSAGE ######## [Duplicates] subscribe = Duplicate @@ -40,25 +47,19 @@ publish = Host subscribe = Host publish = D4_client -[D4Client] -subscribe = D4_client - -[Retro_Hunt_Module] -publish = Tags - -[Tracker_Typo_Squatting] +[Tracker_Typo_Squatting] # TODO MOVE ME subscribe = Host publish = Tags -[Tracker_Term] +[Tracker_Term] # TODO MOVE ME subscribe = Item publish = Tags -[Tracker_Regex] +[Tracker_Regex] # TODO MOVE ME subscribe = Item publish = Tags -[Tracker_Yara] +[Tracker_Yara] # TODO MOVE ME subscribe = Item publish = Tags @@ -70,7 +71,7 @@ publish = Tags subscribe = Item publish = Tags -[Languages] +[Languages] # TODO MOVE ME subscribe = Item [Categ] @@ -106,13 +107,6 @@ publish = Tags subscribe = Url publish = Tags -[Tags] -subscribe = Tags -publish = Tag_feed - -[MISP_Thehive_Auto_Push] -subscribe = Tag_feed - #[SentimentAnalysis] #subscribe = Item @@ -151,9 +145,6 @@ publish = Tags [SubmitPaste] publish = Importers -[Crawler] -publish = Importers,Tags - [IPAddress] subscribe = Item publish = Tags @@ -164,6 +155,32 @@ publish = Tags #[Sync_module] #publish = Sync +######## IMAGE ######## + +[Exif] +subscribe = Image +publish = Tags + + +######## CORE ######## + +[Tags] +subscribe = Tags +publish = Tag_feed + +[Retro_Hunt_Module] +publish = Tags + +######## OTHER ######## + +[D4Client] +subscribe = D4_client + +[MISP_Thehive_Auto_Push] +subscribe = Tag_feed + +######## DOC ######## + # [My_Module_Name] # subscribe = Global # Queue name # publish = Tags # Queue name