diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 34ccca8c..8532f488 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -103,14 +103,16 @@ class FeederImporter(AbstractImporter): if data_obj: objs.add(data_obj) + objs_messages = [] for obj in objs: if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() - return obj, f'{feeder_name} {gzip64_content}' + relay_message = f'{feeder_name} {gzip64_content}' + objs_messages.append({'obj': obj, 'message': relay_message}) else: # Messages save on DB if obj.exists() and obj.type != 'chat': - return obj, f'{feeder_name}' - + objs_messages.append({'obj': obj, 'message': feeder_name}) + return objs_messages class FeederModuleImporter(AbstractModule): def __init__(self): @@ -128,10 +130,8 @@ class FeederModuleImporter(AbstractModule): def compute(self, message): # TODO HANDLE Invalid JSON json_data = json.loads(message) - # TODO multiple objs + messages - obj, relay_message = self.importer.importer(json_data) - #### - self.add_message_to_queue(obj=obj, message=relay_message) + for obj_message in self.importer.importer(json_data): + self.add_message_to_queue(obj=obj_message['obj'], message=obj_message['message']) # Launch Importer diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 1dacbfe8..15edcd0a 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1558,14 +1558,14 @@ def get_obj_date(object_type, object_id): return None # API QUERY -def api_delete_obj_tags(tags=[], object_id=None, object_type="item"): +def api_delete_obj_tags(tags=[], object_id=None, object_type="item", subtype=''): if not object_id: return ({'status': 'error', 'reason': 'object id not found'}, 404) if not tags: return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400) for tag in tags: - res = delete_object_tag(tag, object_type, object_id, subtype='') + res = delete_object_tag(tag, object_type, object_id, subtype=subtype) if res: return res diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 029af5fb..9530b75d 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -22,6 +22,11 @@ AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', ' AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} +# TODO by object TYPE ???? +AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded', + 'domain', 'favicon', 'file-name', + 'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) + def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') if not ail_uuid: @@ -49,6 +54,9 @@ def generate_uuid(): def get_all_objects(): return AIL_OBJECTS +def is_object_type(obj_type): + return obj_type in AIL_OBJECTS + def get_objects_with_subtypes(): return AIL_OBJECTS_WITH_SUBTYPES @@ -69,6 +77,9 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype return r_object.smembers(f'all_chat:subtypes') return [] +def get_default_correlation_objects(): + return AIL_OBJECTS_CORRELATIONS_DEFAULT + def get_obj_queued(): return ['item', 'image'] diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 451fcc47..e91f42b9 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -139,6 +139,10 @@ class AILQueue: def error(self): r_queues.hdel(f'modules', f'{self.pid}:{self.name}') + def end(self): + self.clear() + r_queues.hdel(f'modules', f'{self.pid}:{self.name}') + def get_queues_modules(): return r_queues.hkeys('queues') diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index b8d51975..397d5aca 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -323,6 +323,63 @@ def get_username_meta_from_global_id(username_global_id): username = Usernames.Username(username_id, instance_uuid) return username.get_meta() + +# TODO Filter +## Instance type +## Chats IDS +## SubChats IDS +## Threads IDS +## Daterange +def get_messages_iterator(filters={}): + + for instance_uuid in get_chat_service_instances(): + + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + messages, _ = subchannel._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads + + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + _, _ = thread._get_messages(nb=-1) + for mess in messages: + message_id, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + + # messages + messages, _ = chat._get_messages(nb=-1) + for mess in messages: + _, _, message_id = mess[0].split(':', ) + yield Messages.Message(message_id) + # threads ??? + +def get_nb_messages_iterator(filters={}): + nb_messages = 0 + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + # subchannels + for subchannel_gid in chat.get_subchannels(): + _, _, subchannel_id = subchannel_gid.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(subchannel_id, instance_uuid) + nb_messages += subchannel.get_nb_messages() + # threads + for threads in chat.get_threads(): + thread = ChatThreads.ChatThread(threads['id'], instance_uuid) + nb_messages += thread.get_nb_messages() + # messages + nb_messages += chat.get_nb_messages() + return nb_messages + #### API #### def api_get_chat_service_instance(chat_instance_uuid): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index dde776b0..0e6dadd0 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -51,7 +51,7 @@ class Chat(AbstractChatObject): def get_link(self, flask_context=False): if flask_context: - url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + url = url_for('chats_explorer.chats_explorer_chat', subtype=self.subtype, id=self.id) else: url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' return url diff --git a/bin/lib/objects/Images.py b/bin/lib/objects/Images.py index 280ac06c..9d71b7d5 100755 --- a/bin/lib/objects/Images.py +++ b/bin/lib/objects/Images.py @@ -71,7 +71,10 @@ class Image(AbstractDaterangeObject): return file_content def get_content(self, r_type='str'): - return self.get_file_content() + if r_type == 'str': + return None + else: + return self.get_file_content() def get_misp_object(self): obj_attrs = [] diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 395ec680..8f1d16d4 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -85,7 +85,8 @@ class Message(AbstractObject): if r_type == 'str': return content elif r_type == 'bytes': - return content.encode() + if content: + return content.encode() def get_date(self): timestamp = self.get_timestamp() @@ -352,7 +353,6 @@ def create(obj_id, content, translation=None, tags=[]): message.create(content, translation=translation, tags=tags) return message - # TODO Encode translation diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 5777bd17..73ae7525 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -49,9 +49,9 @@ class UserAccount(AbstractSubtypeObject): def get_link(self, flask_context=False): if flask_context: - url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + url = url_for('chats_explorer.objects_user_account', type=self.type, subtype=self.subtype, id=self.id) else: - url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + url = f'{baseurl}/objects/user-account?&subtype={self.subtype}&id={self.id}' return url def get_svg_icon(self): # TODO change icon/color @@ -127,6 +127,13 @@ class UserAccount(AbstractSubtypeObject): def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) + def get_messages(self): + messages = [] + for mess in self.get_correlation('message'): + messages.append(f'message:{mess}') + return messages + + def get_messages_by_chat_obj(self, chat_obj): messages = [] for mess in self.get_correlation_iter_obj(chat_obj, 'message'): diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 5c083e0f..6f4f750d 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -12,12 +12,14 @@ from lib.exceptions import AILObjectUnknown from lib.ConfigLoader import ConfigLoader -from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes +from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes, get_default_correlation_objects from lib import correlations_engine from lib import relationships_engine from lib import btc_ail from lib import Tag +from lib import chats_viewer + from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads @@ -32,7 +34,7 @@ from lib.objects import FilesNames from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Images -from lib.objects.Messages import Message +from lib.objects import Messages from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles @@ -53,13 +55,16 @@ def is_object_subtype(obj_type): def is_valid_object_subtype(obj_type, subtype): return subtype in get_object_all_subtypes(obj_type) -def sanitize_objs_types(objs): +def sanitize_objs_types(objs, default=False): l_types = [] for obj in objs: if is_valid_object_type(obj): l_types.append(obj) if not l_types: - l_types = get_all_objects() + if default: + l_types = get_default_correlation_objects() + else: + l_types = get_all_objects() return l_types #### OBJECT #### @@ -87,7 +92,7 @@ def get_object(obj_type, subtype, obj_id): elif obj_type == 'image': return Images.Image(obj_id) elif obj_type == 'message': - return Message(obj_id) + return Messages.Message(obj_id) elif obj_type == 'screenshot': return Screenshot(obj_id) elif obj_type == 'title': @@ -249,8 +254,9 @@ def get_objects_meta(objs, options=set(), flask_context=False): def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) - meta = obj.get_meta() - meta['icon'] = obj.get_svg_icon() + meta = obj.get_meta(options={'chat', 'chats', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads', 'username'}) + # meta['icon'] = obj.get_svg_icon() + meta['svg_icon'] = obj.get_svg_icon() if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': @@ -293,6 +299,9 @@ def obj_iterator(obj_type, filters): return get_all_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.get_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_messages_iterator(filters=filters) + def card_objs_iterators(filters): nb = 0 @@ -307,6 +316,8 @@ def card_obj_iterator(obj_type, filters): return get_nb_items_objects(filters=filters) elif obj_type == 'pgp': return Pgps.nb_all_pgps_objects(filters=filters) + elif obj_type == 'message': + return chats_viewer.get_nb_messages_iterator(filters=filters) def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME """ diff --git a/bin/lib/telegram.py b/bin/lib/telegram.py index aca08d38..02984066 100755 --- a/bin/lib/telegram.py +++ b/bin/lib/telegram.py @@ -22,11 +22,8 @@ REGEX_JOIN_HASH = re.compile(r'[0-9a-zA-z-]+') ## ## -def save_item_correlation(username, item_id, item_date): - Username.save_item_correlation('telegram', username, item_id, item_date) - -def save_telegram_invite_hash(invite_hash, item_id): - r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}') +def save_telegram_invite_hash(invite_hash, obj_global_id): + r_obj.sadd('telegram:invite_code', f'{invite_hash};{obj_global_id}') def get_data_from_telegram_url(base_url, url_path): dict_url = {} diff --git a/bin/modules/ApiKey.py b/bin/modules/ApiKey.py index bf54c095..2a9d11f8 100755 --- a/bin/modules/ApiKey.py +++ b/bin/modules/ApiKey.py @@ -61,7 +61,7 @@ class ApiKey(AbstractModule): if google_api_key: print(f'found google api key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="google-api-key"' self.add_message_to_queue(message=tag, queue='Tags') @@ -69,10 +69,10 @@ class ApiKey(AbstractModule): # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY if aws_access_key: print(f'found AWS key: {to_print}') - self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}') if aws_secret_key: print(f'found AWS secret key') - self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}') + self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}') tag = 'infoleak:automatic-detection="aws-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Credential.py b/bin/modules/Credential.py index f20f2ba9..6d0df6a7 100755 --- a/bin/modules/Credential.py +++ b/bin/modules/Credential.py @@ -103,11 +103,11 @@ class Credential(AbstractModule): print(message) - to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{item.get_id()}' + to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}' # num of creds above threshold, publish an alert if nb_cred > self.criticalNumberToAlert: - print(f"========> Found more than 10 credentials in this file : {item.get_id()}") + print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}") self.redis_logger.warning(to_print) tag = 'infoleak:automatic-detection="credential"' diff --git a/bin/modules/CreditCards.py b/bin/modules/CreditCards.py index 1d8411bb..c237bf05 100755 --- a/bin/modules/CreditCards.py +++ b/bin/modules/CreditCards.py @@ -86,7 +86,7 @@ class CreditCards(AbstractModule): # print(creditcard_set) to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};' if creditcard_set: - mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{item.id}' + mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}' print(mess) self.redis_logger.warning(mess) @@ -96,7 +96,7 @@ class CreditCards(AbstractModule): if r_result: return creditcard_set else: - self.redis_logger.info(f'{to_print}CreditCard related;{item.id}') + self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}') if __name__ == '__main__': diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 5a83689f..036b02b4 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -149,7 +149,7 @@ class Cryptocurrencies(AbstractModule, ABC): item.get_date(), item.get_basename()) self.redis_logger.warning('{}Detected {} {} private key;{}'.format( - to_print, len(private_keys), currency['name'], item_id)) + to_print, len(private_keys), currency['name'], self.obj.get_global_id())) else: private_keys = [] diff --git a/bin/modules/CveModule.py b/bin/modules/CveModule.py index 6904ee28..1387686b 100755 --- a/bin/modules/CveModule.py +++ b/bin/modules/CveModule.py @@ -56,7 +56,7 @@ class CveModule(AbstractModule): cve = Cves.Cve(cve_id) cve.add(date, item) - warning = f'{item_id} contains CVEs {cves}' + warning = f'{self.obj.get_global_id()} contains CVEs {cves}' print(warning) self.redis_logger.warning(warning) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index b4620ee2..fa0a232f 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -82,20 +82,20 @@ class DomClassifier(AbstractModule): localizeddomains = self.dom_classifier.include(expression=self.cc_tld) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}") if self.cc: localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) if localizeddomains: print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}") if r_result: return self.dom_classifier.vdomain except IOError as err: self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") - raise Exception(f"CRC Checksum Failed on: {item.get_id()}") + raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}") if __name__ == "__main__": diff --git a/bin/modules/Duplicates.py b/bin/modules/Duplicates.py index 57641979..edee6025 100755 --- a/bin/modules/Duplicates.py +++ b/bin/modules/Duplicates.py @@ -92,10 +92,10 @@ class Duplicates(AbstractModule): Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id()) if nb_duplicates: - self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{item.get_id()}') + self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}') y = time.time() - print(f'{item.get_id()} Processed in {y-x} sec') + print(f'{self.obj.get_global_id()} Processed in {y-x} sec') # self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x)) diff --git a/bin/modules/Global.py b/bin/modules/Global.py index 3f473aed..763a7b89 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -81,10 +81,9 @@ class Global(AbstractModule): def compute(self, message, r_result=False): # TODO move OBJ ID sanitization to importer # Recovering the streamed message infos - gzip64encoded = message if self.obj.type == 'item': - if gzip64encoded: + if message: # Creating the full filepath filename = os.path.join(self.ITEMS_FOLDER, self.obj.id) @@ -97,7 +96,7 @@ class Global(AbstractModule): else: # Decode compressed base64 - decoded = base64.standard_b64decode(gzip64encoded) + decoded = base64.standard_b64decode(message) new_file_content = self.gunzip_bytes_obj(filename, decoded) # TODO REWRITE ME diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 31a0ff68..8899ca43 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -82,8 +82,8 @@ class IPAddress(AbstractModule): matching_ips.append(address) if len(matching_ips) > 0: - self.logger.info(f'{item.get_id()} contains {len(matching_ips)} IPs') - self.redis_logger.warning(f'{item.get_id()} contains {item.get_id()} IPs') + self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs') + self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs') # Tag message with IP tag = 'infoleak:automatic-detection="ip"' diff --git a/bin/modules/Iban.py b/bin/modules/Iban.py index aa1aa5d6..7b0c66d0 100755 --- a/bin/modules/Iban.py +++ b/bin/modules/Iban.py @@ -95,7 +95,7 @@ class Iban(AbstractModule): # Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1) to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};' - self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}') + self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}') # Tags tag = 'infoleak:automatic-detection="iban"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/Keys.py b/bin/modules/Keys.py index a2e7288d..2fdb96f8 100755 --- a/bin/modules/Keys.py +++ b/bin/modules/Keys.py @@ -63,7 +63,7 @@ class Keys(AbstractModule): get_pgp_content = False if KeyEnum.PGP_MESSAGE.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a PGP enc message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message') tag = 'infoleak:automatic-detection="pgp-message"' self.add_message_to_queue(message=tag, queue='Tags') @@ -81,21 +81,21 @@ class Keys(AbstractModule): get_pgp_content = True if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message') tag = 'infoleak:automatic-detection="pgp-private-key"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.CERTIFICATE.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a certificate message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message') tag = 'infoleak:automatic-detection="certificate"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.RSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message') print('rsa private key message found') tag = 'infoleak:automatic-detection="rsa-private-key"' @@ -103,7 +103,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message') print('private key message found') tag = 'infoleak:automatic-detection="private-key"' @@ -111,7 +111,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message') print('encrypted private key message found') tag = 'infoleak:automatic-detection="encrypted-private-key"' @@ -119,7 +119,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print('openssh private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -127,7 +127,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message') print('SSH2 private key message found') tag = 'infoleak:automatic-detection="private-ssh-key"' @@ -135,7 +135,7 @@ class Keys(AbstractModule): # find = True if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message') print('OpenVPN Static key message found') tag = 'infoleak:automatic-detection="vpn-static-key"' @@ -143,21 +143,21 @@ class Keys(AbstractModule): # find = True if KeyEnum.DSA_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message') tag = 'infoleak:automatic-detection="dsa-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.EC_PRIVATE_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has an ec private key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message') tag = 'infoleak:automatic-detection="ec-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PUBLIC_KEY.value in content: - self.redis_logger.warning(f'{item.get_basename()} has a public key message') + self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message') tag = 'infoleak:automatic-detection="public-key"' self.add_message_to_queue(message=tag, queue='Tags') diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index de4d1287..fc7a5956 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -70,7 +70,7 @@ class LibInjection(AbstractModule): print(f"Detected (libinjection) SQL in URL: {item_id}") print(unquote(url)) - to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' + to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' self.redis_logger.warning(to_print) # Add tag diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index 82cac546..8a3a66a4 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -172,7 +172,7 @@ class Mail(AbstractModule): # for tld in mx_tlds: # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) - msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item.id}' + msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}' if num_valid_email > self.mail_threshold: print(f'{item.id} Checked {num_valid_email} e-mail(s)') self.redis_logger.warning(msg) diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 681bae0c..34e50247 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -98,8 +98,8 @@ class Onion(AbstractModule): print(f'{domain} added to crawler queue: {task_uuid}') else: to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};' - print(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') - self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') + print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') + self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}') # TAG Item tag = 'infoleak:automatic-detection="onion"' diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index 0c01a40f..8eae16cc 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -62,7 +62,7 @@ class Phone(AbstractModule): tag = 'infoleak:automatic-detection="phone-number"' self.add_message_to_queue(message=tag, queue='Tags') - self.redis_logger.warning(f'{item.get_id()} contains {len(phone)} Phone numbers') + self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers') # # List of the regex results in the Item, may be null # results = self.REG_PHONE.findall(content) diff --git a/bin/modules/SQLInjectionDetection.py b/bin/modules/SQLInjectionDetection.py index 7c1bcb4f..953fe82e 100755 --- a/bin/modules/SQLInjectionDetection.py +++ b/bin/modules/SQLInjectionDetection.py @@ -53,7 +53,7 @@ class SQLInjectionDetection(AbstractModule): print(f"Detected SQL in URL: {item.id}") print(urllib.request.unquote(url)) - to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item.id}' + to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}' self.redis_logger.warning(to_print) # Tag diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 33ea1c80..0f613eae 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -41,7 +41,7 @@ class Tags(AbstractModule): # Create a new tag item.add_tag(tag) - print(f'{item.get_id()}: Tagged {tag}') + print(f'{self.obj.get_global_id()}: Tagged {tag}') # Forward message to channel self.add_message_to_queue(message=tag, queue='Tag_feed') diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 140948c2..0cba6ef5 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -62,7 +62,7 @@ class Telegram(AbstractModule): print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: - telegram.save_telegram_invite_hash(invite_hash, item.id) + telegram.save_telegram_invite_hash(invite_hash, self.obj.get_global_id()) print(f'invite code: {invite_hash}') invite_code_found = True diff --git a/bin/modules/Urls.py b/bin/modules/Urls.py index c5a3dfe4..ed510a84 100755 --- a/bin/modules/Urls.py +++ b/bin/modules/Urls.py @@ -78,7 +78,7 @@ class Urls(AbstractModule): except AttributeError: url = url_decoded['url'] - print(url, item.get_id()) + print(url, self.obj.get_global_id()) self.add_message_to_queue(message=str(url), queue='Url') self.logger.debug(f"url_parsed: {url}") diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index affd6c6a..273f7580 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -179,7 +179,10 @@ class AbstractModule(ABC): trace = traceback.format_tb(err.__traceback__) trace = ''.join(trace) self.logger.critical(f"Error in module {self.module_name}: {__name__} : {err}") - self.logger.critical(f"Module {self.module_name} input message: {message}") + if message: + self.logger.critical(f"Module {self.module_name} input message: {message}") + if self.obj: + self.logger.critical(f"{self.module_name} Obj: {self.obj.get_global_id()}") self.logger.critical(trace) if isinstance(err, ModuleQueueError): diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index f6854781..6830a663 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -88,6 +88,9 @@ class Retro_Hunt_Module(AbstractModule): for obj in ail_objects.obj_iterator(obj_type, filters): self.obj = obj content = obj.get_content(r_type='bytes') + if not content: + continue + rule.match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 43fa5764..c23d1589 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -116,8 +116,8 @@ class Tracker_Regex(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked regex found: {tracker_name} in {obj_id}') - self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}') + print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index a3067674..2d0a02a5 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -93,7 +93,7 @@ class Tracker_Term(AbstractModule): try: dict_words_freq = Tracker.get_text_word_frequency(content) except TimeoutException: - self.redis_logger.warning(f"{obj.get_id()} processing timeout") + self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout") else: signal.alarm(0) @@ -124,8 +124,8 @@ class Tracker_Term(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked term {tracker_uuid} found: {tracker_name} in {obj_id}') - self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}') + print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(), obj_id) diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index 9e093b3e..d81d430f 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -75,8 +75,8 @@ class Tracker_Typo_Squatting(AbstractModule): if ail_objects.is_filtered(obj, filters): continue - print(f'new tracked typosquatting found: {tracked} in {obj_id}') - self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}') + print(f'new tracked typosquatting found: {tracked} in {self.obj.get_global_id()}') + self.redis_logger.warning(f'tracker typosquatting: {tracked} in {self.obj.get_global_id()}') tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 344c77a7..29542553 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -62,13 +62,15 @@ class Tracker_Yara(AbstractModule): return None content = self.obj.get_content(r_type='bytes') + if not content: + return None try: yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60) if yara_match: - self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}') - print(f'{self.obj.get_id()}: {yara_match}') + self.redis_logger.warning(f'tracker yara: new match {self.obj.get_global_id()}: {yara_match}') + print(f'{self.obj.get_global_id()}: {yara_match}') except yara.TimeoutError: print(f'{self.obj.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') diff --git a/requirements.txt b/requirements.txt index bf236ba4..1bcb3415 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ textblob>=0.15.3 html2text>=2020.1.16 beautifulsoup4>4.8.2 -#Crawler +# Crawler scrapy>2.0.0 scrapy-splash>=0.7.2 @@ -46,6 +46,9 @@ gcld3 libretranslatepy lexilang +# Demoji +git+https://github.com/ail-project/demoji + #Graph numpy>1.18.1 matplotlib>3.2.1 diff --git a/tools/reprocess_objects.py b/tools/reprocess_objects.py new file mode 100755 index 00000000..6d0ffd16 --- /dev/null +++ b/tools/reprocess_objects.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Reprocess AIL Objects by Object Type +================ + +Send ALL objects by type in queues + +""" + +import argparse +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ail_core import is_object_type +from lib import ail_queues +from lib.objects import ail_objects + +def reprocess_message_objects(object_type): + queue = ail_queues.AILQueue('FeederModuleImporter', -1) + for obj in ail_objects.obj_iterator(object_type, filters={}): + queue.send_message(obj.get_global_id(), message='reprocess') + queue.end() + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Reprocess AIL Objects') + parser.add_argument('-t', '--type', type=str, help='AIL Object Type', required=True) + + args = parser.parse_args() + if not args.type: + parser.print_help() + sys.exit(0) + + obj_type = args.type + if not is_object_type(obj_type): + raise Exception(f'Invalid Object Type: {obj_type}') + if obj_type not in ['item', 'message']: # TODO image + raise Exception(f'Currently not supported Object Type: {obj_type}') + + reprocess_message_objects(obj_type) \ No newline at end of file diff --git a/update/v5.4/Update.sh b/update/v5.4/Update.sh index 87f2ab7f..3ae2aeb0 100755 --- a/update/v5.4/Update.sh +++ b/update/v5.4/Update.sh @@ -25,6 +25,7 @@ echo -e $GREEN"Updating python packages ..."$DEFAULT echo "" pip install -U pylacus pip install -U lexilang +pip install git+https://github.com/ail-project/demoji bash ${AIL_BIN}/LAUNCH.sh -lrv diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 24c1bd84..b851a5b1 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -92,7 +92,9 @@ def chats_explorer_chat(): else: chat = chat[0] languages = Language.get_translation_languages() - return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) + return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(chat['id'], chat['type'], chat['subtype']), + translation_languages=languages, translation_target=target) @chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET']) @login_required @@ -137,7 +139,9 @@ def objects_subchannel_messages(): else: subchannel = subchannel[0] languages = Language.get_translation_languages() - return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) + return render_template('SubChannelMessages.html', subchannel=subchannel, + ail_tags=Tag.get_modal_add_tags(subchannel['id'], subchannel['type'], subchannel['subtype']), + bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) @chats_explorer.route("/chats/explorer/thread", methods=['GET']) @login_required @@ -279,4 +283,5 @@ def objects_user_account(): user_account = user_account[0] languages = Language.get_translation_languages() return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']), translation_languages=languages, translation_target=target) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 9cd21b24..1222755d 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -165,7 +165,7 @@ def show_correlation(): related_btc = bool(request.args.get('related_btc', False)) - filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True) # check if obj_id exist if not ail_objects.exists_obj(obj_type, subtype, obj_id): @@ -190,8 +190,11 @@ def show_correlation(): else: dict_object["subtype"] = '' dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) + dict_object["metadata_card"]['tags_safe'] = True return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, - tags_selector_data=Tag.get_tags_selector_data()) + tags_selector_data=Tag.get_tags_selector_data(), + meta=dict_object["metadata_card"], + ail_tags=dict_object["metadata_card"]["add_tags_modal"]) @correlation.route('/correlation/get/description') @login_required @@ -206,7 +209,10 @@ def get_description(): return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 # object exist else: - res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'}, + options = {'icon', 'tags', 'tags_safe'} + if obj_type == 'message': + options.add('content') + res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=True) if 'tags' in res: res['tags'] = list(res['tags']) diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py index cf3cf688..70764179 100644 --- a/var/www/blueprints/investigations_b.py +++ b/var/www/blueprints/investigations_b.py @@ -209,6 +209,14 @@ def unregister_investigation(): def get_investigations_selector_json(): return jsonify(Investigations.get_investigations_selector()) +@investigations_b.route("/object/gid") +@login_required +@login_read_only +def get_object_gid(): + obj_global_id = request.args.get('gid') + ail_obj = ail_objects.get_obj_from_global_id(obj_global_id) + url = ail_obj.get_link(flask_context=True) + return redirect(url) # # @investigations_b.route("/object/item") #completely shows the paste in a new tab diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index f833ab16..2bf7faef 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -170,7 +170,11 @@ def tag_confirm(): if not obj.exists(): abort(404) Tag.confirm_tag(tag, obj) - return redirect(obj.get_link(flask_context=True)) + + if request.referrer: + return redirect(request.referrer) + else: + return redirect(obj.get_link(flask_context=True)) @tags_ui.route('/tag/add_tags') @login_required @@ -192,22 +196,27 @@ def add_tags(): if res[1] != 200: return str(res[0]) - return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True)) -@tags_ui.route('/tag/delete_tag') +@tags_ui.route('/tag/delete_tag') # TODO FIX REQUEST PARAMETER @login_required @login_analyst def delete_tag(): - - object_type = request.args.get('object_type') - object_id = request.args.get('object_id') - subtype = '' # TODO: handle subtype object + object_type = request.args.get('type') + subtype = request.args.get('subtype', '') + object_id = request.args.get('id') tag = request.args.get('tag') - res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type) + res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type, subtype=subtype) if res[1] != 200: return str(res[0]) - return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) + if request.referrer: + return redirect(request.referrer) + else: + return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) @tags_ui.route('/tag/get_all_tags') diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 3f0df03f..64b842fe 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -44,6 +44,7 @@ def event_stream(): pubsub.psubscribe("Script" + '.*') for msg in pubsub.listen(): + # print(msg) type = msg['type'] pattern = msg['pattern'] channel = msg['channel'] @@ -77,7 +78,7 @@ def dashboard_alert(log): log = log[46:].split(';') if len(log) == 6: date_time = datetime_from_utc_to_local(utc_str) - path = url_for('objects_item.showItem', id=log[5]) + path = url_for('investigations_b.get_object_gid', gid=log[5]) res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], 'paste': log[3], 'message': log[4], 'path': path} diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index db5b4fb4..1ac1577b 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -162,7 +162,7 @@ - + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
+
+

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :

+ {% if meta['icon'] %} +
{{ meta['id'] }}
+ {% endif %} + + + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_chat_subchannel.html b/var/www/templates/chats_explorer/card_chat_subchannel.html new file mode 100644 index 00000000..3fd9a33e --- /dev/null +++ b/var/www/templates/chats_explorer/card_chat_subchannel.html @@ -0,0 +1,103 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
+
+

{% if meta['chat']['name'] %}{{ meta['chat']['name'] }} {% else %} {{ meta['chat']['id'] }}{% endif %} - {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :

{{ meta["id"] }} + + + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_image.html b/var/www/templates/chats_explorer/card_image.html new file mode 100644 index 00000000..e97cea81 --- /dev/null +++ b/var/www/templates/chats_explorer/card_image.html @@ -0,0 +1,104 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + + +
+
+

{{ meta["id"] }} :

+ + + {% with obj_type='image', obj_id=meta['id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/card_user_account.html b/var/www/templates/chats_explorer/card_user_account.html new file mode 100644 index 00000000..f5514967 --- /dev/null +++ b/var/www/templates/chats_explorer/card_user_account.html @@ -0,0 +1,97 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + +
+
+

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %}

+ {% if meta['icon'] %} +
{{ meta['id'] }}
+ {% endif %} + + + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index ecf45fd5..872f3b55 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -54,67 +54,9 @@
-
- -
-

{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :

- {% if chat['icon'] %} -
{{ chat['id'] }}
- {% endif %} -
    -
  • - - - - - - - - - - - - - - - - - - - - - - - -
    NameIDCreated atFirst SeenLast SeenNB Sub-ChannelsParticipants
    {{ chat['name'] }}{{ chat['id'] }}{{ chat['created_at'] }} - {% if chat['first_seen'] %} - {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} - {% endif %} - - {% if chat['last_seen'] %} - {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }} - {% endif %} - {{ chat['nb_subchannels'] }} - {{ chat['nb_participants']}} -
    - {% if chat['info'] %} -
  • - 
    {{ chat['info'] }}
    - {% if chat['translation_info'] %} -
    - 
    {{ chat['translation_info'] }}
    - {% endif %} -
    • - {% endif %} - -
- -
-
- - {% for tag in chat['tags_messages'] %} - {{ tag }} {{ chat['tags_messages'][tag] }} - {% endfor %} + {% with meta=chat %} + {% include 'chats_explorer/card_chat.html' %} + {% endwith %} {% if chat['subchannels'] %}

Sub-Channels:

diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html index 7d869733..308f5959 100644 --- a/var/www/templates/chats_explorer/user_account.html +++ b/var/www/templates/chats_explorer/user_account.html @@ -32,75 +32,7 @@
-
- -
-

{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %}

- {% if meta['icon'] %} -
{{ meta['id'] }}
- {% endif %} -
    -
  • - - - - - - - - - - - - - - - - - - - -
    usernameIDFirst SeenLast SeenNB Chats
    {{ meta['username']['id'] }}{{ meta['id'] }} - {% if meta['first_seen'] %} - {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} - {% endif %} - - {% if meta['last_seen'] %} - {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} - {% endif %} - {{ meta['chats'] | length }}
    - {% if meta['info'] %} -
  • - 
    {{ meta['info'] }}
    - {% if meta['translation_info'] %} -
    - 
    {{ meta['translation_info'] }}
    - {% endif %} -
    • - {% endif %} - -
- -
- -{#
#} -{# {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%}#} -{# {% include 'modals/investigations_register_obj.html' %}#} -{# {% endwith %}#} -{#
#} -{# #} -{#
#} -{#
#} -
- -
-
+ {% include 'chats_explorer/card_user_account.html' %} {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} {% include 'chats_explorer/block_translation.html' %} diff --git a/var/www/templates/correlation/metadata_card_chat.html b/var/www/templates/correlation/metadata_card_chat.html deleted file mode 100644 index 4e672e1a..00000000 --- a/var/www/templates/correlation/metadata_card_chat.html +++ /dev/null @@ -1,78 +0,0 @@ - - - -{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#} -{# {% include 'modals/add_tags.html' %}#} -{#{% endwith %}#} - -{% include 'modals/edit_tag.html' %} - -
-
-

{{ dict_object["correlation_id"] }}

- {{ dict_object }} -
{{ dict_object["correlation_id"] }}
-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - -
    Object typeFirst seenLast seenNb seen
    - - - - {{ dict_object["metadata"]["icon"]["icon"] }} - - - {{ dict_object["object_type"] }} - {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    -
    -
    -
    -
    -
    -
    • - -
  • -
    -
    - Tags: - {% for tag in dict_object["metadata"]['tags'] %} - - {% endfor %} - -
    -
    • -
- - {% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %} - {% include 'modals/investigations_register_obj.html' %} - {% endwith %} - - -
-
- diff --git a/var/www/templates/correlation/metadata_card_cookie_name.html b/var/www/templates/correlation/metadata_card_cookie_name.html index 5a926182..514c8485 100644 --- a/var/www/templates/correlation/metadata_card_cookie_name.html +++ b/var/www/templates/correlation/metadata_card_cookie_name.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_cryptocurrency.html b/var/www/templates/correlation/metadata_card_cryptocurrency.html index 2698caa5..30321b67 100644 --- a/var/www/templates/correlation/metadata_card_cryptocurrency.html +++ b/var/www/templates/correlation/metadata_card_cryptocurrency.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_cve.html b/var/www/templates/correlation/metadata_card_cve.html index 1e166ddc..2a4141c9 100644 --- a/var/www/templates/correlation/metadata_card_cve.html +++ b/var/www/templates/correlation/metadata_card_cve.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index 1e292a80..25d9e499 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -32,7 +32,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata_card"]["mimetype"] }} diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html index 0e538916..ffa09537 100644 --- a/var/www/templates/correlation/metadata_card_domain.html +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -21,7 +21,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_etag.html b/var/www/templates/correlation/metadata_card_etag.html index cc599227..1ce11821 100644 --- a/var/www/templates/correlation/metadata_card_etag.html +++ b/var/www/templates/correlation/metadata_card_etag.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_favicon.html b/var/www/templates/correlation/metadata_card_favicon.html index 6b12ab3b..496cbb77 100644 --- a/var/www/templates/correlation/metadata_card_favicon.html +++ b/var/www/templates/correlation/metadata_card_favicon.html @@ -29,7 +29,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_hhhash.html b/var/www/templates/correlation/metadata_card_hhhash.html index c1474605..5b37ced3 100644 --- a/var/www/templates/correlation/metadata_card_hhhash.html +++ b/var/www/templates/correlation/metadata_card_hhhash.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_item.html b/var/www/templates/correlation/metadata_card_item.html index 194ec200..cf5395d1 100644 --- a/var/www/templates/correlation/metadata_card_item.html +++ b/var/www/templates/correlation/metadata_card_item.html @@ -18,7 +18,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_pgp.html b/var/www/templates/correlation/metadata_card_pgp.html index 03d73a2b..bb870dcd 100644 --- a/var/www/templates/correlation/metadata_card_pgp.html +++ b/var/www/templates/correlation/metadata_card_pgp.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/metadata_card_title.html b/var/www/templates/correlation/metadata_card_title.html index cd943349..7b1140f0 100644 --- a/var/www/templates/correlation/metadata_card_title.html +++ b/var/www/templates/correlation/metadata_card_title.html @@ -30,7 +30,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["object_type"] }} diff --git a/var/www/templates/correlation/metadata_card_username.html b/var/www/templates/correlation/metadata_card_username.html index e1d4dbe1..c1e45cd6 100644 --- a/var/www/templates/correlation/metadata_card_username.html +++ b/var/www/templates/correlation/metadata_card_username.html @@ -31,7 +31,7 @@ - {{ dict_object["metadata_card"]["icon"]["icon"] }} + {{ dict_object["metadata_card"]["svg_icon"]["icon"] }} {{ dict_object["metadata"]["type_id"] }} diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 04c4ee0b..cdd0bc64 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -99,8 +99,15 @@
+ {% set is_correlation = True %} {% if dict_object["object_type"] == "pgp" %} {% include 'correlation/metadata_card_pgp.html' %} + {% elif dict_object["object_type"] == "chat" %} + {% include 'chats_explorer/card_chat.html' %} + {% elif dict_object["object_type"] == "chat-subchannel" %} + {% include 'chats_explorer/card_chat_subchannel.html' %} + {% elif dict_object["object_type"] == "user-account" %} + {% include 'chats_explorer/card_user_account.html' %} {% elif dict_object["object_type"] == "cryptocurrency" %} {% include 'correlation/metadata_card_cryptocurrency.html' %} {% elif dict_object["object_type"] == "username" %} @@ -121,6 +128,8 @@ {% include 'correlation/metadata_card_etag.html' %} {% elif dict_object["object_type"] == "hhhash" %} {% include 'correlation/metadata_card_hhhash.html' %} + {% elif dict_object["object_type"] == "image" %} + {% include 'chats_explorer/card_image.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% elif dict_object["object_type"] == "favicon" %} @@ -204,6 +213,22 @@
+
    +
  • Direct Correlations
    • +
  • + {% for obj_type in dict_object['nb_correl'] %} +
    +
    + {{ obj_type }} +
    +
    + {{ dict_object['nb_correl'][obj_type] }} +
    +
    + {% endfor %} +
    • +
+
  • Select Correlation
    • @@ -343,21 +368,6 @@

-
    -
  • Direct Correlations
    • -
  • - {% for obj_type in dict_object['nb_correl'] %} -
    -
    - {{ obj_type }} -
    -
    - {{ dict_object['nb_correl'][obj_type] }} -
    -
    - {% endfor %} -
    • -
@@ -714,7 +724,9 @@ if (d.popover) { } desc = desc + "
" } else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") { - desc = desc + "
" + sanitize_text(key) + "
" + sanitize_text(data[key]) + "
" + if (data[key]) { + desc = desc + "
" + sanitize_text(key) + "
" + sanitize_text(data[key]) + "
" + } } }); desc = desc + "" diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html index bff41724..9ad5cc8b 100644 --- a/var/www/templates/correlation/show_relationship.html +++ b/var/www/templates/correlation/show_relationship.html @@ -108,7 +108,7 @@ {% elif dict_object["object_type"] == "decoded" %} {% include 'correlation/metadata_card_decoded.html' %} {% elif dict_object["object_type"] == "chat" %} - {% include 'correlation/metadata_card_chat.html' %} + {% include 'chats_explorer/card_chat.html' %} {% elif dict_object["object_type"] == "cve" %} {% include 'correlation/metadata_card_cve.html' %} {% elif dict_object["object_type"] == "domain" %} diff --git a/var/www/templates/domains/card_img_domain.html b/var/www/templates/domains/card_img_domain.html index 66447a72..b9eecfd3 100644 --- a/var/www/templates/domains/card_img_domain.html +++ b/var/www/templates/domains/card_img_domain.html @@ -55,7 +55,7 @@ - {{dict_domain["first_seen"]}} + {{dict_domain["last_seen"]}} diff --git a/var/www/templates/hunter/add_retro_hunt_task.html b/var/www/templates/hunter/add_retro_hunt_task.html index d8763f61..fada045e 100644 --- a/var/www/templates/hunter/add_retro_hunt_task.html +++ b/var/www/templates/hunter/add_retro_hunt_task.html @@ -78,6 +78,10 @@
+
+ + +
{#
#} {# #} {# #} diff --git a/var/www/templates/modals/edit_tag.html b/var/www/templates/modals/edit_tag.html index 512c8277..ad1e4189 100644 --- a/var/www/templates/modals/edit_tag.html +++ b/var/www/templates/modals/edit_tag.html @@ -51,6 +51,6 @@ $('#edit_tags_modal').on('show.bs.modal', function (event) { tag_confirm.show(); modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid); } - modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid); + modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid); }) diff --git a/var/www/templates/objects/block_object_footer_small.html b/var/www/templates/objects/block_object_footer_small.html new file mode 100644 index 00000000..670c38ca --- /dev/null +++ b/var/www/templates/objects/block_object_footer_small.html @@ -0,0 +1,12 @@ +
+{#
#} +{# #} +{# #} +{# #} +{#
#} +
+ + + +
+