From f766cbebda7dc65bddc102eed8c4fb578570d65b Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 27 Nov 2023 16:25:09 +0100 Subject: [PATCH] chg: [chats] add message file-name object + str emoticon reactions --- bin/importer/feeders/abstract_chats_feeder.py | 27 +++++ bin/lib/ail_core.py | 3 +- bin/lib/correlations_engine.py | 3 +- bin/lib/objects/FilesNames.py | 101 ++++++++++++++++++ bin/lib/objects/Messages.py | 25 ++++- bin/lib/objects/UsersAccount.py | 6 ++ bin/lib/objects/abstract_chat_object.py | 4 +- bin/lib/objects/ail_objects.py | 3 + .../chats_explorer/block_message.html | 10 ++ 9 files changed, 175 insertions(+), 7 deletions(-) create mode 100755 bin/lib/objects/FilesNames.py diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 11d2fe89..e82c5e2f 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -22,6 +22,8 @@ from lib.objects.Chats import Chat from lib.objects import ChatSubChannels from lib.objects import Images from lib.objects import Messages +from lib.objects import FilesNames +# from lib.objects import Files from lib.objects import UsersAccount from lib.objects.Usernames import Username from lib import chats_viewer @@ -83,6 +85,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC): def get_message_id(self): return self.json_data['meta']['id'] + def get_media_name(self): + return self.json_data['meta'].get('media', {}).get('name') + + def get_reactions(self): + return self.json_data['meta'].get('reactions', []) + def get_message_timestamp(self): return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP # if self.json_data['meta'].get('date'): @@ -223,6 +231,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC): user_account.set_icon(img.get_global_id()) new_objs.add(img) + if meta.get('info'): + user_account.set_info(meta['info']) + return user_account # Create abstract class: -> new API endpoint ??? => force field, check if already imported ? @@ -251,11 +262,22 @@ class AbstractChatFeeder(DefaultFeeder, ABC): print(self.obj.type) + # TODO FILES + FILES REF + # get object by meta object type if self.obj.type == 'message': # Content obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation + # FILENAME + media_name = self.get_media_name() + if media_name: + print(media_name) + FilesNames.FilesNames().create(media_name, date, obj) + + for reaction in self.get_reactions(): + obj.add_reaction(reaction['reaction'], int(reaction['count'])) + else: chat_id = self.get_chat_id() message_id = self.get_message_id() @@ -271,6 +293,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC): obj.add(date, message) obj.set_parent(obj_global_id=message.get_global_id()) + # FILENAME + media_name = self.get_media_name() + if media_name: + FilesNames.FilesNames().create(media_name, date, message, file_obj=obj) + for obj in objs: # TODO PERF avoid parsing metas multiple times # CHAT diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 77dd0682..1fd8d97a 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -16,7 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None -AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', +AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', + 'file-name', 'hhhash', 'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index b0b3108d..fbf5b75e 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -49,10 +49,11 @@ CORRELATION_TYPES_BY_OBJ = { "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded + "file-name": ["chat", "message"], "hhhash": ["domain"], "image": ["chat", "message", "user-account"], "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? - "message": ["cve", "cryptocurrency", "decoded", "image", "pgp", "user-account"], # chat ?? + "message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ?? "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], diff --git a/bin/lib/objects/FilesNames.py b/bin/lib/objects/FilesNames.py new file mode 100755 index 00000000..42c9335d --- /dev/null +++ b/bin/lib/objects/FilesNames.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from flask import url_for +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects + +config_loader = ConfigLoader() +r_object = config_loader.get_db_conn("Kvrocks_Objects") +config_loader = None + + +class FileName(AbstractDaterangeObject): + """ + AIL FileName Object. (strings) + """ + + # ID = SHA256 + def __init__(self, name): + super().__init__('file-name', name) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + def get_svg_icon(self): + return {'style': 'far', 'icon': '\uf249', 'color': '#36F5D5', 'radius': 5} + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('file') + + # obj_attrs.append(obj.add_attribute('sha256', value=self.id)) + # obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content())) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['tags'] = self.get_tags(r_list=True) + if 'tags_safe' in options: + meta['tags_safe'] = self.is_tags_safe(meta['tags']) + return meta + + def create(self): # create ALL SET ?????? + pass + + def add_reference(self, date, src_ail_object, file_obj=None): + self.add(date, src_ail_object) + if file_obj: + self.add_correlation(file_obj.type, file_obj.get_subtype(r_str=True), file_obj.get_id()) + +# TODO USE ZSET FOR ALL OBJS IDS ?????? + +class FilesNames(AbstractDaterangeObjects): + """ + CookieName Objects + """ + def __init__(self): + super().__init__('file-name', FileName) + + def sanitize_id_to_search(self, name_to_search): + return name_to_search + + # TODO sanitize file name + def create(self, name, date, src_ail_object, file_obj=None, limit=500, force=False): + if 0 < len(name) <= limit or force or limit < 0: + file_name = self.obj_class(name) + # if not file_name.exists(): + # file_name.create() + file_name.add_reference(date, src_ail_object, file_obj=file_obj) + return file_name + +# if __name__ == '__main__': +# name_to_search = '29ba' +# print(search_screenshots_by_name(name_to_search)) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 59d2e125..671e2670 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -118,9 +118,24 @@ class Message(AbstractObject): user_account = f'user-account:{user_account["user-account"].pop()}' if meta: _, user_account_subtype, user_account_id = user_account.split(':', 3) - user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'}) + user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'}) return user_account + def get_files_names(self): + names = [] + filenames = self.get_correlation('file-name').get('file-name') + if filenames: + for name in filenames: + names.append(name[1:]) + return names + + def get_reactions(self): + return r_object.hgetall(f'meta:reactions:{self.type}::{self.id}') + + # TODO sanitize reactions + def add_reaction(self, reactions, nb_reaction): + r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction) + # Update value on import # reply to -> parent ? # reply/comment - > children ? @@ -232,6 +247,10 @@ class Message(AbstractObject): meta['chat'] = self.get_chat_id() if 'images' in options: meta['images'] = self.get_images() + if 'files-names' in options: + meta['files-names'] = self.get_files_names() + if 'reactions' in options: + meta['reactions'] = self.get_reactions() # meta['encoding'] = None return meta @@ -314,8 +333,8 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None # def create(source, chat_id, message_id, timestamp, content, tags=[]): def create(obj_id, content, translation=None, tags=[]): message = Message(obj_id) - if not message.exists(): - message.create(content, translation=translation, tags=tags) + # if not message.exists(): + message.create(content, translation=translation, tags=tags) return message diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 3acc83bc..ced2d682 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -91,6 +91,12 @@ class UserAccount(AbstractSubtypeObject): def set_icon(self, icon): self._set_field('icon', icon) + def get_info(self): + return self._get_field('info') + + def set_info(self, info): + return self._set_field('info', info) + def _get_timeline_username(self): return Timeline(self.get_global_id(), 'username') diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 6094fb60..e58efe99 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_message_meta(self, message, timestamp=None): # TODO handle file message message = Messages.Message(message[9:]) - meta = message.get_meta(options={'content', 'images', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp) + meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp) return meta def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page @@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): tags = {} messages = {} curr_date = None - for message in self._get_messages(nb=30, page=1): + for message in self._get_messages(nb=50, page=1): timestamp = message[1] date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index d3bd06f3..2a9493e9 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -21,6 +21,7 @@ from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decod from lib.objects.Domains import Domain from lib.objects import Etags from lib.objects.Favicons import Favicon +from lib.objects import FilesNames from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Images @@ -69,6 +70,8 @@ def get_object(obj_type, subtype, obj_id): return Etags.Etag(obj_id) elif obj_type == 'favicon': return Favicon(obj_id) + elif obj_type == 'file-name': + return FilesNames.FileName(obj_id) elif obj_type == 'hhhash': return HHHashs.HHHash(obj_id) elif obj_type == 'image': diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 53f63f61..b06689b9 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -63,7 +63,17 @@ {% endfor %} {% endif %} + {% if message['files-names'] %} + {% for file_name in message['files-names'] %} +
+ {{ file_name }} +
+ {% endfor %} + {% endif %} 
{{ message['content'] }}
+ {% for reaction in message['reactions'] %} + {{ reaction }} {{ message['reactions'][reaction] }} + {% endfor %} {% for tag in message['tags'] %} {{ tag }} {% endfor %}