From 941838ab76a57d3df5a0583658f85c5b432b0463 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 4 Dec 2023 10:26:02 +0100 Subject: [PATCH] chg: [chats] add discord threads, Forum channel --- bin/importer/feeders/abstract_chats_feeder.py | 11 +++-- bin/lib/chats_viewer.py | 12 ++++- bin/lib/objects/ChatSubChannels.py | 7 ++- bin/lib/objects/ChatThreads.py | 2 + bin/lib/objects/Chats.py | 3 ++ bin/lib/objects/abstract_chat_object.py | 7 +-- .../chats_explorer/SubChannelMessages.html | 45 ++++++++++++++++++- .../chats_explorer/ThreadMessages.html | 4 +- 8 files changed, 78 insertions(+), 13 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index c21e0c78..aebf76ef 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -182,8 +182,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return chat - # def process_subchannels(self): - # pass def process_subchannel(self, obj, date, timestamp, reply_id=None): # TODO CREATE DATE meta = self.json_data['meta']['chat']['subchannel'] @@ -216,12 +214,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC): p_subchannel_id = meta['parent'].get('subchannel') p_message_id = meta['parent'].get('message') + # print(thread_id, p_chat_id, p_subchannel_id, p_message_id) + if p_chat_id == self.get_chat_id() and p_subchannel_id == self.get_subchannel_id(): thread = ChatThreads.create(thread_id, self.get_chat_instance_uuid(), p_chat_id, p_subchannel_id, p_message_id, obj_chat) thread.add(date, obj) thread.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) # TODO OTHERS CORRELATIONS TO ADD + if meta.get('name'): + thread.set_name(meta['name']) + return thread # TODO @@ -308,8 +311,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): else: chat_id = self.get_chat_id() + thread_id = self.get_thread_id() + channel_id = self.get_subchannel_id() message_id = self.get_message_id() - message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) + message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, channel_id=channel_id, thread_id=thread_id) message = Messages.Message(message_id) # create empty message if message don't exists if not message.exists(): diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 4ddd7065..2740e6ca 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -290,6 +290,12 @@ def get_chat_meta_from_global_id(chat_global_id): chat = Chats.Chat(chat_id, instance_uuid) return chat.get_meta() +def get_threads_metas(threads): + metas = [] + for thread in threads: + metas.append(ChatThreads.ChatThread(thread['id'], thread['subtype']).get_meta(options={'name', 'nb_messages'})) + return metas + def get_username_meta_from_global_id(username_global_id): _, instance_uuid, username_id = username_global_id.split(':', 2) username = Usernames.Username(username_id, instance_uuid) @@ -305,7 +311,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -326,9 +332,11 @@ def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'threads'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) + if meta.get('threads'): + meta['threads'] = get_threads_metas(meta['threads']) if meta.get('username'): meta['username'] = get_username_meta_from_global_id(meta['username']) meta['messages'], meta['tags_messages'] = subchannel.get_messages() diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index f73488a4..38a50a20 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -84,10 +84,13 @@ class ChatSubChannel(AbstractChatObject): meta['chat'] = self.get_chat() if 'img' in options: meta['img'] = self.get_img() - if 'nb_messages': + if 'nb_messages' in options: meta['nb_messages'] = self.get_nb_messages() - if 'created_at': + if 'created_at' in options: meta['created_at'] = self.get_created_at(date=True) + if 'threads' in options: + meta['threads'] = self.get_threads() + print(meta['threads']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index 0790d4ae..9514a248 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -73,6 +73,8 @@ class ChatThread(AbstractChatObject): meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) + if 'name': + meta['name'] = self.get_name() if 'nb_messages': meta['nb_messages'] = self.get_nb_messages() # created_at ??? diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 22a299c5..b631b2d2 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -86,6 +86,9 @@ class Chat(AbstractChatObject): meta['nb_subchannels'] = self.get_nb_subchannels() if 'created_at': meta['created_at'] = self.get_created_at(date=True) + if 'threads' in options: + meta['threads'] = self.get_threads() + print(meta['threads']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index ee6387d3..be25eecb 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -88,10 +88,10 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_threads(self): threads = [] - for obj_global_id in self.get_childrens(): - obj_type, _ = obj_global_id.split(':', 1) + for child in self.get_childrens(): + obj_type, obj_subtype, obj_id = child.split(':', 2) if obj_type == 'chat-thread': - threads.append(obj_global_id) + threads.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id}) return threads def get_created_at(self, date=False): @@ -242,6 +242,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): for mess_id in self.get_cached_message_reply(message_id): self.add_obj_children(obj_global_id, mess_id) + # def get_deleted_messages(self, message_id): # get_messages_meta ???? diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 922b52da..3b807b96 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -132,6 +132,43 @@ + + {% if subchannel['threads'] %} + + + + + + + + + + + + + {% for thread in subchannel['threads'] %} + + + + + + + + {% endfor %} + +
NameCreated atFirst seenLast seenNb Messages
+ {{ thread['name'] }} {{ thread['nb_messages'] }} Messages + {{ thread["created_at"] }} + {% if thread['first_seen'] %} + {{ thread['first_seen'][0:4] }}-{{ thread['first_seen'][4:6] }}-{{ thread['first_seen'][6:8] }} + {% endif %} + + {% if thread['last_seen'] %} + {{ thread['last_seen'][0:4] }}-{{ thread['last_seen'][4:6] }}-{{ thread['last_seen'][6:8] }} + {% endif %} + {{ thread['nb_messages'] }}
+ {% endif %} + {% for tag in subchannel['tags_messages'] %} {{ tag }} {{ subchannel['tags_messages'][tag] }} {% endfor %} @@ -182,7 +219,13 @@ $(document).ready(function(){ $("#page-Decoded").addClass("active"); $("#nav_chat").addClass("active"); - + {% if subchannel['threads'] %} + $('#tablethreads').DataTable({ + "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]], + "iDisplayLength": 10, + "order": [[ 3, "desc" ]] + }); + {% endif %} }); function toggle_sidebar(){ diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index 921861c9..1e8e588e 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -62,7 +62,7 @@ - + @@ -72,7 +72,7 @@
IDName Parent First seen Last seen
- {{ meta['id'] }} + {{ meta['name'] }} {% if meta['first_seen'] %}