chg: [chats] add messages threads

dev
terrtia 2023-11-29 16:28:25 +01:00
parent f766cbebda
commit 93ef541862
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
13 changed files with 329 additions and 44 deletions

View File

@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
from importer.feeders.Default import DefaultFeeder
from lib.objects.Chats import Chat
from lib.objects import ChatSubChannels
from lib.objects import ChatThreads
from lib.objects import Images
from lib.objects import Messages
from lib.objects import FilesNames
@ -74,13 +75,13 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta']['chat']['id']
def get_subchannel_id(self):
pass
return self.json_data['meta']['chat'].get('subchannel', {}).get('id')
def get_subchannels(self):
pass
def get_thread_id(self):
pass
return self.json_data['meta'].get('thread', {}).get('id')
def get_message_id(self):
return self.json_data['meta']['id']
@ -112,7 +113,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta'].get('reply_to') # TODO change to reply ???
def get_message_reply_id(self):
return self.json_data['meta'].get('reply_to', None)
return self.json_data['meta'].get('reply_to', {}).get('message_id')
def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data'])
@ -125,6 +126,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
#### Create Object ID ####
chat_id = self.get_chat_id()
message_id = self.get_message_id()
thread_id = self.get_thread_id()
# channel id
# thread id
@ -135,11 +137,11 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
self.obj = Images.Image(self.json_data['data-sha256'])
else:
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp)
obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, thread_id=thread_id)
self.obj = Messages.Message(obj_id)
return self.obj
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
@ -170,7 +172,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
chat.add_children(obj_global_id=subchannel.get_global_id())
else:
if obj.type == 'message':
chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
if self.get_thread_id():
self.process_thread(obj, chat, date, timestamp, reply_id=reply_id)
else:
chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
# if meta.get('subchannels'): # TODO Update icon + names
@ -198,9 +203,32 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
subchannel.set_info(meta['info'])
if obj.type == 'message':
subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
if self.get_thread_id():
self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id)
else:
subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
return subchannel
def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None):
meta = self.json_data['meta']['thread']
thread_id = self.get_thread_id()
p_chat_id = meta['parent'].get('chat')
p_subchannel_id = meta['parent'].get('subchannel')
p_message_id = meta['parent'].get('message')
if p_chat_id == self.get_chat_id() and p_subchannel_id == self.get_subchannel_id():
thread = ChatThreads.create(thread_id, self.get_chat_instance_uuid(), p_chat_id, p_subchannel_id, p_message_id, obj_chat)
thread.add(date, obj)
thread.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
# TODO OTHERS CORRELATIONS TO ADD
return thread
# TODO
# else:
# # ADD NEW MESSAGE REF (used by discord)
def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta']['sender']
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
@ -298,6 +326,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if media_name:
FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
for reaction in self.get_reactions():
message.add_reaction(reaction['reaction'], int(reaction['count']))
for obj in objs: # TODO PERF avoid parsing metas multiple times
# CHAT

View File

@ -16,8 +16,8 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None
AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon',
'file-name', 'hhhash',
AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded',
'domain', 'etag', 'favicon', 'file-name', 'hhhash',
'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
def get_ail_uuid():

View File

@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ConfigLoader import ConfigLoader
from lib.objects import Chats
from lib.objects import ChatSubChannels
from lib.objects import ChatThreads
from lib.objects import Messages
from lib.objects import Usernames
@ -324,7 +325,7 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
def api_get_subchannel(chat_id, chat_instance_uuid):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
return {"status": "error", "reason": "Unknown subchannel"}, 404
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
if meta['chat']:
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
@ -333,6 +334,16 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
meta['messages'], meta['tags_messages'] = subchannel.get_messages()
return meta, 200
def api_get_thread(thread_id, thread_instance_uuid):
thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid)
if not thread.exists():
return {"status": "error", "reason": "Unknown thread"}, 404
meta = thread.get_meta({'chat', 'nb_messages'})
# if meta['chat']:
# meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
meta['messages'], meta['tags_messages'] = thread.get_messages()
return meta, 200
def api_get_message(message_id):
message = Messages.Message(message_id)
if not message.exists():

View File

@ -41,7 +41,9 @@ config_loader = None
##################################
CORRELATION_TYPES_BY_OBJ = {
"chat": ["image", "user-account"], # message or direct correlation like cve, bitcoin, ... ???
"chat": ["chat-subchannel", "chat-thread", "image", "user-account"], # message or direct correlation like cve, bitcoin, ... ???
"chat-subchannel": ["chat", "chat-thread", "image", "message", "user-account"],
"chat-thread": ["chat", "chat-subchannel", "image", "message", "user-account"], # TODO user account
"cookie-name": ["domain"],
"cryptocurrency": ["domain", "item", "message"],
"cve": ["domain", "item", "message"],
@ -53,11 +55,11 @@ CORRELATION_TYPES_BY_OBJ = {
"hhhash": ["domain"],
"image": ["chat", "message", "user-account"],
"item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ???
"message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
"message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"], # chat ??
"pgp": ["domain", "item", "message"],
"screenshot": ["domain", "item"],
"title": ["domain", "item"],
"user-account": ["chat", "message"],
"user-account": ["chat", "chat-subchannel", "chat-thread", "message"],
"username": ["domain", "item", "message"], # TODO chat-user/account
}

View File

@ -149,7 +149,7 @@ class ChatSubChannel(AbstractChatObject):
class ChatSubChannels(AbstractChatObjects):
def __init__(self):
super().__init__('chat-subchannels')
super().__init__('chat-subchannel')
# if __name__ == '__main__':
# chat = Chat('test', 'telegram')

View File

@ -15,12 +15,8 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib import ail_core
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from lib.data_retention_engine import update_obj_date
from lib.objects import ail_objects
from lib.timeline_engine import Timeline
from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
from lib.correlations_engine import get_correlation_by_correl_type
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@ -33,13 +29,13 @@ config_loader = None
################################################################################
################################################################################
class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
class ChatThread(AbstractChatObject):
"""
AIL Chat Object. (strings)
"""
def __init__(self, id, subtype):
super(Chat, self).__init__('chat-thread', id, subtype)
super().__init__('chat-thread', id, subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
@ -69,7 +65,7 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
# style = 'fas'
# icon = '\uf007'
style = 'fas'
icon = '\uf086'
icon = '\uf7a4'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_meta(self, options=set()):
@ -77,27 +73,42 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ?????
meta['id'] = self.id
meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True)
if 'username':
meta['username'] = self.get_username()
if 'nb_messages':
meta['nb_messages'] = self.get_nb_messages()
# created_at ???
return meta
def get_misp_object(self):
return
############################################################################
############################################################################
def create(self, container_obj, message_id):
if message_id:
parent_message = container_obj.get_obj_by_message_id(message_id)
if parent_message: # TODO EXCEPTION IF DON'T EXISTS
self.set_parent(obj_global_id=parent_message)
_, _, parent_id = parent_message.split(':', 2)
self.add_correlation('message', '', parent_id)
else:
self.set_parent(obj_global_id=container_obj.get_global_id())
self.add_correlation(container_obj.get_type(), container_obj.get_subtype(r_str=True), container_obj.get_id())
# others optional metas, ... -> # TODO ALL meta in hset
def create(thread_id, chat_instance, chat_id, subchannel_id, message_id, container_obj):
if container_obj.get_type() == 'chat':
new_thread_id = f'{chat_id}/{thread_id}'
# sub-channel
else:
new_thread_id = f'{chat_id}/{subchannel_id}/{thread_id}'
#### Messages #### TODO set parents
thread = ChatThread(new_thread_id, chat_instance)
if not thread.exists():
thread.create(container_obj, message_id)
return thread
# def get_last_message_id(self):
#
# return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
class ChatThreads(AbstractChatObjects):
def __init__(self):
super().__init__('chat-thread')
if __name__ == '__main__':
chat = Chat('test', 'telegram')
r = chat.get_messages()
print(r)
# if __name__ == '__main__':
# chat = Chat('test', 'telegram')
# r = chat.get_messages()
# print(r)

View File

@ -100,6 +100,13 @@ class Message(AbstractObject):
chat_id = self.get_basename().rsplit('_', 1)[0]
return chat_id
def get_thread(self):
for child in self.get_childrens():
obj_type, obj_subtype, obj_id = child.split(':', 2)
if obj_type == 'chat-thread':
nb_messages = r_object.zcard(f'messages:{obj_type}:{obj_subtype}:{obj_id}')
return {'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'nb': nb_messages}
# TODO get Instance ID
# TODO get channel ID
# TODO get thread ID
@ -245,6 +252,10 @@ class Message(AbstractObject):
meta['user-account'] = {'id': 'UNKNOWN'}
if 'chat' in options:
meta['chat'] = self.get_chat_id()
if 'thread' in options:
thread = self.get_thread()
if thread:
meta['thread'] = thread
if 'images' in options:
meta['images'] = self.get_images()
if 'files-names' in options:
@ -318,10 +329,10 @@ class Message(AbstractObject):
def delete(self):
pass
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None):
def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): # TODO CHECK COLLISIONS
timestamp = int(timestamp)
if channel_id and thread_id:
return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ?????
return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}'
elif channel_id:
return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}'
elif thread_id:
@ -329,6 +340,10 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
else:
return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}'
# thread id of message
# thread id of chat
# thread id of subchannel
# TODO Check if already exists
# def create(source, chat_id, message_id, timestamp, content, tags=[]):
def create(obj_id, content, translation=None, tags=[]):

View File

@ -181,7 +181,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def get_message_meta(self, message, timestamp=None): # TODO handle file message
message = Messages.Message(message[9:])
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp)
meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'}, timestamp=timestamp)
return meta
def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page
@ -189,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
tags = {}
messages = {}
curr_date = None
for message in self._get_messages(nb=50, page=1):
for message in self._get_messages(nb=2000, page=1):
timestamp = message[1]
date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
if date_day != curr_date:

View File

@ -180,9 +180,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
self.add_correlation('domain', '', domain)
# TODO:ADD objects + Stats
def create(self, first_seen, last_seen):
self.set_first_seen(first_seen)
self.set_last_seen(last_seen)
# def create(self, first_seen, last_seen):
# self.set_first_seen(first_seen)
# self.set_last_seen(last_seen)
def _delete(self):
pass

View File

@ -14,6 +14,8 @@ from lib import btc_ail
from lib import Tag
from lib.objects import Chats
from lib.objects import ChatSubChannels
from lib.objects import ChatThreads
from lib.objects import CryptoCurrencies
from lib.objects import CookiesNames
from lib.objects.Cves import Cve
@ -62,6 +64,10 @@ def get_object(obj_type, subtype, obj_id):
return Decoded(obj_id)
elif obj_type == 'chat':
return Chats.Chat(obj_id, subtype)
elif obj_type == 'chat-subchannel':
return ChatSubChannels.ChatSubChannel(obj_id, subtype)
elif obj_type == 'chat-thread':
return ChatThreads.ChatThread(obj_id, subtype)
elif obj_type == 'cookie-name':
return CookiesNames.CookieName(obj_id)
elif obj_type == 'cve':

View File

@ -112,6 +112,18 @@ def objects_subchannel_messages():
subchannel = subchannel[0]
return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label)
@chats_explorer.route("/chats/explorer/thread", methods=['GET'])
@login_required
@login_read_only
def objects_thread_messages():
thread_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
thread = chats_viewer.api_get_thread(thread_id, instance_uuid)
if thread[1] != 200:
return create_json_response(thread[0], thread[1])
else:
meta = thread[0]
return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label)
@chats_explorer.route("/objects/message", methods=['GET'])
@login_required

View File

@ -0,0 +1,191 @@
<!DOCTYPE html>
<html>
<head>
<title>Thread Messages - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
{# <link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
{# <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
<style>
.chat-message-left,
.chat-message-right {
display: flex;
flex-shrink: 0;
}
.chat-message-right {
flex-direction: row-reverse;
margin-left: auto
}
.divider:after,
.divider:before {
content: "";
flex: 1;
height: 2px;
background: #eee;
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
{{ meta["id"] }}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>ID</th>
<th>Parent</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb Messages</th>
</tr>
</thead>
<tbody>
<tr>
<td>
{{ meta['id'] }}
</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['nb_messages'] }}</td>
</tr>
</tbody>
</table>
</div>
</div>
</li>
{# <li class="list-group-item py-0">#}
{# <br>#}
{# <div class="mb-3">#}
{# Tags:#}
{# {% for tag in meta['tags'] %}#}
{# <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"#}
{# data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">#}
{# {{ tag }}#}
{# </button>#}
{# {% endfor %}#}
{# <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">#}
{# <i class="far fa-plus-square"></i>#}
{# </button>#}
{# </div>#}
{# </li>#}
</ul>
{# {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}#}
{# {% include 'modals/investigations_register_obj.html' %}#}
{# {% endwith %}#}
{# <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
{# <i class="fas fa-microscope"></i> Investigations#}
{# </button>#}
</div>
</div>
{% for tag in meta['tags_messages'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ meta['tags_messages'][tag] }}</span></span>
{% endfor %}
<div>
<div class="list-group d-inline-block">
{% for date in messages %}
<a class="list-group-item list-group-item-action" href="#date_section_{{ date }}">{{ date }}</a>
{% endfor %}
</div>
</div>
<span class="mt-3">
{% include 'objects/image/block_blur_img_slider.html' %}
</span>
<div class="position-relative">
<div class="chat-messages p-2">
{% for date in meta['messages'] %}
<div class="divider d-flex align-items-center mb-4">
<p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
<span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
</p>
</div>
{% for mess in meta['messages'][date] %}
{% with message=mess %}
{% include 'chats_explorer/block_message.html' %}
{% endwith %}
{% endfor %}
<br>
{% endfor %}
</div>
</div>
</div>
</div>
</div>
<script>
var chart = {};
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

View File

@ -74,6 +74,12 @@
{% for reaction in message['reactions'] %}
<span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
{% endfor %}
{% if message['thread'] %}
<hr class="mb-1">
<div class="my-2 text-center">
<a href="{{ url_for('chats_explorer.objects_thread_messages')}}?uuid={{ message['thread']['subtype'] }}&id={{ message['thread']['id'] }}"><i class="far fa-comment"></i> {{ message['thread']['nb'] }} Messages</a>
</div>
{% endif %}
{% for tag in message['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}