CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [chats] add chats participants + user-accounts basic template

dev
terrtia 2023-12-06 16:26:26 +01:00
parent bef4e69a68
commit 38ce17bc8a
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
17 changed files with 558 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
bin/importer/feeders/abstract_chats_feeder.py
View File

@ -11,7 +11,7 @@ import datetime
import os import os
import sys import sys
from abc import abstractmethod, ABC from abc import ABC
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
@ -144,6 +144,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
meta = self.json_data['meta']['chat'] # todo replace me by function meta = self.json_data['meta']['chat'] # todo replace me by function
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
subchannel = None
thread = None
# date stat + correlation # date stat + correlation
chat.add(date, obj) chat.add(date, obj)
@ -168,24 +170,26 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
chat.update_username_timeline(username.get_global_id(), timestamp) chat.update_username_timeline(username.get_global_id(), timestamp)
if meta.get('subchannel'): if meta.get('subchannel'):
subchannel = self.process_subchannel(obj, date, timestamp, reply_id=reply_id) subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id)
chat.add_children(obj_global_id=subchannel.get_global_id()) chat.add_children(obj_global_id=subchannel.get_global_id())
else: else:
if obj.type == 'message': if obj.type == 'message':
if self.get_thread_id(): if self.get_thread_id():
self.process_thread(obj, chat, date, timestamp, reply_id=reply_id) thread = self.process_thread(obj, chat, date, timestamp, reply_id=reply_id)
else: else:
chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
chats_obj = [chat]
# if meta.get('subchannels'): # TODO Update icon + names if subchannel:
chats_obj.append(subchannel)
return chat if thread:
chats_obj.append(thread)
return chats_obj
def process_subchannel(self, obj, date, timestamp, reply_id=None): # TODO CREATE DATE def process_subchannel(self, obj, date, timestamp, reply_id=None): # TODO CREATE DATE
meta = self.json_data['meta']['chat']['subchannel'] meta = self.json_data['meta']['chat']['subchannel']
subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid()) subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid())
thread = None
# TODO correlation with obj = message/image # TODO correlation with obj = message/image
subchannel.add(date) subchannel.add(date)
@ -202,10 +206,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
if obj.type == 'message': if obj.type == 'message':
if self.get_thread_id(): if self.get_thread_id():
self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id) thread = self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id)
else: else:
subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id)
return subchannel return subchannel, thread
def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None): def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None):
meta = self.json_data['meta']['thread'] meta = self.json_data['meta']['thread']
@ -231,7 +235,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# else: # else:
# # ADD NEW MESSAGE REF (used by discord) # # ADD NEW MESSAGE REF (used by discord)
def process_sender(self, new_objs, obj, date, timestamp): def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta']['sender'] meta = self.json_data['meta']['sender']
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
@ -267,12 +270,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return user_account return user_account
# Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
# 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default
# 2) Create/Get CHAT ID - Done
# 3) Create/Get Channel IF is in channel
# 4) Create/Get Thread IF is in thread
# 5) Create/Update Username and User-account - Done
def process_meta(self): # TODO CHECK MANDATORY FIELDS def process_meta(self): # TODO CHECK MANDATORY FIELDS
""" """
Process JSON meta filed. Process JSON meta filed.
@ -316,7 +313,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
message_id = self.get_message_id() message_id = self.get_message_id()
message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, channel_id=channel_id, thread_id=thread_id) message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, channel_id=channel_id, thread_id=thread_id)
message = Messages.Message(message_id) message = Messages.Message(message_id)
# create empty message if message don't exists # create empty message if message don't exist
if not message.exists(): if not message.exists():
message.create('') message.create('')
objs.add(message) objs.add(message)
@ -336,46 +333,26 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
for obj in objs: # TODO PERF avoid parsing metas multiple times for obj in objs: # TODO PERF avoid parsing metas multiple times
# TODO get created subchannel + thread
# => create correlation user-account with object
# CHAT # CHAT
chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
# SENDER # TODO HANDLE NULL SENDER # SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(new_objs, obj, date, timestamp) user_account = self.process_sender(new_objs, obj, date, timestamp)
# UserAccount---Chat # UserAccount---ChatObjects
user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) for obj_chat in chat_objs:
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
# if chat: # TODO Chat---Username correlation ??? # if chat: # TODO Chat---Username correlation ???
# # Chat---Username => need to handle members and participants # # Chat---Username => need to handle members and participants
# chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
# TODO Sender image -> correlation # TODO Sender image -> correlation
# image # image
# -> subchannel ? # -> subchannel ?
# -> thread id ? # -> thread id ?
return new_objs | objs return new_objs | objs

24
bin/lib/ail_core.py
View File

@ -40,7 +40,7 @@ def get_all_objects():
return AIL_OBJECTS return AIL_OBJECTS
def get_objects_with_subtypes(): def get_objects_with_subtypes():
return ['chat', 'cryptocurrency', 'pgp', 'username'] return ['chat', 'cryptocurrency', 'pgp', 'username', 'user-account']
def get_object_all_subtypes(obj_type): # TODO Dynamic subtype def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
if obj_type == 'chat': if obj_type == 'chat':
@ -53,6 +53,8 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
return ['key', 'mail', 'name'] return ['key', 'mail', 'name']
if obj_type == 'username': if obj_type == 'username':
return ['telegram', 'twitter', 'jabber'] return ['telegram', 'twitter', 'jabber']
if obj_type == 'user-account':
return r_object.smembers(f'all_chat:subtypes')
return [] return []
def get_objects_tracked(): def get_objects_tracked():
@ -75,10 +77,28 @@ def get_all_objects_with_subtypes_tuple():
def unpack_obj_global_id(global_id, r_type='tuple'): def unpack_obj_global_id(global_id, r_type='tuple'):
if r_type == 'dict': if r_type == 'dict':
obj = global_id.split(':', 2) obj = global_id.split(':', 2)
return {'type': obj[0], 'subtype': obj[1], 'id': obj['2']} return {'type': obj[0], 'subtype': obj[1], 'id': obj[2]}
else: # tuple(type, subtype, id) else: # tuple(type, subtype, id)
return global_id.split(':', 2) return global_id.split(':', 2)
def unpack_objs_global_id(objs_global_id, r_type='tuple'):
objs = []
for global_id in objs_global_id:
objs.append(unpack_obj_global_id(global_id, r_type=r_type))
return objs
def unpack_correl_obj__id(obj_type, global_id, r_type='tuple'):
obj = global_id.split(':', 1)
if r_type == 'dict':
return {'type': obj_type, 'subtype': obj[0], 'id': obj[1]}
else: # tuple(type, subtype, id)
return obj_type, obj[0], obj[1]
def unpack_correl_objs_id(obj_type, correl_objs_id, r_type='tuple'):
objs = []
for correl_obj_id in correl_objs_id:
objs.append(unpack_correl_obj__id(obj_type, correl_obj_id, r_type=r_type))
return objs
##-- AIL OBJECTS --## ##-- AIL OBJECTS --##

47
bin/lib/chats_viewer.py
View File

@ -278,6 +278,27 @@ def create_chat_service_instance(protocol, network=None, address=None):
####################################################################################### #######################################################################################
def get_obj_chat(chat_type, chat_subtype, chat_id):
print(chat_type, chat_subtype, chat_id)
if chat_type == 'chat':
return Chats.Chat(chat_id, chat_subtype)
elif chat_type == 'chat-subchannel':
return ChatSubChannels.ChatSubChannel(chat_id, chat_subtype)
elif chat_type == 'chat-thread':
return ChatThreads.ChatThread(chat_id, chat_subtype)
def get_obj_chat_meta(obj_chat, new_options=set()):
options = {}
if obj_chat.type == 'chat':
options = {'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'}
elif obj_chat.type == 'chat-subchannel':
options = {'chat', 'created_at', 'icon', 'nb_messages', 'threads'}
elif obj_chat.type == 'chat-thread':
options = {'chat', 'nb_messages'}
for option in new_options:
options.add(option)
return obj_chat.get_meta(options=options)
def get_subchannels_meta_from_global_id(subchannels): def get_subchannels_meta_from_global_id(subchannels):
meta = [] meta = []
for sub in subchannels: for sub in subchannels:
@ -302,6 +323,8 @@ def get_username_meta_from_global_id(username_global_id):
username = Usernames.Username(username_id, instance_uuid) username = Usernames.Username(username_id, instance_uuid)
return username.get_meta() return username.get_meta()
#### API ####
def api_get_chat_service_instance(chat_instance_uuid): def api_get_chat_service_instance(chat_instance_uuid):
chat_instance = ChatServiceInstance(chat_instance_uuid) chat_instance = ChatServiceInstance(chat_instance_uuid)
if not chat_instance.exists(): if not chat_instance.exists():
@ -312,7 +335,7 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None):
chat = Chats.Chat(chat_id, chat_instance_uuid) chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists(): if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404 return {"status": "error", "reason": "Unknown chat"}, 404
meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'}) meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'})
if meta['username']: if meta['username']:
meta['username'] = get_username_meta_from_global_id(meta['username']) meta['username'] = get_username_meta_from_global_id(meta['username'])
if meta['subchannels']: if meta['subchannels']:
@ -329,11 +352,26 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
# week = chat.get_nb_message_by_week('20231109') # week = chat.get_nb_message_by_week('20231109')
return week, 200 return week, 200
def api_get_chat_participants(chat_type, chat_subtype, chat_id):
if chat_type not in ['chat', 'chat-subchannel', 'chat-thread']:
return {"status": "error", "reason": "Unknown chat type"}, 400
chat_obj = get_obj_chat(chat_type, chat_subtype, chat_id)
if not chat_obj.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
else:
meta = get_obj_chat_meta(chat_obj, new_options={'participants'})
chat_participants = []
for participant in meta['participants']:
user_account = UsersAccount.UserAccount(participant['id'], participant['subtype'])
chat_participants.append(user_account.get_meta({'icon', 'info', 'username'}))
meta['participants'] = chat_participants
return meta, 200
def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None): def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists(): if not subchannel.exists():
return {"status": "error", "reason": "Unknown subchannel"}, 404 return {"status": "error", "reason": "Unknown subchannel"}, 404
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'threads'}) meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'})
if meta['chat']: if meta['chat']:
meta['chat'] = get_chat_meta_from_global_id(meta['chat']) meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
if meta.get('threads'): if meta.get('threads'):
@ -347,7 +385,7 @@ def api_get_thread(thread_id, thread_instance_uuid, translation_target=None):
thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid) thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid)
if not thread.exists(): if not thread.exists():
return {"status": "error", "reason": "Unknown thread"}, 404 return {"status": "error", "reason": "Unknown thread"}, 404
meta = thread.get_meta({'chat', 'nb_messages'}) meta = thread.get_meta({'chat', 'nb_messages', 'nb_participants'})
# if meta['chat']: # if meta['chat']:
# meta['chat'] = get_chat_meta_from_global_id(meta['chat']) # meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target) meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target)
@ -367,8 +405,7 @@ def api_get_user_account(user_id, instance_uuid):
user_account = UsersAccount.UserAccount(user_id, instance_uuid) user_account = UsersAccount.UserAccount(user_id, instance_uuid)
if not user_account.exists(): if not user_account.exists():
return {"status": "error", "reason": "Unknown user-account"}, 404 return {"status": "error", "reason": "Unknown user-account"}, 404
meta = user_account.get_meta({'icon', 'username'}) meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'})
print(meta)
return meta, 200 return meta, 200
# # # # # # # # # # LATER # # # # # # # # # # LATER

7
bin/lib/objects/ChatSubChannels.py
View File

@ -70,7 +70,7 @@ class ChatSubChannel(AbstractChatObject):
# else: # else:
# style = 'fas' # style = 'fas'
# icon = '\uf007' # icon = '\uf007'
style = 'fas' style = 'far'
icon = '\uf086' icon = '\uf086'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
@ -90,7 +90,10 @@ class ChatSubChannel(AbstractChatObject):
meta['created_at'] = self.get_created_at(date=True) meta['created_at'] = self.get_created_at(date=True)
if 'threads' in options: if 'threads' in options:
meta['threads'] = self.get_threads() meta['threads'] = self.get_threads()
print(meta['threads']) if 'participants' in options:
meta['participants'] = self.get_participants()
if 'nb_participants' in options:
meta['nb_participants'] = self.get_nb_participants()
return meta return meta
def get_misp_object(self): def get_misp_object(self):

4
bin/lib/objects/ChatThreads.py
View File

@ -77,6 +77,10 @@ class ChatThread(AbstractChatObject):
meta['name'] = self.get_name() meta['name'] = self.get_name()
if 'nb_messages': if 'nb_messages':
meta['nb_messages'] = self.get_nb_messages() meta['nb_messages'] = self.get_nb_messages()
if 'participants':
meta['participants'] = self.get_participants()
if 'nb_participants':
meta['nb_participants'] = self.get_nb_participants()
# created_at ??? # created_at ???
return meta return meta

8
bin/lib/objects/Chats.py
View File

@ -74,10 +74,14 @@ class Chat(AbstractChatObject):
meta = self._get_meta(options=options) meta = self._get_meta(options=options)
meta['name'] = self.get_name() meta['name'] = self.get_name()
meta['tags'] = self.get_tags(r_list=True) meta['tags'] = self.get_tags(r_list=True)
if 'icon': if 'icon' in options:
meta['icon'] = self.get_icon() meta['icon'] = self.get_icon()
if 'info': if 'info' in options:
meta['info'] = self.get_info() meta['info'] = self.get_info()
if 'participants' in options:
meta['participants'] = self.get_participants()
if 'nb_participants' in options:
meta['nb_participants'] = self.get_nb_participants()
if 'username' in options: if 'username' in options:
meta['username'] = self.get_username() meta['username'] = self.get_username()
if 'subchannels' in options: if 'subchannels' in options:

47
bin/lib/objects/UsersAccount.py
View File

@ -54,16 +54,7 @@ class UserAccount(AbstractSubtypeObject):
return url return url
def get_svg_icon(self): # TODO change icon/color def get_svg_icon(self): # TODO change icon/color
if self.subtype == 'telegram': return {'style': 'fas', 'icon': '\uf2bd', 'color': '#4dffff', 'radius': 5}
style = 'fab'
icon = '\uf2c6'
elif self.subtype == 'twitter':
style = 'fab'
icon = '\uf099'
else:
style = 'fas'
icon = '\uf007'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_first_name(self): def get_first_name(self):
return self._get_field('firstname') return self._get_field('firstname')
@ -97,6 +88,25 @@ class UserAccount(AbstractSubtypeObject):
def set_info(self, info): def set_info(self, info):
return self._set_field('info', info) return self._set_field('info', info)
# TODO MESSAGES:
# 1) ALL MESSAGES + NB
# 2) ALL MESSAGES TIMESTAMP
# 3) ALL MESSAGES TIMESTAMP By: - chats
# - subchannel
# - thread
def get_chats(self):
chats = self.get_correlation('chat')['chat']
return chats
def get_chat_subchannels(self):
chats = self.get_correlation('chat-subchannel')['chat-subchannel']
return chats
def get_chat_threads(self):
chats = self.get_correlation('chat-thread')['chat-thread']
return chats
def _get_timeline_username(self): def _get_timeline_username(self):
return Timeline(self.get_global_id(), 'username') return Timeline(self.get_global_id(), 'username')
@ -109,20 +119,31 @@ class UserAccount(AbstractSubtypeObject):
def update_username_timeline(self, username_global_id, timestamp): def update_username_timeline(self, username_global_id, timestamp):
self._get_timeline_username().add_timestamp(timestamp, username_global_id) self._get_timeline_username().add_timestamp(timestamp, username_global_id)
def get_meta(self, options=set()): def get_meta(self, options=set()): # TODO Username timeline
meta = self._get_meta(options=options) meta = self._get_meta(options=options)
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True) # TODO add in options ???? meta['tags'] = self.get_tags(r_list=True) # TODO add in options ????
if 'username' in options: if 'username' in options:
meta['username'] = self.get_username() meta['username'] = self.get_username()
if meta['username'] and 'username_meta' in options: if meta['username']:
_, username_account_subtype, username_account_id = meta['username'].split(':', 3) _, username_account_subtype, username_account_id = meta['username'].split(':', 3)
meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta() if 'username_meta' in options:
meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta()
else:
meta['username'] = {'type': 'username', 'subtype': username_account_subtype, 'id': username_account_id}
if 'usernames' in options: if 'usernames' in options:
meta['usernames'] = self.get_usernames() meta['usernames'] = self.get_usernames()
if 'icon' in options: if 'icon' in options:
meta['icon'] = self.get_icon() meta['icon'] = self.get_icon()
if 'info' in options:
meta['info'] = self.get_info()
if 'chats' in options:
meta['chats'] = self.get_chats()
if 'subchannels' in options:
meta['subchannels'] = self.get_chat_subchannels()
if 'threads' in options:
meta['threads'] = self.get_chat_threads()
return meta return meta
def get_misp_object(self): def get_misp_object(self):

7
bin/lib/objects/abstract_chat_object.py
View File

@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.objects.abstract_subtype_object import AbstractSubtypeObject from lib.objects.abstract_subtype_object import AbstractSubtypeObject
from lib.ail_core import get_object_all_subtypes, zscan_iter ################ from lib.ail_core import unpack_correl_objs_id, zscan_iter ################
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects import Messages from lib.objects import Messages
from packages import Date from packages import Date
@ -244,8 +244,11 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
# def get_deleted_messages(self, message_id): # def get_deleted_messages(self, message_id):
def get_participants(self):
return unpack_correl_objs_id('user-account', self.get_correlation('user-account')['user-account'], r_type='dict')
# get_messages_meta ???? def get_nb_participants(self):
return self.get_nb_correlation('user-account')
# TODO move me to abstract subtype # TODO move me to abstract subtype
class AbstractChatObjects(ABC): class AbstractChatObjects(ABC):

27
var/www/blueprints/chats_explorer.py
View File

@ -132,6 +132,20 @@ def objects_thread_messages():
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target)
@chats_explorer.route("/chats/explorer/participants", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_chat_participants():
chat_type = request.args.get('type')
chat_id = request.args.get('id')
chat_subtype = request.args.get('subtype')
meta = chats_viewer.api_get_chat_participants(chat_type, chat_subtype,chat_id)
if meta[1] != 200:
return create_json_response(meta[0], meta[1])
else:
meta = meta[0]
return render_template('chat_participants.html', meta=meta, bootstrap_label=bootstrap_label)
@chats_explorer.route("/objects/message", methods=['GET']) @chats_explorer.route("/objects/message", methods=['GET'])
@login_required @login_required
@login_read_only @login_read_only
@ -145,3 +159,16 @@ def objects_message():
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label,
modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message'))
@chats_explorer.route("/objects/user-account", methods=['GET'])
@login_required
@login_read_only
def objects_user_account():
instance_uuid = request.args.get('subtype')
user_id = request.args.get('id')
user_account = chats_viewer.api_get_user_account(user_id, instance_uuid)
if user_account[1] != 200:
return create_json_response(user_account[0], user_account[1])
else:
user_account = user_account[0]
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label)

12
var/www/blueprints/objects_subtypes.py
View File

@ -43,7 +43,8 @@ def subtypes_objects_dashboard(obj_type, f_request):
date_to = f_request.form.get('to') date_to = f_request.form.get('to')
subtype = f_request.form.get('subtype') subtype = f_request.form.get('subtype')
show_objects = bool(f_request.form.get('show_objects')) show_objects = bool(f_request.form.get('show_objects'))
endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{obj_type}') t_obj_type = obj_type.replace('-', '_')
endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{t_obj_type}')
endpoint_dashboard = f'{endpoint_dashboard}?from={date_from}&to={date_to}' endpoint_dashboard = f'{endpoint_dashboard}?from={date_from}&to={date_to}'
if subtype: if subtype:
if subtype == 'All types': if subtype == 'All types':
@ -81,7 +82,8 @@ def subtypes_objects_dashboard(obj_type, f_request):
for obj_t, obj_subtype, obj_id in subtypes_objs: for obj_t, obj_subtype, obj_id in subtypes_objs:
objs.append(ail_objects.get_object_meta(obj_t, obj_subtype, obj_id, options={'sparkline'}, flask_context=True)) objs.append(ail_objects.get_object_meta(obj_t, obj_subtype, obj_id, options={'sparkline'}, flask_context=True))
endpoint_dashboard = f'objects_subtypes.objects_dashboard_{obj_type}' t_obj_type = obj_type.replace('-', '_')
endpoint_dashboard = f'objects_subtypes.objects_dashboard_{t_obj_type}'
return render_template('subtypes_objs_dashboard.html', date_from=date_from, date_to=date_to, return render_template('subtypes_objs_dashboard.html', date_from=date_from, date_to=date_to,
daily_type_chart = daily_type_chart, show_objects=show_objects, daily_type_chart = daily_type_chart, show_objects=show_objects,
obj_type=obj_type, subtype=subtype, objs=objs, obj_type=obj_type, subtype=subtype, objs=objs,
@ -115,6 +117,12 @@ def objects_dashboard_pgp():
def objects_dashboard_username(): def objects_dashboard_username():
return subtypes_objects_dashboard('username', request) return subtypes_objects_dashboard('username', request)
@objects_subtypes.route("/objects/user-accounts", methods=['GET'])
@login_required
@login_read_only
def objects_dashboard_user_account():
return subtypes_objects_dashboard('user-account', request)
# TODO REDIRECT # TODO REDIRECT
@objects_subtypes.route("/objects/subtypes/post", methods=['POST']) @objects_subtypes.route("/objects/subtypes/post", methods=['POST'])
@login_required @login_required

4
var/www/templates/chats_explorer/SubChannelMessages.html
View File

@ -68,6 +68,7 @@
<th>Last seen</th> <th>Last seen</th>
<th>Username</th> <th>Username</th>
<th>Nb Messages</th> <th>Nb Messages</th>
<th>Participants</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -94,6 +95,9 @@
{% endif %} {% endif %}
</td> </td>
<td>{{ subchannel['nb_messages'] }}</td> <td>{{ subchannel['nb_messages'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat-subchannel&subtype={{ subchannel['subtype'] }}&id={{ subchannel['id'] }}"><i class="far fa-user-circle"></i> {{ subchannel['nb_participants']}}</a>
</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

5
var/www/templates/chats_explorer/ThreadMessages.html
View File

@ -67,6 +67,7 @@
<th>First seen</th> <th>First seen</th>
<th>Last seen</th> <th>Last seen</th>
<th>Nb Messages</th> <th>Nb Messages</th>
<th>Participants</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -74,6 +75,7 @@
<td> <td>
{{ meta['name'] }} {{ meta['name'] }}
</td> </td>
<td></td>
<td> <td>
{% if meta['first_seen'] %} {% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
@ -85,6 +87,9 @@
{% endif %} {% endif %}
</td> </td>
<td>{{ meta['nb_messages'] }}</td> <td>{{ meta['nb_messages'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat-thread&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}"><i class="far fa-user-circle"></i> {{ meta['nb_participants']}}</a>
</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

6
var/www/templates/chats_explorer/block_message.html
View File

@ -26,8 +26,10 @@
<div class="chat-message-left pb-1"> <div class="chat-message-left pb-1">
<div> <div>
<img src="{% if message['user-account']['icon'] %}{{ url_for('objects_image.image', filename=message['user-account']['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}" <a href="{{ url_for('chats_explorer.objects_user_account')}}?subtype={{ message['user-account']['subtype'] }}&id={{ message['user-account']['id'] }}">
class="rounded-circle mr-1" alt="{{ message['user-account']['id'] }}" width="40" height="40"> <img src="{% if message['user-account']['icon'] %}{{ url_for('objects_image.image', filename=message['user-account']['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
class="rounded-circle mr-1" alt="{{ message['user-account']['id'] }}" width="40" height="40">
</a>
<div class="text-muted small text-nowrap mt-2">{{ message['hour'] }}</div> <div class="text-muted small text-nowrap mt-2">{{ message['hour'] }}</div>
</div> </div>
<div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto"> <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">

174
var/www/templates/chats_explorer/chat_participants.html Normal file
View File

@ -0,0 +1,174 @@
<!DOCTYPE html>
<html>
<head>
<title>Chats Protocols - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
{# <div class="card my-3">#} TODO CHAT abstract metadata
{##}
{# <div class="card-header" style="background-color:#d9edf7;font-size: 15px">#}
{# <h4 class="text-secondary">{% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :</h4>#}
{# {% if chat['icon'] %}#}
{# <div><img src="{{ url_for('objects_image.image', filename=chat['icon'])}}" class="mb-2" alt="{{ chat['id'] }}" width="200" height="200"></div>#}
{# {% endif %}#}
{# <ul class="list-group mb-2">#}
{# <li class="list-group-item py-0">#}
{# <table class="table">#}
{# <thead class="">#}
{# <tr>#}
{# <th>Name</th>#}
{# <th>ID</th>#}
{# <th>Created at</th>#}
{# <th>First Seen</th>#}
{# <th>Last Seen</th>#}
{# <th>NB Sub-Channels</th>#}
{# <th>Participants</th>#}
{# </tr>#}
{# </thead>#}
{# <tbody style="font-size: 15px;">#}
{# <tr>#}
{# <td>{{ chat['name'] }}</td>#}
{# <td>{{ chat['id'] }}</td>#}
{# <td>{{ chat['created_at'] }}</td>#}
{# <td>#}
{# {% if chat['first_seen'] %}#}
{# {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>#}
{# {% if chat['last_seen'] %}#}
{# {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>{{ chat['nb_subchannels'] }}</td>#}
{# <td>#}
{# <a href="{{ url_for('chats_explorer.objects_thread_messages')}}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}"><i class="far fa-user-circle"></i> {{ chat['participants'] | length}}</a>#}
{##}
{# </td>#}
{# </tr>#}
{# </tbody>#}
{# </table>#}
{# {% if chat['info'] %}#}
{# <li class="list-group-item py-0">#}
{# <pre class="my-0">{{ chat['info'] }}</pre>#}
{# </li>#}
{# {% endif %}#}
{# </li>#}
{# </ul>#}
{##}
{# </div>#}
{# </div>#}
<h4>Participants:</h4>
<table id="tableparticipantss" class="table">
<thead class="bg-dark text-white">
<tr>
<th>Icon</th>
<th>Username</th>
<th>ID</th>
<th>info</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Messages</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for user_meta in meta["participants"] %}
<tr>
<td>
<a href="{{ url_for('chats_explorer.objects_user_account')}}?subtype={{ user_meta['subtype'] }}&id={{ user_meta['id'] }}">
<img src="{% if user_meta['icon'] %}{{ url_for('objects_image.image', filename=user_meta['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
class="rounded-circle mr-1" alt="{{ user_meta['id'] }}" width="40" height="40">
<a>
</td>
<td>
{% if user_meta['username'] %}
<b>{{ user_meta['username']['id'] }}</b>
{% endif %}
</td>
<td><a href="{{ url_for('chats_explorer.objects_user_account') }}?subtype={{ user_meta['subtype'] }}&id={{ user_meta['id'] }}">{{ user_meta['id'] }}</a></td>
<td>
{% if user_meta['info'] %}
{{ user_meta['info'] }}
{% endif %}
</td>
<td>
{% if user_meta['first_seen'] %}
{{ user_meta['first_seen'][0:4] }}-{{ user_meta['first_seen'][4:6] }}-{{ user_meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if user_meta['last_seen'] %}
{{ user_meta['last_seen'][0:4] }}-{{ user_meta['last_seen'][4:6] }}-{{ user_meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ user_meta['nb_messages'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
$('#tableparticipantss').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 5, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

6
var/www/templates/chats_explorer/chat_viewer.html
View File

@ -66,18 +66,17 @@
<table class="table"> <table class="table">
<thead class=""> <thead class="">
<tr> <tr>
<th>Icon</th>
<th>Name</th> <th>Name</th>
<th>ID</th> <th>ID</th>
<th>Created at</th> <th>Created at</th>
<th>First Seen</th> <th>First Seen</th>
<th>Last Seen</th> <th>Last Seen</th>
<th>NB Sub-Channels</th> <th>NB Sub-Channels</th>
<th>Participants</th>
</tr> </tr>
</thead> </thead>
<tbody style="font-size: 15px;"> <tbody style="font-size: 15px;">
<tr> <tr>
<td></td>
<td>{{ chat['name'] }}</td> <td>{{ chat['name'] }}</td>
<td>{{ chat['id'] }}</td> <td>{{ chat['id'] }}</td>
<td>{{ chat['created_at'] }}</td> <td>{{ chat['created_at'] }}</td>
@ -92,6 +91,9 @@
{% endif %} {% endif %}
</td> </td>
<td>{{ chat['nb_subchannels'] }}</td> <td>{{ chat['nb_subchannels'] }}</td>
<td>
<a href="{{ url_for('chats_explorer.chats_explorer_chat_participants')}}?type=chat&subtype={{ chat['subtype'] }}&id={{ chat['id'] }}"><i class="far fa-user-circle"></i> {{ chat['nb_participants']}}</a>
</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

184
var/www/templates/chats_explorer/user_account.html Normal file
View File

@ -0,0 +1,184 @@
<!DOCTYPE html>
<html>
<head>
<title>User Account - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %} </h4>
{% if meta['icon'] %}
<div><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="mb-2" alt="{{ meta['id'] }}" width="250" height="250"></div>
{% endif %}
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<table class="table">
<thead class="">
<tr>
<th>username</th>
<th>ID</th>
<th>Created at</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Chats</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{ meta['username']['id'] }}</td>
<td>{{ meta['id'] }}</td>
<td>{{ meta['created_at'] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
{% endif %}
</td>
<td>
{% if meta['last_seen'] %}
{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
{% endif %}
</td>
<td>{{ meta['chats'] | length }}</td>
</tr>
</tbody>
</table>
{% if meta['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ meta['info'] }}</pre>
</li>
{% endif %}
</li>
</ul>
<div class="d-flex flex-row-reverse bd-highlight">
<div>
<a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}" target="_blank">
<button class="btn btn-lg btn-info"><i class="fas fa-project-diagram"></i> Correlations Graph</button>
</a>
</div>
{# <div>#}
{# {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%}#}
{# {% include 'modals/investigations_register_obj.html' %}#}
{# {% endwith %}#}
{# <div class="mr-2">#}
{# <button type="button" class="btn btn-lg btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">#}
{# <i class="fas fa-microscope"></i> Investigations#}
{# </button>#}
{# </div>#}
{# </div>#}
</div>
</div>
</div>
{# {% if meta['subchannels'] %}#}
{# <h4>Sub-Channels:</h4>#}
{# <table id="tablesubchannels" class="table">#}
{# <thead class="bg-dark text-white">#}
{# <tr>#}
{# <th></th>#}
{# <th></th>#}
{# <th></th>#}
{# <th></th>#}
{# </tr>#}
{# </thead>#}
{# <tbody style="font-size: 15px;">#}
{# {% for meta in meta["subchannels"] %}#}
{# <tr>#}
{# <td>#}
{# <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">#}
{# </td>#}
{# <td><b>{{ meta['name'] }}</b></td>#}
{# <td><a href="{{ url_for('metas_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>#}
{# <td>{{ meta['created_at'] }}</td>#}
{# <td>#}
{# {% if meta['first_seen'] %}#}
{# {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>#}
{# {% if meta['last_seen'] %}#}
{# {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}#}
{# {% endif %}#}
{# </td>#}
{# <td>{{ meta['nb_messages'] }}</td>#}
{# </tr>#}
{# {% endfor %}#}
{# </tbody>#}
{# </table>#}
{##}
{# {% endif %}#}
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
{# {% if meta['subchannels'] %}#}
{# $('#tablesubchannels').DataTable({#}
{# "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],#}
{# "iDisplayLength": 10,#}
{# "order": [[ 5, "desc" ]]#}
{# });#}
{# {% endif %}#}
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

6
var/www/templates/sidebars/sidebar_objects.html
View File

@ -106,6 +106,12 @@
<span>Username</span> <span>Username</span>
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('objects_subtypes.objects_dashboard_user_account')}}" id="nav_dashboard_user_account">
<i class="fas fa-user-circle"></i>
<span>User-Acoount</span>
</a>
</li>
</ul> </ul>