CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [translation] translate chats info, users info and subchannels names

dev
terrtia 2024-01-29 14:36:53 +01:00
parent 6363a4f1cf
commit 896b411eaf
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
12 changed files with 95 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bin/lib/Language.py
View File

@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate')
config_loader = None
@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True):
content = content.replace(it, '')
return content
#### AIL Objects ####
def get_obj_translation(obj_global_id, content, field='', source=None, target='en'):
"""
Returns translated content
"""
translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}')
if translation:
# DEBUG
# print('cache')
# r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0)
return translation
translation = LanguageTranslator().translate(content, source=source, target=target)
if translation:
r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation)
r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300)
return translation
## --AIL Objects-- ##
class LanguagesDetector:

16
bin/lib/chats_viewer.py
View File

@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()):
options.add(option)
return obj_chat.get_meta(options=options)
def get_subchannels_meta_from_global_id(subchannels):
def get_subchannels_meta_from_global_id(subchannels, translation_target=None):
meta = []
for sub in subchannels:
_, instance_uuid, sub_id = sub.split(':', 2)
subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'}))
meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target))
return meta
def get_chat_meta_from_global_id(chat_global_id):
@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'})
meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
if meta['username']:
meta['username'] = get_username_meta_from_global_id(meta['username'])
if meta['subchannels']:
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target)
else:
if translation_target not in Language.LIST_LANGUAGES:
if translation_target not in Language.get_translation_languages():
translation_target = None
meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
return meta, 200
@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists():
return {"status": "error", "reason": "Unknown subchannel"}, 404
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'})
meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target)
if meta['chat']:
meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
if meta.get('threads'):
@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None):
meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target)
return meta, 200
def api_get_user_account(user_id, instance_uuid):
def api_get_user_account(user_id, instance_uuid, translation_target=None):
user_account = UsersAccount.UserAccount(user_id, instance_uuid)
if not user_account.exists():
return {"status": "error", "reason": "Unknown user-account"}, 404
meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'})
meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
return meta, 200
# # # # # # # # # # LATER

4
bin/lib/objects/ChatSubChannels.py
View File

@ -76,7 +76,7 @@ class ChatSubChannel(AbstractChatObject):
# TODO TIME LAST MESSAGES
def get_meta(self, options=set()):
def get_meta(self, options=set(), translation_target=None):
meta = self._get_meta(options=options)
meta['tags'] = self.get_tags(r_list=True)
meta['name'] = self.get_name()
@ -95,6 +95,8 @@ class ChatSubChannel(AbstractChatObject):
meta['participants'] = self.get_participants()
if 'nb_participants' in options:
meta['nb_participants'] = self.get_nb_participants()
if 'translation' in options and translation_target:
meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target)
return meta
def get_misp_object(self):

4
bin/lib/objects/Chats.py
View File

@ -70,7 +70,7 @@ class Chat(AbstractChatObject):
icon = '\uf086'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
def get_meta(self, options=set()):
def get_meta(self, options=set(), translation_target=None):
meta = self._get_meta(options=options)
meta['name'] = self.get_name()
meta['tags'] = self.get_tags(r_list=True)
@ -79,6 +79,8 @@ class Chat(AbstractChatObject):
meta['img'] = meta['icon']
if 'info' in options:
meta['info'] = self.get_info()
if 'translation' in options and translation_target:
meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
if 'participants' in options:
meta['participants'] = self.get_participants()
if 'nb_participants' in options:

3
bin/lib/objects/Messages.py
View File

@ -179,6 +179,7 @@ class Message(AbstractObject):
"""
Returns translated content
"""
# return self._get_field('translated')
global_id = self.get_global_id()
translation = r_cache.get(f'translation:{target}:{global_id}')
@ -289,7 +290,7 @@ class Message(AbstractObject):
if 'reactions' in options:
meta['reactions'] = self.get_reactions()
if 'translation' in options and translation_target:
meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target)
meta['translation'] = self.translate(content=meta.get('content'), target=translation_target)
# meta['encoding'] = None
return meta

14
bin/lib/objects/UsersAccount.py
View File

@ -5,6 +5,7 @@ import os
import sys
# import re
# from datetime import datetime
from flask import url_for
from pymisp import MISPObject
@ -88,6 +89,13 @@ class UserAccount(AbstractSubtypeObject):
def set_info(self, info):
return self._set_field('info', info)
# def get_created_at(self, date=False):
# created_at = self._get_field('created_at')
# if date and created_at:
# created_at = datetime.fromtimestamp(float(created_at))
# created_at = created_at.isoformat(' ')
# return created_at
# TODO MESSAGES:
# 1) ALL MESSAGES + NB
# 2) ALL MESSAGES TIMESTAMP
@ -122,7 +130,7 @@ class UserAccount(AbstractSubtypeObject):
def get_messages_by_chat_obj(self, chat_obj):
return self.get_correlation_iter_obj(chat_obj, 'message')
def get_meta(self, options=set()): # TODO Username timeline
def get_meta(self, options=set(), translation_target=None): # TODO Username timeline
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['subtype'] = self.subtype
@ -141,6 +149,10 @@ class UserAccount(AbstractSubtypeObject):
meta['icon'] = self.get_icon()
if 'info' in options:
meta['info'] = self.get_info()
if 'translation' in options and translation_target:
meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
# if 'created_at':
# meta['created_at'] = self.get_created_at(date=True)
if 'chats' in options:
meta['chats'] = self.get_chats()
if 'subchannels' in options:

11
bin/lib/objects/abstract_object.py
View File

@ -25,6 +25,7 @@ from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
from lib.Language import get_obj_translation
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
logging.config.dictConfig(ail_logger.get_config(name='ail'))
@ -301,6 +302,16 @@ class AbstractObject(ABC):
## -Relationship- ##
## Translation ##
def translate(self, content=None, field='', source=None, target='en'):
global_id = self.get_global_id()
if not content:
content = self.get_content()
return get_obj_translation(global_id, content, field=field, source=source, target=target)
## -Translation- ##
## Parent ##
def is_parent(self):

9
var/www/blueprints/chats_explorer.py
View File

@ -182,9 +182,14 @@ def objects_message():
def objects_user_account():
instance_uuid = request.args.get('subtype')
user_id = request.args.get('id')
user_account = chats_viewer.api_get_user_account(user_id, instance_uuid)
target = request.args.get('target')
if target == "Don't Translate":
target = None
user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
if user_account[1] != 200:
return create_json_response(user_account[0], user_account[1])
else:
user_account = user_account[0]
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label)
languages = Language.get_translation_languages()
return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
translation_languages=languages, translation_target=target)

3
var/www/templates/chats_explorer/SubChannelMessages.html
View File

@ -75,6 +75,9 @@
<tr>
<td>
{{ subchannel['name'] }}
{% if subchannel['translation_name'] %}
<div class="text-secondary">{{ subchannel['translation_name'] }}</div>
{% endif %}
</td>
<td>{{ subchannel["created_at"] }}</td>
<td>

2
var/www/templates/chats_explorer/chat_instance.html
View File

@ -71,7 +71,7 @@
<th>First Seen</th>
<th>Last Seen</th>
<th>SubChannels</th>
<th>Messages</th>
<th><i class="fas fa-comment-dots"></i></th>
</tr>
</thead>
<tbody style="font-size: 15px;">

22
var/www/templates/chats_explorer/chat_viewer.html
View File

@ -100,6 +100,10 @@
{% if chat['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ chat['info'] }}</pre>
{% if chat['translation_info'] %}
<hr class="m-1">
<pre class="my-0 text-secondary">{{ chat['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
</li>
@ -112,8 +116,12 @@
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ chat['tags_messages'][tag] }}</span></span>
{% endfor %}
{% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% if chat['subchannels'] %}
<h4>Sub-Channels:</h4>
<h4 class="mt-2">Sub-Channels:</h4>
<table id="tablesubchannels" class="table">
<thead class="bg-dark text-white">
<tr>
@ -123,7 +131,7 @@
<th>Created at</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Messages</th>
<th><i class="fas fa-comment-dots"></i></th>
</tr>
</thead>
<tbody style="font-size: 15px;">
@ -132,7 +140,12 @@
<td>
<img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
</td>
<td><b>{{ meta['name'] }}</b></td>
<td>
<b>{{ meta['name'] }}</b>
{% if meta['translation_name'] %}
<div class="text-secondary">{{ meta['translation_name'] }}</div>
{% endif %}
</td>
<td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
<td>{{ meta['created_at'] }}</td>
<td>
@ -161,9 +174,6 @@
{% include 'objects/image/block_blur_img_slider.html' %}
</span>
{% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %}
{% set date_from=chat['first_seen'] %}
{% set date_to=chat['last_seen'] %}

10
var/www/templates/chats_explorer/user_account.html
View File

@ -46,7 +46,6 @@
<tr>
<th>username</th>
<th>ID</th>
<th>Created at</th>
<th>First Seen</th>
<th>Last Seen</th>
<th>NB Chats</th>
@ -56,7 +55,6 @@
<tr>
<td>{{ meta['username']['id'] }}</td>
<td>{{ meta['id'] }}</td>
<td>{{ meta['created_at'] }}</td>
<td>
{% if meta['first_seen'] %}
{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
@ -74,6 +72,10 @@
{% if meta['info'] %}
<li class="list-group-item py-0">
<pre class="my-0">{{ meta['info'] }}</pre>
{% if meta['translation_info'] %}
<hr>
<pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
{% endif %}
</li>
{% endif %}
</li>
@ -100,6 +102,10 @@
</div>
</div>
{% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %}
{% endwith %}
{# {% if meta['subchannels'] %}#}
{# <h4>Sub-Channels:</h4>#}