chg: [translation] translate chats info, users info and subchannels names

bin/lib/Language.py

@@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib.ConfigLoader import ConfigLoader
 
 config_loader = ConfigLoader()
+r_cache = config_loader.get_redis_conn("Redis_Cache")
 TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate')
 config_loader = None
 
@@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True):
             content = content.replace(it, '')
     return content
 
+#### AIL Objects ####
+
+def get_obj_translation(obj_global_id, content, field='', source=None, target='en'):
+    """
+    Returns translated content
+    """
+    translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}')
+    if translation:
+        # DEBUG
+        # print('cache')
+        # r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0)
+        return translation
+    translation = LanguageTranslator().translate(content, source=source, target=target)
+    if translation:
+        r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation)
+        r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300)
+    return translation
+
+## --AIL Objects-- ##
 
 class LanguagesDetector:
 

bin/lib/chats_viewer.py

@@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()):
         options.add(option)
     return obj_chat.get_meta(options=options)
 
-def get_subchannels_meta_from_global_id(subchannels):
+def get_subchannels_meta_from_global_id(subchannels, translation_target=None):
     meta = []
     for sub in subchannels:
         _, instance_uuid, sub_id = sub.split(':', 2)
         subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid)
-        meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'}))
+        meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target))
     return meta
 
 def get_chat_meta_from_global_id(chat_global_id):
@@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
     chat = Chats.Chat(chat_id, chat_instance_uuid)
     if not chat.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
-    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'})
+    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
     if meta['username']:
         meta['username'] = get_username_meta_from_global_id(meta['username'])
     if meta['subchannels']:
-        meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'])
+        meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target)
     else:
-        if translation_target not in Language.LIST_LANGUAGES:
+        if translation_target not in Language.get_translation_languages():
             translation_target = None
         meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
     return meta, 200
@@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=
     subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
     if not subchannel.exists():
         return {"status": "error", "reason": "Unknown subchannel"}, 404
-    meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'})
+    meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target)
     if meta['chat']:
         meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
     if meta.get('threads'):
@@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None):
     meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target)
     return meta, 200
 
-def api_get_user_account(user_id, instance_uuid):
+def api_get_user_account(user_id, instance_uuid, translation_target=None):
     user_account = UsersAccount.UserAccount(user_id, instance_uuid)
     if not user_account.exists():
         return {"status": "error", "reason": "Unknown user-account"}, 404
-    meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'})
+    meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
     return meta, 200
 
 # # # # # # # # # # LATER

bin/lib/objects/ChatSubChannels.py

@@ -76,7 +76,7 @@ class ChatSubChannel(AbstractChatObject):
 
     # TODO TIME LAST MESSAGES
 
-    def get_meta(self, options=set()):
+    def get_meta(self, options=set(), translation_target=None):
         meta = self._get_meta(options=options)
         meta['tags'] = self.get_tags(r_list=True)
         meta['name'] = self.get_name()
@@ -95,6 +95,8 @@ class ChatSubChannel(AbstractChatObject):
             meta['participants'] = self.get_participants()
         if 'nb_participants' in options:
             meta['nb_participants'] = self.get_nb_participants()
+        if 'translation' in options and translation_target:
+            meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target)
         return meta
 
     def get_misp_object(self):

bin/lib/objects/Chats.py

@@ -70,7 +70,7 @@ class Chat(AbstractChatObject):
         icon = '\uf086'
         return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
 
-    def get_meta(self, options=set()):
+    def get_meta(self, options=set(), translation_target=None):
         meta = self._get_meta(options=options)
         meta['name'] = self.get_name()
         meta['tags'] = self.get_tags(r_list=True)
@@ -79,6 +79,8 @@ class Chat(AbstractChatObject):
             meta['img'] = meta['icon']
         if 'info' in options:
             meta['info'] = self.get_info()
+            if 'translation' in options and translation_target:
+                meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
         if 'participants' in options:
             meta['participants'] = self.get_participants()
         if 'nb_participants' in options:

bin/lib/objects/Messages.py

@@ -179,6 +179,7 @@ class Message(AbstractObject):
         """
         Returns translated content
         """
+
         # return self._get_field('translated')
         global_id = self.get_global_id()
         translation = r_cache.get(f'translation:{target}:{global_id}')
@@ -289,7 +290,7 @@ class Message(AbstractObject):
         if 'reactions' in options:
             meta['reactions'] = self.get_reactions()
         if 'translation' in options and translation_target:
-            meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target)
+            meta['translation'] = self.translate(content=meta.get('content'), target=translation_target)
 
         # meta['encoding'] = None
         return meta

bin/lib/objects/UsersAccount.py

@@ -5,6 +5,7 @@ import os
 import sys
 # import re
 
+# from datetime import datetime
 from flask import url_for
 from pymisp import MISPObject
 
@@ -88,6 +89,13 @@ class UserAccount(AbstractSubtypeObject):
     def set_info(self, info):
         return self._set_field('info', info)
 
+    # def get_created_at(self, date=False):
+    #     created_at = self._get_field('created_at')
+    #     if date and created_at:
+    #         created_at = datetime.fromtimestamp(float(created_at))
+    #         created_at = created_at.isoformat(' ')
+    #     return created_at
+
     # TODO MESSAGES:
     # 1) ALL MESSAGES + NB
     # 2) ALL MESSAGES TIMESTAMP
@@ -122,7 +130,7 @@ class UserAccount(AbstractSubtypeObject):
     def get_messages_by_chat_obj(self, chat_obj):
         return self.get_correlation_iter_obj(chat_obj, 'message')
 
-    def get_meta(self, options=set()): # TODO Username timeline
+    def get_meta(self, options=set(), translation_target=None): # TODO Username timeline
         meta = self._get_meta(options=options)
         meta['id'] = self.id
         meta['subtype'] = self.subtype
@@ -141,6 +149,10 @@ class UserAccount(AbstractSubtypeObject):
             meta['icon'] = self.get_icon()
         if 'info' in options:
             meta['info'] = self.get_info()
+            if 'translation' in options and translation_target:
+                meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target)
+        # if 'created_at':
+        #     meta['created_at'] = self.get_created_at(date=True)
         if 'chats' in options:
             meta['chats'] = self.get_chats()
         if 'subchannels' in options:

bin/lib/objects/abstract_object.py

@@ -25,6 +25,7 @@ from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
 from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
+from lib.Language import get_obj_translation
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -301,6 +302,16 @@ class AbstractObject(ABC):
 
     ## -Relationship- ##
 
+    ## Translation ##
+
+    def translate(self, content=None, field='', source=None, target='en'):
+        global_id = self.get_global_id()
+        if not content:
+            content = self.get_content()
+        return get_obj_translation(global_id, content, field=field, source=source, target=target)
+
+    ## -Translation- ##
+
     ## Parent ##
 
     def is_parent(self):

var/www/blueprints/chats_explorer.py

@@ -182,9 +182,14 @@ def objects_message():
 def objects_user_account():
     instance_uuid = request.args.get('subtype')
     user_id = request.args.get('id')
-    user_account = chats_viewer.api_get_user_account(user_id, instance_uuid)
+    target = request.args.get('target')
+    if target == "Don't Translate":
+        target = None
+    user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
     if user_account[1] != 200:
         return create_json_response(user_account[0], user_account[1])
     else:
         user_account = user_account[0]
-        return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label)
+        languages = Language.get_translation_languages()
+        return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
+                               translation_languages=languages, translation_target=target)

var/www/templates/chats_explorer/SubChannelMessages.html

@@ -75,6 +75,9 @@
                                                 <tr>
                                                     <td>
                                                         {{ subchannel['name'] }}
+                                                        {% if subchannel['translation_name'] %}
+                                                            <div class="text-secondary">{{ subchannel['translation_name'] }}</div>
+                                                        {% endif %}
                                                     </td>
                                                     <td>{{ subchannel["created_at"] }}</td>
                                                     <td>

var/www/templates/chats_explorer/chat_instance.html

@@ -71,7 +71,7 @@
                         <th>First Seen</th>
                         <th>Last Seen</th>
                         <th>SubChannels</th>
-                        <th>Messages</th>
+                        <th><i class="fas fa-comment-dots"></i></th>
                     </tr>
                     </thead>
                     <tbody style="font-size: 15px;">

var/www/templates/chats_explorer/chat_viewer.html

@@ -100,6 +100,10 @@
                             {% if chat['info'] %}
                                 <li class="list-group-item py-0">
                                     <pre class="my-0">{{ chat['info'] }}</pre>
+                                    {% if chat['translation_info'] %}
+                                        <hr class="m-1">
+                                        <pre class="my-0 text-secondary">{{ chat['translation_info'] }}</pre>
+                                    {% endif %}
                                 </li>
                             {% endif %}
                             </li>
@@ -112,8 +116,12 @@
                     <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }} <span class="badge badge-light">{{ chat['tags_messages'][tag] }}</span></span>
                 {%  endfor %}
 
+                {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
+                    {% include 'chats_explorer/block_translation.html' %}
+                {% endwith %}
+
                 {% if chat['subchannels'] %}
-                    <h4>Sub-Channels:</h4>
+                    <h4 class="mt-2">Sub-Channels:</h4>
                     <table id="tablesubchannels" class="table">
                         <thead class="bg-dark text-white">
                         <tr>
@@ -123,7 +131,7 @@
                             <th>Created at</th>
                             <th>First Seen</th>
                             <th>Last Seen</th>
-                            <th>NB Messages</th>
+                            <th><i class="fas fa-comment-dots"></i></th>
                         </tr>
                         </thead>
                         <tbody style="font-size: 15px;">
@@ -132,7 +140,12 @@
                                 <td>
                                    <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">
                                 </td>
-                                <td><b>{{ meta['name'] }}</b></td>
+                                <td>
+                                    <b>{{ meta['name'] }}</b>
+                                    {% if meta['translation_name'] %}
+                                        <div class="text-secondary">{{ meta['translation_name'] }}</div>
+                                    {% endif %}
+                                </td>
                                 <td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>
                                 <td>{{ meta['created_at'] }}</td>
                                 <td>
@@ -161,9 +174,6 @@
                         {% include 'objects/image/block_blur_img_slider.html' %}
                     </span>
 
-                    {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}
-                        {% include 'chats_explorer/block_translation.html' %}
-                    {% endwith %}
                     {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb']  %}
                         {% set date_from=chat['first_seen'] %}
                         {% set date_to=chat['last_seen'] %}

var/www/templates/chats_explorer/user_account.html

@@ -46,7 +46,6 @@
                                     <tr>
                                         <th>username</th>
                                         <th>ID</th>
-                                        <th>Created at</th>
                                         <th>First Seen</th>
                                         <th>Last Seen</th>
                                         <th>NB Chats</th>
@@ -56,7 +55,6 @@
                                     <tr>
                                             <td>{{ meta['username']['id'] }}</td>
                                             <td>{{ meta['id']  }}</td>
-                                            <td>{{ meta['created_at'] }}</td>
                                             <td>
                                                 {% if meta['first_seen'] %}
                                                     {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}
@@ -74,6 +72,10 @@
                             {% if meta['info'] %}
                                 <li class="list-group-item py-0">
                                     <pre class="my-0">{{ meta['info'] }}</pre>
+                                    {% if meta['translation_info'] %}
+                                        <hr>
+                                        <pre class="my-0 text-secondary">{{ meta['translation_info'] }}</pre>
+                                    {% endif %}
                                 </li>
                             {% endif %}
                             </li>
@@ -100,6 +102,10 @@
                     </div>
                 </div>
 
+                {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
+                    {% include 'chats_explorer/block_translation.html' %}
+                {% endwith %}
+
 
 {#                {% if meta['subchannels'] %}#}
 {#                    <h4>Sub-Channels:</h4>#}