diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 8052bf28..6e77d5ea 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -17,6 +17,7 @@ from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") +r_lang = config_loader.get_db_conn("Kvrocks_Languages") TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate') config_loader = None @@ -256,9 +257,6 @@ def get_iso_from_languages(l_languages, sort=False): return l_iso -class LanguageDetector: - pass - def get_translator_instance(): return TRANSLATOR_URL @@ -299,25 +297,98 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True): content = content.replace(it, '') return content -#### AIL Objects #### +#### LANGUAGE ENGINE #### -def get_obj_translation(obj_global_id, content, field='', source=None, target='en'): +# first seen +# last seen +# language by date -> iter on object date ???? + +## Langs +def get_language_obj_types(language): + return r_lang.smembers(f'languages:{language}') + +def get_language_objs(language, obj_type, obj_subtype=''): + return r_lang.smembers(f'langs:{obj_type}:{obj_subtype}:{language}') + +# def get_languages_objs(languages, obj_type, obj_subtype='') + +## Objs +def get_objs_languages(obj_type, obj_subtype=''): + if obj_subtype: + return r_lang.smembers(f'objs:lang:{obj_type}:{obj_subtype}') + else: + return r_lang.smembers(f'objs:langs:{obj_type}') + +## Obj +def get_obj_languages(obj_type, obj_subtype, obj_id): + return r_lang.smembers(f'obj:lang:{obj_type}:{obj_subtype}:{obj_id}') + +# TODO ADD language to CHAT GLOBAL SET +def add_obj_language(language, obj_type, obj_subtype, obj_id): # (s) + if not obj_subtype: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + + r_lang.sadd(f'objs:langs:{obj_type}', language) + r_lang.sadd(f'objs:lang:{obj_type}:{obj_subtype}', language) + r_lang.sadd(f'obj:lang:{obj_global_id}', language) + + r_lang.sadd(f'languages:{language}', f'{obj_type}:{obj_subtype}') ################### REMOVE ME ??? + r_lang.sadd(f'langs:{obj_type}:{obj_subtype}:{language}', obj_global_id) + +def remove_obj_language(language, obj_type, obj_subtype, obj_id): + if not obj_subtype: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + r_lang.srem(f'obj:lang:{obj_global_id}', language) + + r_lang.srem(f'langs:{obj_type}:{obj_subtype}:{language}', obj_global_id) + if not r_lang.exists(f'langs:{obj_type}:{obj_subtype}:{language}'): + r_lang.srem(f'objs:lang:{obj_type}:{obj_subtype}', language) + r_lang.srem(f'languages:{language}', f'{obj_type}:{obj_subtype}') + if not r_lang.exists(f'objs:lang:{obj_type}:{obj_subtype}'): + if r_lang.scard(f'objs:langs:{obj_type}', language) <= 1: + r_lang.srem(f'objs:langs:{obj_type}', language) + +def edit_obj_language(language, obj_type, obj_subtype, obj_id): + remove_obj_language(language, obj_type, obj_subtype, obj_id) + add_obj_language(language, obj_type, obj_subtype, obj_id) + + +## Translation +def _get_obj_translation(obj_global_id, language, field=''): + return r_lang.hget(f'tr:{obj_global_id}:{field}', language) + +def get_obj_translation(obj_global_id, language, source=None, content=None, field=''): """ - Returns translated content + Returns translated content """ - translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}') + translation = r_cache.get(f'translation:{language}:{obj_global_id}:{field}') if translation: # DEBUG # print('cache') - # r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0) + # r_cache.expire(f'translation:{language}:{obj_global_id}:{field}', 0) return translation - translation = LanguageTranslator().translate(content, source=source, target=target) + # TODO HANDLE FIELDS TRANSLATION + translation = _get_obj_translation(obj_global_id, language, field=field) + if not translation: + translation = LanguageTranslator().translate(content, source=source, target=language) if translation: - r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation) - r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300) + r_cache.set(f'translation:{language}:{obj_global_id}:{field}', translation) + r_cache.expire(f'translation:{language}:{obj_global_id}:{field}', 300) return translation -## --AIL Objects-- ## + +# TODO Force to edit ???? +def set_obj_translation(obj_global_id, language, translation, field=''): + r_cache.delete(f'translation:{language}:{obj_global_id}:') + return r_lang.hset(f'tr:{obj_global_id}:{field}', language, translation) + + +## --LANGUAGE ENGINE-- ## + + +#### AIL Objects #### class LanguagesDetector: diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 07cd806f..2a60fb46 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -400,6 +400,19 @@ def api_get_message(message_id, translation_target=None): meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) return meta, 200 +def api_manually_translate_message(message_id, translation_target, translation): + message = Messages.Message(message_id) + if not message.exists(): + return {"status": "error", "reason": "Unknown uuid"}, 404 + if len(translation) > 200000: # TODO REVIEW LIMIT + return {"status": "error", "reason": "Max Size reached"}, 400 + if translation_target not in Language.get_translation_languages(): + return {"status": "error", "reason": "Unknown Language"}, 400 + if translation: + message.set_translation(translation_target, translation) + # TODO SANITYZE translation + return None, 200 + def api_get_user_account(user_id, instance_uuid, translation_target=None): user_account = UsersAccount.UserAccount(user_id, instance_uuid) if not user_account.exists(): diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 1a794614..551ad155 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -25,7 +25,7 @@ from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship -from lib.Language import get_obj_translation +from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, get_obj_translation, set_obj_translation from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -302,15 +302,31 @@ class AbstractObject(ABC): ## -Relationship- ## - ## Translation ## + ## Language ## + + def get_languages(self): + return get_obj_languages(self.type, self.get_subtype(r_str=True), self.id) + + def add_language(self, language): + return add_obj_language(language, self.type, self.get_subtype(r_str=True), self.id) + + def remove_language(self, language): + return remove_obj_language(language, self.type, self.get_subtype(r_str=True), self.id) + + def get_translation(self, language, field=''): + return get_obj_translation(self.get_global_id(), language, field=field) + + def set_translation(self, language, translation, field=''): + return set_obj_translation(self.get_global_id(), language, translation, field=field) def translate(self, content=None, field='', source=None, target='en'): global_id = self.get_global_id() if not content: content = self.get_content() - return get_obj_translation(global_id, content, field=field, source=source, target=target) + translation = get_obj_translation(global_id, target, source=source, content=content, field=field) + return translation - ## -Translation- ## + ## -Language- ## ## Parent ## diff --git a/configs/6383.conf b/configs/6383.conf index 0b889dbe..d4903431 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -903,6 +903,7 @@ namespace.cor ail_correls namespace.crawl ail_crawlers namespace.db ail_datas namespace.dup ail_dups +namespace.lg ail_langs namespace.obj ail_objs namespace.rel ail_rels namespace.stat ail_stats diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index f93ec08a..892b6f64 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -191,6 +191,11 @@ host = localhost port = 6383 password = ail_crawlers +[Kvrocks_Languages] +host = localhost +port = 6383 +password = ail_langs + [Kvrocks_Objects] host = localhost port = 6383 diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 5b834128..04b83d9d 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -221,6 +221,21 @@ def objects_message(): translation_languages=languages, translation_target=target, modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) +@chats_explorer.route("/objects/message/translate", methods=['POST']) +@login_required +@login_read_only +def objects_message_translate(): + message_id = request.form.get('id') + target = request.form.get('target') + translation = request.form.get('translation') + if target == "Don't Translate": + target = None + resp = chats_viewer.api_manually_translate_message(message_id, target, translation) + if resp[1] != 200: + return create_json_response(resp[0], resp[1]) + else: + return redirect(url_for('chats_explorer.objects_message', id=message_id, target=target)) + @chats_explorer.route("/objects/user-account", methods=['GET']) @login_required @login_read_only diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index cf407128..3343210a 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -80,6 +80,25 @@ {% if message['translation'] %}
{{ message['translation'] }}+ + {% set mess_id_escape= message['id'] | replace("/", "_") %} + +